Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a07ff07864099795f0ef4d933d28ad9679ab08369382487621535abb56f31408
+oid sha256:ac561155f14c161be89688cbd6791c190ab41f1ce825521ecbfa6d19d6a0c5dc
 size 577859976

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa549d0c6d2379df26a07b7c3966ffb132bd6ae13022a5b02c772d9255bfb476
+oid sha256:101e64149047c3d744fdcb436b5dba2ca98b22115ce1eff72b44b64343dda473
 size 1155913545

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d49f728245aa60e4fc9c5baaee9c0ba88b2cadcc8481faf5625d828b7dcb728
+oid sha256:2b87c169cee0ea524ed1c27bb8054d01ebdc65124a471acea18a35e7102d6ec2
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d73d3175aefaa69aadf2bfd7a28a272f3f909699b9e8910d2a5b7143d3350e78
+oid sha256:860361ada6eb369eac489bd111b66d88a5634713630c0b874711d0b984bd4e88
 size 1064

last-checkpoint/spm_char.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae27ad61041da617aec8ffe1c20890111b46b4a3cbb00984dac63f733f1004b9
+oid sha256:b531f1de35ab0170af3d23ae8a6e847481a4d0a40cde9036ea8160d511bf6b16
 size 238738

last-checkpoint/trainer_state.json

@@ -1,169 +1,66 @@
 {
-  "best_metric": 0.4254090189933777,
-  "best_model_checkpoint": "./speecht5-tts-vie\\checkpoint-10000",
-  "epoch": 20.408163265306122,
+  "best_metric": 0.436745822429657,
+  "best_model_checkpoint": "./speecht5-tts-vie\\checkpoint-5000",
+  "epoch": 2.7487630566245187,
   "eval_steps": 1000,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 2.0408163265306123,
-      "grad_norm": 1.8910266160964966,
-      "learning_rate": 8.316666666666668e-06,
-      "loss": 0.7092,
+      "epoch": 0.5497526113249038,
+      "eval_loss": 0.5397089123725891,
+      "eval_runtime": 246.0666,
+      "eval_samples_per_second": 50.669,
+      "eval_steps_per_second": 3.17,
       "step": 1000
     },
     {
-      "epoch": 2.0408163265306123,
-      "eval_loss": 0.5270479917526245,
-      "eval_runtime": 389.8964,
-      "eval_samples_per_second": 17.212,
-      "eval_steps_per_second": 0.539,
-      "step": 1000
-    },
-    {
-      "epoch": 4.081632653061225,
-      "grad_norm": 1.5691834688186646,
-      "learning_rate": 1.665e-05,
-      "loss": 0.5566,
-      "step": 2000
-    },
-    {
-      "epoch": 4.081632653061225,
-      "eval_loss": 0.48652538657188416,
-      "eval_runtime": 379.2783,
-      "eval_samples_per_second": 17.694,
-      "eval_steps_per_second": 0.554,
+      "epoch": 1.0995052226498077,
+      "eval_loss": 0.5124471187591553,
+      "eval_runtime": 233.1296,
+      "eval_samples_per_second": 53.481,
+      "eval_steps_per_second": 3.346,
       "step": 2000
     },
     {
-      "epoch": 6.122448979591836,
-      "grad_norm": 4.319777488708496,
-      "learning_rate": 2.4983333333333335e-05,
-      "loss": 0.5055,
-      "step": 3000
-    },
-    {
-      "epoch": 6.122448979591836,
-      "eval_loss": 0.4476528465747833,
-      "eval_runtime": 376.1339,
-      "eval_samples_per_second": 17.842,
-      "eval_steps_per_second": 0.558,
+      "epoch": 1.6492578339747115,
+      "eval_loss": 0.4633851647377014,
+      "eval_runtime": 232.6741,
+      "eval_samples_per_second": 53.586,
+      "eval_steps_per_second": 3.352,
       "step": 3000
     },
     {
-      "epoch": 8.16326530612245,
-      "grad_norm": 3.316411256790161,
-      "learning_rate": 3.3316666666666666e-05,
-      "loss": 0.4789,
-      "step": 4000
-    },
-    {
-      "epoch": 8.16326530612245,
-      "eval_loss": 0.4305626153945923,
-      "eval_runtime": 379.5072,
-      "eval_samples_per_second": 17.683,
-      "eval_steps_per_second": 0.553,
+      "epoch": 2.1990104452996153,
+      "eval_loss": 0.4454704225063324,
+      "eval_runtime": 233.2342,
+      "eval_samples_per_second": 53.457,
+      "eval_steps_per_second": 3.344,
       "step": 4000
     },
     {
-      "epoch": 10.204081632653061,
-      "grad_norm": 2.388119697570801,
+      "epoch": 2.7487630566245187,
+      "grad_norm": 3.8025379180908203,
       "learning_rate": 4.1641666666666665e-05,
-      "loss": 0.4636,
+      "loss": 0.5624,
       "step": 5000
     },
     {
-      "epoch": 10.204081632653061,
-      "eval_loss": 0.42910683155059814,
-      "eval_runtime": 403.7863,
-      "eval_samples_per_second": 16.62,
-      "eval_steps_per_second": 0.52,
+      "epoch": 2.7487630566245187,
+      "eval_loss": 0.436745822429657,
+      "eval_runtime": 231.6026,
+      "eval_samples_per_second": 53.834,
+      "eval_steps_per_second": 3.368,
       "step": 5000
-    },
-    {
-      "epoch": 12.244897959183673,
-      "grad_norm": 1.92360520362854,
-      "learning_rate": 4.9975e-05,
-      "loss": 0.4555,
-      "step": 6000
-    },
-    {
-      "epoch": 12.244897959183673,
-      "eval_loss": 0.42244505882263184,
-      "eval_runtime": 379.3426,
-      "eval_samples_per_second": 17.691,
-      "eval_steps_per_second": 0.554,
-      "step": 6000
-    },
-    {
-      "epoch": 14.285714285714286,
-      "grad_norm": 2.303443670272827,
-      "learning_rate": 5.83e-05,
-      "loss": 0.4471,
-      "step": 7000
-    },
-    {
-      "epoch": 14.285714285714286,
-      "eval_loss": 0.4228420555591583,
-      "eval_runtime": 470.6545,
-      "eval_samples_per_second": 14.259,
-      "eval_steps_per_second": 0.446,
-      "step": 7000
-    },
-    {
-      "epoch": 16.3265306122449,
-      "grad_norm": 1.5780209302902222,
-      "learning_rate": 6.663333333333333e-05,
-      "loss": 0.4421,
-      "step": 8000
-    },
-    {
-      "epoch": 16.3265306122449,
-      "eval_loss": 0.42195653915405273,
-      "eval_runtime": 458.745,
-      "eval_samples_per_second": 14.629,
-      "eval_steps_per_second": 0.458,
-      "step": 8000
-    },
-    {
-      "epoch": 18.367346938775512,
-      "grad_norm": 2.230262041091919,
-      "learning_rate": 7.495e-05,
-      "loss": 0.438,
-      "step": 9000
-    },
-    {
-      "epoch": 18.367346938775512,
-      "eval_loss": 0.4210755228996277,
-      "eval_runtime": 449.0465,
-      "eval_samples_per_second": 14.945,
-      "eval_steps_per_second": 0.468,
-      "step": 9000
-    },
-    {
-      "epoch": 20.408163265306122,
-      "grad_norm": 2.0010855197906494,
-      "learning_rate": 8.328333333333333e-05,
-      "loss": 0.4375,
-      "step": 10000
-    },
-    {
-      "epoch": 20.408163265306122,
-      "eval_loss": 0.4254090189933777,
-      "eval_runtime": 433.4203,
-      "eval_samples_per_second": 15.484,
-      "eval_steps_per_second": 0.485,
-      "step": 10000
     }
   ],
-  "logging_steps": 1000,
+  "logging_steps": 5000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 205,
-  "save_steps": 10000,
+  "num_train_epochs": 55,
+  "save_steps": 5000,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -176,8 +73,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.444146373783773e+16,
-  "train_batch_size": 32,
+  "total_flos": 7921363059826128.0,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95f25db4da551fc09cd5e5e24eb8b5d820beb00de5789c705e05ff4264602603
+oid sha256:512bd16045d9f54ac4b4eaf8dda145bb7ffb9b33ba3722d48f942a6a81d08d16
 size 5304