Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27d48542b9818e4e4a293d1d8c9dda9547b0bc96dbdc8160cf051a53ac7d0273
+oid sha256:028bf0a6547388360f9d61c31c3d8c9ca8de5904332d974377c9e95b583d383e
 size 577859976

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d86043f2600663b53b4e1f7c4d965a4b0603ae1690e1cc9d9132974bf6057dfe
+oid sha256:bcf8b26ebf1a2190706ede053bdef8e0b54454417751cf199f3280cb7f98d6ae
 size 1155913545

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d18ddaeceed99e79aa4ac5c182e18d8899aa358265dfca0c3e14dbc0359a6800
+oid sha256:d3a189c587f71772a90681cf9309062314d7cbe24d1b7f1e4297726490f4dbbb
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:237b16a7ce9f6b5a014af36041f5a3bce3b97b7e8b3060da10452bd5dd7eb0ee
+oid sha256:6fa224d225c0288a6e0a18c72b41fbc853d1c25b9331613205b078bf1090ba2c
 size 1064

last-checkpoint/trainer_state.json

@@ -1,254 +1,49 @@
 {
-  "best_metric": 0.41689586639404297,
-  "best_model_checkpoint": "./speecht5-tts-vie\\checkpoint-20000",
-  "epoch": 13.743815283122595,
-  "eval_steps": 1000,
-  "global_step": 25000,
+  "best_metric": NaN,
+  "best_model_checkpoint": "./speecht5-tts-vie\\checkpoint-10000",
+  "epoch": 5.497526113249038,
+  "eval_steps": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.5497526113249038,
-      "eval_loss": 0.5397089123725891,
-      "eval_runtime": 246.0666,
-      "eval_samples_per_second": 50.669,
-      "eval_steps_per_second": 3.17,
-      "step": 1000
-    },
-    {
-      "epoch": 1.0995052226498077,
-      "eval_loss": 0.5124471187591553,
-      "eval_runtime": 233.1296,
-      "eval_samples_per_second": 53.481,
-      "eval_steps_per_second": 3.346,
-      "step": 2000
-    },
-    {
-      "epoch": 1.6492578339747115,
-      "eval_loss": 0.4633851647377014,
-      "eval_runtime": 232.6741,
-      "eval_samples_per_second": 53.586,
-      "eval_steps_per_second": 3.352,
-      "step": 3000
-    },
-    {
-      "epoch": 2.1990104452996153,
-      "eval_loss": 0.4454704225063324,
-      "eval_runtime": 233.2342,
-      "eval_samples_per_second": 53.457,
-      "eval_steps_per_second": 3.344,
-      "step": 4000
-    },
     {
       "epoch": 2.7487630566245187,
-      "grad_norm": 3.8025379180908203,
-      "learning_rate": 4.1641666666666665e-05,
-      "loss": 0.5624,
+      "grad_norm": NaN,
+      "learning_rate": 0.0016583333333333333,
+      "loss": 1.0616,
       "step": 5000
     },
     {
       "epoch": 2.7487630566245187,
-      "eval_loss": 0.436745822429657,
-      "eval_runtime": 231.6026,
-      "eval_samples_per_second": 53.834,
-      "eval_steps_per_second": 3.368,
+      "eval_loss": NaN,
+      "eval_runtime": 332.2788,
+      "eval_samples_per_second": 37.523,
+      "eval_steps_per_second": 2.347,
       "step": 5000
     },
-    {
-      "epoch": 3.298515667949423,
-      "eval_loss": 0.43301576375961304,
-      "eval_runtime": 231.9457,
-      "eval_samples_per_second": 53.754,
-      "eval_steps_per_second": 3.363,
-      "step": 6000
-    },
-    {
-      "epoch": 3.8482682792743264,
-      "eval_loss": 0.43043527007102966,
-      "eval_runtime": 232.7937,
-      "eval_samples_per_second": 53.558,
-      "eval_steps_per_second": 3.351,
-      "step": 7000
-    },
-    {
-      "epoch": 4.398020890599231,
-      "eval_loss": 0.42659255862236023,
-      "eval_runtime": 232.6331,
-      "eval_samples_per_second": 53.595,
-      "eval_steps_per_second": 3.353,
-      "step": 8000
-    },
-    {
-      "epoch": 4.947773501924134,
-      "eval_loss": 0.4237697124481201,
-      "eval_runtime": 335.3241,
-      "eval_samples_per_second": 37.182,
-      "eval_steps_per_second": 2.326,
-      "step": 9000
-    },
     {
       "epoch": 5.497526113249038,
-      "grad_norm": 3.957423210144043,
-      "learning_rate": 8.329166666666667e-05,
-      "loss": 0.4668,
+      "grad_norm": NaN,
+      "learning_rate": 0.0016583333333333333,
+      "loss": 0.0,
       "step": 10000
     },
     {
       "epoch": 5.497526113249038,
-      "eval_loss": 0.43070951104164124,
-      "eval_runtime": 338.8195,
-      "eval_samples_per_second": 36.798,
-      "eval_steps_per_second": 2.302,
+      "eval_loss": NaN,
+      "eval_runtime": 309.5607,
+      "eval_samples_per_second": 40.276,
+      "eval_steps_per_second": 2.52,
       "step": 10000
-    },
-    {
-      "epoch": 6.047278724573942,
-      "eval_loss": 0.42637017369270325,
-      "eval_runtime": 337.6097,
-      "eval_samples_per_second": 36.93,
-      "eval_steps_per_second": 2.31,
-      "step": 11000
-    },
-    {
-      "epoch": 6.597031335898846,
-      "eval_loss": 0.4261023998260498,
-      "eval_runtime": 351.5943,
-      "eval_samples_per_second": 35.461,
-      "eval_steps_per_second": 2.218,
-      "step": 12000
-    },
-    {
-      "epoch": 7.146783947223749,
-      "eval_loss": 0.42608416080474854,
-      "eval_runtime": 335.0357,
-      "eval_samples_per_second": 37.214,
-      "eval_steps_per_second": 2.328,
-      "step": 13000
-    },
-    {
-      "epoch": 7.696536558548653,
-      "eval_loss": 0.4257422685623169,
-      "eval_runtime": 334.2636,
-      "eval_samples_per_second": 37.3,
-      "eval_steps_per_second": 2.333,
-      "step": 14000
-    },
-    {
-      "epoch": 8.246289169873556,
-      "grad_norm": 1.6813654899597168,
-      "learning_rate": 9.659886363636365e-05,
-      "loss": 0.4563,
-      "step": 15000
-    },
-    {
-      "epoch": 8.246289169873556,
-      "eval_loss": 0.4219256639480591,
-      "eval_runtime": 336.8999,
-      "eval_samples_per_second": 37.008,
-      "eval_steps_per_second": 2.315,
-      "step": 15000
-    },
-    {
-      "epoch": 8.796041781198461,
-      "eval_loss": 0.42107194662094116,
-      "eval_runtime": 339.383,
-      "eval_samples_per_second": 36.737,
-      "eval_steps_per_second": 2.298,
-      "step": 16000
-    },
-    {
-      "epoch": 9.345794392523365,
-      "eval_loss": 0.42018377780914307,
-      "eval_runtime": 336.8885,
-      "eval_samples_per_second": 37.009,
-      "eval_steps_per_second": 2.315,
-      "step": 17000
-    },
-    {
-      "epoch": 9.895547003848268,
-      "eval_loss": 0.4306410551071167,
-      "eval_runtime": 338.8623,
-      "eval_samples_per_second": 36.794,
-      "eval_steps_per_second": 2.302,
-      "step": 18000
-    },
-    {
-      "epoch": 10.445299615173171,
-      "eval_loss": 0.4173683226108551,
-      "eval_runtime": 382.8942,
-      "eval_samples_per_second": 32.563,
-      "eval_steps_per_second": 2.037,
-      "step": 19000
-    },
-    {
-      "epoch": 10.995052226498077,
-      "grad_norm": 2.4157445430755615,
-      "learning_rate": 9.091931818181819e-05,
-      "loss": 0.4432,
-      "step": 20000
-    },
-    {
-      "epoch": 10.995052226498077,
-      "eval_loss": 0.41689586639404297,
-      "eval_runtime": 399.9359,
-      "eval_samples_per_second": 31.175,
-      "eval_steps_per_second": 1.95,
-      "step": 20000
-    },
-    {
-      "epoch": 11.54480483782298,
-      "eval_loss": 0.4168870151042938,
-      "eval_runtime": 381.0473,
-      "eval_samples_per_second": 32.72,
-      "eval_steps_per_second": 2.047,
-      "step": 21000
-    },
-    {
-      "epoch": 12.094557449147883,
-      "eval_loss": 0.42138534784317017,
-      "eval_runtime": 372.9871,
-      "eval_samples_per_second": 33.427,
-      "eval_steps_per_second": 2.091,
-      "step": 22000
-    },
-    {
-      "epoch": 12.644310060472787,
-      "eval_loss": 0.42539912462234497,
-      "eval_runtime": 449.8771,
-      "eval_samples_per_second": 27.714,
-      "eval_steps_per_second": 1.734,
-      "step": 23000
-    },
-    {
-      "epoch": 13.194062671797692,
-      "eval_loss": 0.41935691237449646,
-      "eval_runtime": 477.984,
-      "eval_samples_per_second": 26.085,
-      "eval_steps_per_second": 1.632,
-      "step": 24000
-    },
-    {
-      "epoch": 13.743815283122595,
-      "grad_norm": 1.6161022186279297,
-      "learning_rate": 8.523863636363637e-05,
-      "loss": 0.4325,
-      "step": 25000
-    },
-    {
-      "epoch": 13.743815283122595,
-      "eval_loss": 0.41808393597602844,
-      "eval_runtime": 457.4883,
-      "eval_samples_per_second": 27.253,
-      "eval_steps_per_second": 1.705,
-      "step": 25000
     }
   ],
   "logging_steps": 5000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 55,
-  "save_steps": 5000,
+  "save_steps": 10000,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -261,7 +56,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.959974797228648e+16,
+  "total_flos": 1.5859575977955336e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:512bd16045d9f54ac4b4eaf8dda145bb7ffb9b33ba3722d48f942a6a81d08d16
+oid sha256:3ff7b688be8240deab18d13754e0ca0ed159df97ff4667f041eadd1a25f5182d
 size 5304