Training in progress, step 9000, checkpoint

last-checkpoint/global_step9000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:679e19d21666fbd176c864156b8c7c5845f88b7b624f9f324e9eb22bab3c33a8
+size 197282509

last-checkpoint/global_step9000/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6356d5f6dbd695e172a73eb0cd4421c7df86c325fd2f348d4dd4f6b845ac6360
+size 180416968

last-checkpoint/global_step9000/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daa070acd5ce24cc23b06b7e3a05c59cfe018dedd7e0549cb622be5c6e28c3ee
+size 180416776

last-checkpoint/global_step9000/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:919f84c4ecb818f8992b84076e993b1f58c302d515e1a4ae2b55e824c168f7bf
+size 180416776

last-checkpoint/global_step9000/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c1939255a8747c77a8d857a22c8263f74db18d6ae16f0202ffd6e0337c417d
+size 180416904

last-checkpoint/global_step9000/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3107ded7bc86a29baf90b127f260aa7a678cf5b4b3a2a5e5a6dd2a12337bfa1c
+size 180416712

last-checkpoint/global_step9000/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:830ba11f654bbe488f96b088b48dfdea45f74335584890df96d7dfcd32adbaa9
+size 180417096

last-checkpoint/latest

@@ -1 +1 @@
-global_step8500
+global_step9000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7d95a2a6398a3976684f9f296988a5e39dde3afb33e0d969993bcbd7dcacaca
+oid sha256:2ab65fcd6eef541b0eab0651d0717f1badda42d7fd92bfb4075b331da2037828
 size 188836816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:564a5d159c6784b209d5ba2261c4de8c25ee8803b9c92f51fd1e9fd52c1b3fc4
+oid sha256:04a7c505ffdf7349443f4571264a08aa5d00e509401833cfbc8ee36e3f78ae54
 size 15536

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cbf33bcff4e9bc7aead0f12ef6877a895f2040493ae419349b5a3b110b5036e
+oid sha256:95d7ca58acb8d3876c3a7f75f1478ea294c4dc3f3a656bcf55937f21b39bafeb
 size 15536

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ce0b2876ec405f01219dce5c2551d896b5f10a920f11726ba040105793eaabb
+oid sha256:2b892c9e25d561484c4d46a997b69468880c163891890ad309278d371355e9ba
 size 15536

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59a83285a5f0517959619b1f7f7827cf51e67685851a9aed6f10647402c51355
-size 15472
+oid sha256:a8a81bb1b67d6e9cc674626ce2fca3b1db95526692490a7cca8768ee12d6e2e7
+size 15536

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d45bda445d230c33252857d7ba14a68c6656232ceb63109a76eba6e8e0d43306
+oid sha256:bfa5c5c42cdaccb27d45e096ddb8bbb4eafd2783ecc504d34c4ed6a4e70b1732
 size 15472

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:976477c91777116536dc7d0dffbe92366832796b5f9e9ba3c59be4799747b19a
+oid sha256:abd2f2fb50fa21042e610837b6b9a95060d6b9c41e23ab7ce19a6228e9dd5bb9
 size 15536

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b71db0e061be5d6991df2e521349de0fd5a5ff99a4bfc1c920bc3894a6254c9f
+oid sha256:dc228f89b1f6600cf303d1bda56efe53cce614e2f06ad7a65194b3b38bd74bd7
 size 1256

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 229.72972972972974,
+  "epoch": 243.24324324324326,
   "eval_steps": 1500,
-  "global_step": 8500,
+  "global_step": 9000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1233,6 +1233,85 @@
       "learning_rate": 0.0001,
       "loss": 0.0225,
       "step": 8500
+    },
+    {
+      "epoch": 231.0810810810811,
+      "grad_norm": 0.31986966729164124,
+      "learning_rate": 0.0001,
+      "loss": 0.0217,
+      "step": 8550
+    },
+    {
+      "epoch": 232.43243243243242,
+      "grad_norm": 0.27697187662124634,
+      "learning_rate": 0.0001,
+      "loss": 0.0205,
+      "step": 8600
+    },
+    {
+      "epoch": 233.78378378378378,
+      "grad_norm": 0.3078053593635559,
+      "learning_rate": 0.0001,
+      "loss": 0.0185,
+      "step": 8650
+    },
+    {
+      "epoch": 235.13513513513513,
+      "grad_norm": 0.24676857888698578,
+      "learning_rate": 0.0001,
+      "loss": 0.0202,
+      "step": 8700
+    },
+    {
+      "epoch": 236.48648648648648,
+      "grad_norm": 0.2980283498764038,
+      "learning_rate": 0.0001,
+      "loss": 0.0202,
+      "step": 8750
+    },
+    {
+      "epoch": 237.83783783783784,
+      "grad_norm": 0.34748488664627075,
+      "learning_rate": 0.0001,
+      "loss": 0.0188,
+      "step": 8800
+    },
+    {
+      "epoch": 239.1891891891892,
+      "grad_norm": 0.31379759311676025,
+      "learning_rate": 0.0001,
+      "loss": 0.0195,
+      "step": 8850
+    },
+    {
+      "epoch": 240.54054054054055,
+      "grad_norm": 0.31512585282325745,
+      "learning_rate": 0.0001,
+      "loss": 0.0197,
+      "step": 8900
+    },
+    {
+      "epoch": 241.8918918918919,
+      "grad_norm": 0.28801149129867554,
+      "learning_rate": 0.0001,
+      "loss": 0.0188,
+      "step": 8950
+    },
+    {
+      "epoch": 243.24324324324326,
+      "grad_norm": 0.29776033759117126,
+      "learning_rate": 0.0001,
+      "loss": 0.0196,
+      "step": 9000
+    },
+    {
+      "epoch": 243.24324324324326,
+      "eval_loss": 0.6647829413414001,
+      "eval_runtime": 22.108,
+      "eval_samples_per_second": 70.879,
+      "eval_steps_per_second": 0.226,
+      "eval_wer": 0.19295266397792402,
+      "step": 9000
     }
   ],
   "logging_steps": 50,
@@ -1252,7 +1331,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.3861586914904637e+20,
+  "total_flos": 2.5263079974749025e+20,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null