Training in progress, step 14000, checkpoint

last-checkpoint/global_step14000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02bebdd5de7c1d9d90fd26218dd96282e121d09b9159d013a3569c5ae2c27fb4
+size 197282509

last-checkpoint/global_step14000/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ae4fc9e42b351442985cc36fc81d83793b3e473a6f6865a668706310202596
+size 180416968

last-checkpoint/global_step14000/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22e28d2d81cadfbfa12f1daa7195939246db3a1882547be253c1065d67b53a35
+size 180416776

last-checkpoint/global_step14000/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2177a4547d04661db0774eed72bb7cb123a6920000ee365b2db13d38253268a
+size 180416776

last-checkpoint/global_step14000/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5df125d700c5947f620d5a131df8d5d4a4008f3e5cef4ffb68cae3654ce17846
+size 180416904

last-checkpoint/global_step14000/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c6054940157be9517d6c0fb4a1b3db1fd611b318ec63baaec52c87627def3b2
+size 180416712

last-checkpoint/global_step14000/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fba5f1db40a076f08bcec8c4e7a63747bfc2c5b8fae2ba648851d7414b9049d
+size 180417096

last-checkpoint/latest

@@ -1 +1 @@
-global_step13500
+global_step14000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f47d9162e02ba41e897ddef94a76ffaa3207a4df8da19436801ac9e0c88a914
+oid sha256:bc76e91f7c430955a9c99095113fcd1babc25289a48cfc4ca0d735607be3435b
 size 188836816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:788efdf3708b2732ff69f48cfcf90acb73209cbf37f4011afd17f780037d0bde
-size 15472
+oid sha256:d1693ed0aa550d028c5296d44f56e801f84a1280db6a97f42dfe34e34110ae4f
+size 15536

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02511e2c03e0fc687eab778c597ffd93ac9524f885cceb188795c890db9da40f
+oid sha256:11ff77e693e531050a6d1812f76b9871e17f20f68e215508928ca64785e5251e
 size 15472

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac10c908fb6bec1972b4bdb5a09206b46f0da880c841bad70dfeea31a2d3dc43
+oid sha256:d62b18b7eb0aded1dd39102c6afe436923d8a04ed3a046654b037fa2fdb91a70
 size 15536

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b640e5fc41f6d806cfd29c4c5ae9a3eaaad7dc279270fec7c85758b95f0d0fe2
-size 15472
+oid sha256:8a7769491a119bd1e33a2abce5162152cd680e687dbe03d7959eadbc1484228e
+size 15536

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4050cd1167c153e66f8ca000461cc46a739bb2b257520cff1a327f7c1619281
-size 15472
+oid sha256:be5c0e12d74adafdf73a4301f0fd601009322deaaa04b1bdc85c19322d09712b
+size 15536

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7c6478909adce0c04574166ab664c21b92e3341c20752151283cf5134c7a672
-size 15536
+oid sha256:0a6fbc22976e4200d2dfe699e869535d064502235a2c6d93f138d76d6ce8e9c2
+size 15472

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d1c4e3be38cfe4c157c13a4bf7e6948b0f6fc4113c7fedf2b2ada4c3fee3150
+oid sha256:1bcf66ac1e7571a4edc6b80653947ebcc0595575727313ab43eebe76504b10ff
 size 1256

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 364.86486486486484,
+  "epoch": 378.3783783783784,
   "eval_steps": 1500,
-  "global_step": 13500,
+  "global_step": 14000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1969,6 +1969,76 @@
       "eval_steps_per_second": 0.286,
       "eval_wer": 0.20101889195499895,
       "step": 13500
+    },
+    {
+      "epoch": 366.2162162162162,
+      "grad_norm": 0.2017332762479782,
+      "learning_rate": 0.0001,
+      "loss": 0.0134,
+      "step": 13550
+    },
+    {
+      "epoch": 367.56756756756755,
+      "grad_norm": 0.26539239287376404,
+      "learning_rate": 0.0001,
+      "loss": 0.0148,
+      "step": 13600
+    },
+    {
+      "epoch": 368.9189189189189,
+      "grad_norm": 0.2736688256263733,
+      "learning_rate": 0.0001,
+      "loss": 0.0148,
+      "step": 13650
+    },
+    {
+      "epoch": 370.27027027027026,
+      "grad_norm": 0.28902319073677063,
+      "learning_rate": 0.0001,
+      "loss": 0.0137,
+      "step": 13700
+    },
+    {
+      "epoch": 371.6216216216216,
+      "grad_norm": 0.1861814558506012,
+      "learning_rate": 0.0001,
+      "loss": 0.0132,
+      "step": 13750
+    },
+    {
+      "epoch": 372.97297297297297,
+      "grad_norm": 0.2393738031387329,
+      "learning_rate": 0.0001,
+      "loss": 0.0125,
+      "step": 13800
+    },
+    {
+      "epoch": 374.3243243243243,
+      "grad_norm": 0.3993573486804962,
+      "learning_rate": 0.0001,
+      "loss": 0.0125,
+      "step": 13850
+    },
+    {
+      "epoch": 375.6756756756757,
+      "grad_norm": 0.3024432361125946,
+      "learning_rate": 0.0001,
+      "loss": 0.0134,
+      "step": 13900
+    },
+    {
+      "epoch": 377.02702702702703,
+      "grad_norm": 0.35330072045326233,
+      "learning_rate": 0.0001,
+      "loss": 0.0143,
+      "step": 13950
+    },
+    {
+      "epoch": 378.3783783783784,
+      "grad_norm": 0.21859917044639587,
+      "learning_rate": 0.0001,
+      "loss": 0.014,
+      "step": 14000
     }
   ],
   "logging_steps": 50,
@@ -1988,7 +2058,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.789600807493776e+20,
+  "total_flos": 3.929919945418631e+20,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null