Training in progress, step 12500, checkpoint

last-checkpoint/global_step12500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d471f4d4e89a53746b0e4a77b195db83faf739af77bf5baec751b0fb175acc6
+size 197282509

last-checkpoint/global_step12500/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7eaf9773d725fdb93f06baf3dc7e73f199a813f44e0b1d39af31e10f200d549
+size 180416968

last-checkpoint/global_step12500/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12a3531002d674e9e9bc13b3d11ccd44c3f04eff8843f7d15670890e4bdb391d
+size 180416776

last-checkpoint/global_step12500/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4650ebb550dd159fc85dc4057ed8f347f71bb91103d7547fdf9c57742299984
+size 180416776

last-checkpoint/global_step12500/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e2a6205ea3f09a2469f8ed6b8b95fabc4c0313b35371e3dfb4becc615f33c9
+size 180416904

last-checkpoint/global_step12500/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fcc6ed5077225dfbc3ba935b9353af036bf0b9a9cb642b0257712b9167d6c2c
+size 180416712

last-checkpoint/global_step12500/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e96d7a6ffec4f1d7591687afcfbdb6ec6d72fd25a7696dc1c62d712439a5af62
+size 180417096

last-checkpoint/latest

@@ -1 +1 @@
-global_step12000
+global_step12500

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3eb524258c867572c5919743a319ba3f9fd58d56945160073acf2004ff2a347
+oid sha256:24b88f70912df99acfaa93c847d111588238f2f3c89f9126b68e97bd70f7fbb5
 size 188836816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d4eb7bb9203a8e87629e01b60e46978419c91258dd1b91a531ac85a815aa2a6
-size 15536
+oid sha256:271c89ce6ee409be2ea9aa3bf701ff66e7a7cb9264b73eefc11e757372a91e5e
+size 15472

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc6279e9322e24454c048154973915312c4b82bc7072d6a884572f22541832c7
+oid sha256:41eea4946eeaa383e314ebd734ddda2972689f784d350627374fb05cf2b160ab
 size 15536

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac94466e1b648ce8b7f170dae606c7a2937de2716494049583e11afa457a9c6f
-size 15536
+oid sha256:eccd0e35b1a3fb6b4961836705b55374915efc0fe4aa8bbcd2fcc5652edb5192
+size 15472

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79616bee9b7b5ede30d31d95265feb076065b5289cf77fde96901f070bc79757
-size 15536
+oid sha256:1155b24d828f1b4cc9581115f945c46af6cafa8b0dcd3abce9639d3f64bf6fa7
+size 15472

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd553d6ac7842152a5181c58c8e33c096716a917f28c6b50289f0d04c7e92d11
-size 15536
+oid sha256:d24951e84bf84c9244c0763d9d797270b2b1f7662cfade2d3d2824db197031c1
+size 15472

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40e25258d05927a3c348374e136c22c574766db63da03125f9f306341a45778c
-size 15536
+oid sha256:baa66995193ecfcc4672c56a227acadd5b4de025d048272586e219600d9fb650
+size 15472

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79e5daf4f6059fe7c5db5e2882115706bf42b98796f6165f76b8c067e144ae88
+oid sha256:9f500d7068793f6d489446a9d3939e8a5fbc68fd372f8eeb2605b20dd2e2ab5a
 size 1256

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 324.3243243243243,
+  "epoch": 337.8378378378378,
   "eval_steps": 1500,
-  "global_step": 12000,
+  "global_step": 12500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1750,6 +1750,76 @@
       "eval_steps_per_second": 0.288,
       "eval_wer": 0.19910846953937592,
       "step": 12000
+    },
+    {
+      "epoch": 325.6756756756757,
+      "grad_norm": 0.28629496693611145,
+      "learning_rate": 0.0001,
+      "loss": 0.0159,
+      "step": 12050
+    },
+    {
+      "epoch": 327.02702702702703,
+      "grad_norm": 0.24716606736183167,
+      "learning_rate": 0.0001,
+      "loss": 0.0152,
+      "step": 12100
+    },
+    {
+      "epoch": 328.3783783783784,
+      "grad_norm": 0.2562699019908905,
+      "learning_rate": 0.0001,
+      "loss": 0.0153,
+      "step": 12150
+    },
+    {
+      "epoch": 329.72972972972974,
+      "grad_norm": 0.27679792046546936,
+      "learning_rate": 0.0001,
+      "loss": 0.0147,
+      "step": 12200
+    },
+    {
+      "epoch": 331.0810810810811,
+      "grad_norm": 0.2266552597284317,
+      "learning_rate": 0.0001,
+      "loss": 0.0149,
+      "step": 12250
+    },
+    {
+      "epoch": 332.43243243243245,
+      "grad_norm": 0.2453828752040863,
+      "learning_rate": 0.0001,
+      "loss": 0.0151,
+      "step": 12300
+    },
+    {
+      "epoch": 333.7837837837838,
+      "grad_norm": 0.3448384702205658,
+      "learning_rate": 0.0001,
+      "loss": 0.014,
+      "step": 12350
+    },
+    {
+      "epoch": 335.13513513513516,
+      "grad_norm": 0.20089378952980042,
+      "learning_rate": 0.0001,
+      "loss": 0.0148,
+      "step": 12400
+    },
+    {
+      "epoch": 336.4864864864865,
+      "grad_norm": 0.2895062267780304,
+      "learning_rate": 0.0001,
+      "loss": 0.016,
+      "step": 12450
+    },
+    {
+      "epoch": 337.8378378378378,
+      "grad_norm": 0.3491511642932892,
+      "learning_rate": 0.0001,
+      "loss": 0.0153,
+      "step": 12500
     }
   ],
   "logging_steps": 50,
@@ -1769,7 +1839,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.3684338707834234e+20,
+  "total_flos": 3.5089183189071246e+20,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null