Training in progress, step 16000, checkpoint

last-checkpoint/global_step16000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74a25234c1f780e15e83309158ecb07c49c8c3351f6b3d5ebfbdf10c3d34478a
+size 197282509

last-checkpoint/global_step16000/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b74f02b0b5a6b6d5843ca46bc53062ab759eed1584bc29a69cffa6a1736ec15
+size 180416968

last-checkpoint/global_step16000/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff92b38172cd635a23c777b6b6eefb0da4d30754fdf34e7d0630ddca213f14e
+size 180416776

last-checkpoint/global_step16000/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11febf62c9f73266c25236ab50dfab41409f97bdb56adbf4d4edf021564aa534
+size 180416776

last-checkpoint/global_step16000/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e31ccf4cfa0c45c8d9396cff77752649a49b0f35441d5d054bece6e2267085
+size 180416904

last-checkpoint/global_step16000/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b1074e1d46fa74656185ee0b7b31dcbb97450cf0590305861d259d6c3219803
+size 180416712

last-checkpoint/global_step16000/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cd2bae6b2bd55970d2a0ec1ce3814d6bc32e58d931c3bb34290e3f28f220dab
+size 180417096

last-checkpoint/latest

@@ -1 +1 @@
-global_step15500
+global_step16000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbbfba19cf8c4868fcb591e428153e0dfab01e0000556ed74857ebf4d4d0b09c
+oid sha256:86cdbcf702f36b103531085178dbb194b5ed24a617c693786569c5f06a28a997
 size 188836816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e532e7f78ce215481f83407b0560a37e2979f8d3b5916593aae7cfa436c82ca8
+oid sha256:1762a051d2ef63782191c6ddb670046fd603a79f169ade2a20e3ec7968d73ae2
 size 15536

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54deac076126a8a5ab13f060f479d26e551e362b17517f4b1320311a9393ea91
+oid sha256:528307c815a7fc3ae0e940360ffd3bbc2afb3e407b6c50c9bc322b05f89b5a94
 size 15536

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1caa5002a3d632d2580f3767aeb17e7dcb9094ef0f0cec447e0bbe64dba963f
+oid sha256:94b4c77667b44a108f2c4524ecb78b00f15dea830fc559e3dd27f09695096d4e
 size 15536

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df07a8cc413b3fabbbcb751fedf4cd462f781f2168fcd682f0e751cf69c3eb1b
+oid sha256:a2af1fcdccef02a571531525d33c8c2ab59e4a0d3036f87ff6b31fa9cbedb4da
 size 15536

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e047643b78223ae0aa07482134e20fe15812cc1e4c689a1b49ea591c7cdc5750
+oid sha256:89c6761be70409b46c72197709a9276709a6e1e13c93978aa3d44a91c5dd5845
 size 15536

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b303e24ad7fee581c0e7fb49dd8e1426edcd88dd63fef654033f2a15ec87a19a
-size 15472
+oid sha256:ecd7e3c5995ac03f69428fef9fdfff61483fb597ae67d40f95f01eb5737db5e4
+size 15536

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2272465e853654bd253a9da64924193b698e19218c7dbd82f58e52260bc06930
+oid sha256:90507ecd8d24438d3d06f3d874381f825e91ab823f24d3428100127026ea819d
 size 1256

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 418.9189189189189,
+  "epoch": 432.43243243243245,
   "eval_steps": 1500,
-  "global_step": 15500,
+  "global_step": 16000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2258,6 +2258,76 @@
       "learning_rate": 0.0001,
       "loss": 0.0129,
       "step": 15500
+    },
+    {
+      "epoch": 420.27027027027026,
+      "grad_norm": 0.35932862758636475,
+      "learning_rate": 0.0001,
+      "loss": 0.0133,
+      "step": 15550
+    },
+    {
+      "epoch": 421.6216216216216,
+      "grad_norm": 0.20093189179897308,
+      "learning_rate": 0.0001,
+      "loss": 0.0126,
+      "step": 15600
+    },
+    {
+      "epoch": 422.97297297297297,
+      "grad_norm": 0.32909420132637024,
+      "learning_rate": 0.0001,
+      "loss": 0.0128,
+      "step": 15650
+    },
+    {
+      "epoch": 424.3243243243243,
+      "grad_norm": 0.28278329968452454,
+      "learning_rate": 0.0001,
+      "loss": 0.0117,
+      "step": 15700
+    },
+    {
+      "epoch": 425.6756756756757,
+      "grad_norm": 0.1597350388765335,
+      "learning_rate": 0.0001,
+      "loss": 0.0119,
+      "step": 15750
+    },
+    {
+      "epoch": 427.02702702702703,
+      "grad_norm": 0.20241086184978485,
+      "learning_rate": 0.0001,
+      "loss": 0.0114,
+      "step": 15800
+    },
+    {
+      "epoch": 428.3783783783784,
+      "grad_norm": 0.24632301926612854,
+      "learning_rate": 0.0001,
+      "loss": 0.0115,
+      "step": 15850
+    },
+    {
+      "epoch": 429.72972972972974,
+      "grad_norm": 0.36104726791381836,
+      "learning_rate": 0.0001,
+      "loss": 0.0114,
+      "step": 15900
+    },
+    {
+      "epoch": 431.0810810810811,
+      "grad_norm": 0.23273630440235138,
+      "learning_rate": 0.0001,
+      "loss": 0.0115,
+      "step": 15950
+    },
+    {
+      "epoch": 432.43243243243245,
+      "grad_norm": 0.2528134882450104,
+      "learning_rate": 0.0001,
+      "loss": 0.0113,
+      "step": 16000
     }
   ],
   "logging_steps": 50,
@@ -2277,7 +2347,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.351020525227858e+20,
+  "total_flos": 4.491310429887309e+20,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null