Training in progress, step 500

checkpoint-100/latest

@@ -1 +0,0 @@
-global_step100

checkpoint-100/trainer_state.json

@@ -1,76 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.03333333333333333,
-  "global_step": 100,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.0,
-      "learning_rate": 0.0003,
-      "loss": 9.4994,
-      "step": 10
-    },
-    {
-      "epoch": 0.01,
-      "learning_rate": 0.0003,
-      "loss": 7.7223,
-      "step": 20
-    },
-    {
-      "epoch": 0.01,
-      "learning_rate": 0.0003,
-      "loss": 7.3713,
-      "step": 30
-    },
-    {
-      "epoch": 0.01,
-      "learning_rate": 0.0003,
-      "loss": 7.2695,
-      "step": 40
-    },
-    {
-      "epoch": 0.02,
-      "learning_rate": 0.0003,
-      "loss": 7.2811,
-      "step": 50
-    },
-    {
-      "epoch": 0.02,
-      "learning_rate": 0.0003,
-      "loss": 7.2291,
-      "step": 60
-    },
-    {
-      "epoch": 0.02,
-      "learning_rate": 0.0003,
-      "loss": 7.1263,
-      "step": 70
-    },
-    {
-      "epoch": 0.03,
-      "learning_rate": 0.0003,
-      "loss": 7.0881,
-      "step": 80
-    },
-    {
-      "epoch": 0.03,
-      "learning_rate": 0.0003,
-      "loss": 6.5989,
-      "step": 90
-    },
-    {
-      "epoch": 0.03,
-      "learning_rate": 0.0003,
-      "loss": 6.0091,
-      "step": 100
-    }
-  ],
-  "max_steps": 3000,
-  "num_train_epochs": 9223372036854775807,
-  "total_flos": 41875931136000.0,
-  "trial_name": null,
-  "trial_params": null
-}

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fe1a55e447470a69df52a08a8a6c31e9a156978a4ba2a625c8a30702b84488d
+oid sha256:784c0410b2a330cc433f981f1f5121bebabb1f3c1d781da96b8ef18c2e0db0c5
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bd87bf4c0c7c127cd0eb3ca94ae8ce5f8df7c07968f2625452f262034d79b04
+oid sha256:ab7a186e1928203b48ef1c8148819f63ee35e5e289f3d284193685d064f813fb
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38e715b0c78de79c8c4274d18035541dc36006f12d7eb04db4fb4d051be9c65a
+oid sha256:84d6ee17959a269d6d3e3b64d1b432370d3b086eb26b333e9a61a532db3253be
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c4a28cf894bb5c203707bb5f458ca5180f20e561bf60496dac39b246ec8ab60
+oid sha256:c55b861ec085a2534503a9e5de7134d033ee8cb3625c35db2f3528370ae95c8d
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d812020511b6b6863b40a143452d94b2d9019f6bae0b386cfce3bdc9a8b522b5
+oid sha256:c905c2c37c0f2f94f4a5ae2d10d79032961adea61c10e1cbb2614f6e20d67fda
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:074a945d31ce70971139ebc0522d768a0e93721d8c011cac7f0a70070cd32ede
+oid sha256:4824286c51d845b36a99e1044983ef94a086ad5d007203581ec617fd89edb112
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95eb8c9c22f6e04c73e889c9064eaf020d163a78b9d9bfbde0fddb0d1e3c77cd
+oid sha256:70ae89087b135570ef769d71356ac53cb1162eeecd4e647c2ea4069b8e19094b
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41217fdfa05706b0ba5ec07a6668750369d829aa48488053279099d2c42dda93
+oid sha256:83612a4a188eb3093e946a69b95f9a0c3ddbdfe58f70767b69309388acb7cc83
 size 10107626487

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c132881e587ed89040fabcb6e0487cda037e7e63cc0e402e2256ac72d9df210
+oid sha256:d8f31bdab7ff163280101e1923f64561cf4a6d974c46612977fe58c68ac04e22
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bfaad6c2340d432686efd4237fadcfea88c5e8917fc21b82322ef3c50fb33aa
+oid sha256:3129eceffca8659953e6eac5b948129a0984740d232358cc99e9dab7d4426c27
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e8660f1738925a1b5009f1c37b11854cf879e34ece11e4da2b8a18ccc95dec9
+oid sha256:251ecd563c2937490b8ba62da01d1c0bdd9597f24d5d788954b9c7073385b0ba
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9df8089958af2d5c7ac712c33503ecd4ad01b734acb1bedd9160ca10db525a7
+oid sha256:541652ce8ea40c93f93845b200b752fd459ea34dc30b036e763b7c0bbfe00041
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:462e29cf69d3535709cabb7bfb58d3ca268590048ac7fc4c2ae1f326058e58c3
+oid sha256:35ea0ff60f07ad502cd98ed64c92b5bf21728b26f8ec4ad810862d72b85b770f
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c85beff6bc0047809de0ea778d57221d319ac97e0628ff2449ab761c4a2a0a59
+oid sha256:85e23b7fcb735641a5db63cc2d01383e208488fea01050c3b293389297ce7410
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebca5e9a54db9920ece454e1e56264f9066a382467c5ed9d1dc9224615abca56
+oid sha256:a80c0240d46abab60a4dda1cd710e6cd7d2379e125e99d7f698dc8732ee786ed
 size 168086

{checkpoint-100/global_step100 → checkpoint-500/global_step500}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5a4e8e62fb989ae6d935bce6357e8482a8b690b389444481585b41b2ee56251
+oid sha256:fdeed66d696bbb7c4c3335a09232fd9fad0c780a4b1c78f93d28188e80446add
 size 168086

checkpoint-500/latest

@@ -0,0 +1 @@
+global_step500

{checkpoint-100 → checkpoint-500}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3930d3a7fa35b76e46303c3755740429a6f81598f49260a3a5d24ba4072cc58
+oid sha256:be1ee9759da30984c76dadddd1d8c7c94c3758ab4a57cf347cff3fa275944df4
 size 9976576392

{checkpoint-100 → checkpoint-500}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3375a48a87312ddf013e22779c80f47d877865d88ac39cfc1d7a625d513af7d8
+oid sha256:3c8789b1f2769d0cd20c1574875ba9b08d058a97d8713b8a2a36e33e481dd3d1
 size 3500296504

checkpoint-500/trainer_state.json

@@ -0,0 +1,316 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.16666666666666666,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003,
+      "loss": 9.4994,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003,
+      "loss": 7.7223,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003,
+      "loss": 7.3713,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003,
+      "loss": 7.2695,
+      "step": 40
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0003,
+      "loss": 7.2811,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0003,
+      "loss": 7.2291,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0003,
+      "loss": 7.1263,
+      "step": 70
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0003,
+      "loss": 7.0881,
+      "step": 80
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0003,
+      "loss": 6.5989,
+      "step": 90
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0003,
+      "loss": 6.0091,
+      "step": 100
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0003,
+      "loss": 5.5511,
+      "step": 110
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0003,
+      "loss": 5.3413,
+      "step": 120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0003,
+      "loss": 5.1775,
+      "step": 130
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0003,
+      "loss": 5.104,
+      "step": 140
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0003,
+      "loss": 5.0133,
+      "step": 150
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0003,
+      "loss": 4.9843,
+      "step": 160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0003,
+      "loss": 4.9256,
+      "step": 170
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0003,
+      "loss": 4.8692,
+      "step": 180
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0003,
+      "loss": 4.9233,
+      "step": 190
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003,
+      "loss": 4.9139,
+      "step": 200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003,
+      "loss": 4.8974,
+      "step": 210
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003,
+      "loss": 4.9036,
+      "step": 220
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.8635,
+      "step": 230
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.7883,
+      "step": 240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.7681,
+      "step": 250
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.7572,
+      "step": 260
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.7552,
+      "step": 270
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.706,
+      "step": 280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.7015,
+      "step": 290
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.695,
+      "step": 300
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.6808,
+      "step": 310
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.6423,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.613,
+      "step": 330
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.5851,
+      "step": 340
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.5882,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6228,
+      "step": 360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6269,
+      "step": 370
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.5364,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4992,
+      "step": 390
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4799,
+      "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.3733,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.1788,
+      "step": 420
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 3.6706,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.8767,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3927,
+      "step": 450
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3062,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.2299,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1739,
+      "step": 480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1303,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.1092,
+      "step": 500
+    }
+  ],
+  "max_steps": 3000,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 209379655680000.0,
+  "trial_name": null,
+  "trial_params": null
+}

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbac0f36a764288570496a92b7a47a3fb69f7d7bbb9586a63782011ab9d573a0
-size 10431
+oid sha256:7125e059489f5648613066e3ee25f5b3db395f89a0c3ea565999ab8326362e4e
+size 12001