Training in progress, step 250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85664231c2a3e543092a1181741d5d9eac1439a1c33e6b98767b64913ffe2a3f
+oid sha256:1b185d60d1a95b109d35cc8c5048fbf846bc58f2e1aee39250a914046d9a6821
 size 29034840

last-checkpoint/global_step250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e089e624dcb03c9a557addae98e28a1ff99ed8c27affc524f6b83c84f63aa30b
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21e3b552e6bc86cd418b1e5c73503e2c02f4fc76cf42fcc842672223ea8deb43
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f2fab74918c8708d742b0da69de2aacb74a49fb4ce82cc39ada283c47bc0a76
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6efb3b2ba11216a55634ea32419ee2c65d9e487a36e2b0b2d5120de21d1f009a
+size 43429616

last-checkpoint/global_step250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e80b196c408fdbb93be47104d2e2ebb39806a8e2f671eb97b21520c1760537b
+size 637299

last-checkpoint/global_step250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50783d840386ee88fb64fd6d7bb5cc3c88c65818b6030a5cb92b901026aa8a7b
+size 637171

last-checkpoint/global_step250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a9975bb0ef92a03508b158d7fea0b886e853ff0adb21cca084bd76368bdf9a
+size 637171

last-checkpoint/global_step250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76693721fccb62522fa35aaa24f5d6f23796a4391cc1cbb825c3e42813ab3ab8
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step200
+global_step250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
+oid sha256:418a5f105ae834c3075024076916b2a9475918fe034c12d0dd5b6d91f1aba467
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
+oid sha256:6e07ace389d24bc1307b74f42a1e7b8f0117b0db853e2df64ff3f15cb92916a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
+oid sha256:da6a990f346d7014dffb28fa2bc7d3b890bd3c53712503fce3656da48d3d6e50
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
+oid sha256:e95f356ca38179b05993f55daece0223e96fa10b9a1b9ea2102a739211333f63
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e05d920ea4f4a1582a5e47a75c3d6324d03507ef9a1a2a59cabb1a001a2756e0
+oid sha256:95727ac30e88a4b409e48fc76a5fc825943310a473f6f375c508be1f196dccaa
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.8917127847671509,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-100",
-  "epoch": 0.059084194977843424,
+  "best_metric": 0.810763955116272,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-250",
+  "epoch": 0.07385524372230429,
   "eval_steps": 50,
-  "global_step": 200,
+  "global_step": 250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -363,11 +363,100 @@
       "eval_steps_per_second": 0.786,
       "num_input_tokens_seen": 2071176,
       "step": 200
+    },
+    {
+      "epoch": 0.060561299852289516,
+      "grad_norm": 3.485842845243708,
+      "learning_rate": 6.0471976401179945e-05,
+      "loss": 0.8075,
+      "num_input_tokens_seen": 2122328,
+      "step": 205
+    },
+    {
+      "epoch": 0.0620384047267356,
+      "grad_norm": 1.9945831681716613,
+      "learning_rate": 6.194690265486725e-05,
+      "loss": 0.8207,
+      "num_input_tokens_seen": 2174744,
+      "step": 210
+    },
+    {
+      "epoch": 0.06351550960118169,
+      "grad_norm": 2.5224102035468907,
+      "learning_rate": 6.342182890855458e-05,
+      "loss": 0.7867,
+      "num_input_tokens_seen": 2227136,
+      "step": 215
+    },
+    {
+      "epoch": 0.06499261447562776,
+      "grad_norm": 2.7703394460607833,
+      "learning_rate": 6.48967551622419e-05,
+      "loss": 0.8256,
+      "num_input_tokens_seen": 2278568,
+      "step": 220
+    },
+    {
+      "epoch": 0.06646971935007386,
+      "grad_norm": 4.676590157125056,
+      "learning_rate": 6.637168141592921e-05,
+      "loss": 0.7897,
+      "num_input_tokens_seen": 2330224,
+      "step": 225
+    },
+    {
+      "epoch": 0.06794682422451995,
+      "grad_norm": 1.6101062596215647,
+      "learning_rate": 6.784660766961653e-05,
+      "loss": 0.792,
+      "num_input_tokens_seen": 2381344,
+      "step": 230
+    },
+    {
+      "epoch": 0.06942392909896603,
+      "grad_norm": 3.202676293331468,
+      "learning_rate": 6.932153392330384e-05,
+      "loss": 0.8309,
+      "num_input_tokens_seen": 2432136,
+      "step": 235
+    },
+    {
+      "epoch": 0.07090103397341212,
+      "grad_norm": 1.3573723032246008,
+      "learning_rate": 7.079646017699115e-05,
+      "loss": 0.7974,
+      "num_input_tokens_seen": 2483568,
+      "step": 240
+    },
+    {
+      "epoch": 0.0723781388478582,
+      "grad_norm": 1.317885929036595,
+      "learning_rate": 7.227138643067847e-05,
+      "loss": 0.7739,
+      "num_input_tokens_seen": 2535040,
+      "step": 245
+    },
+    {
+      "epoch": 0.07385524372230429,
+      "grad_norm": 2.1810508869311067,
+      "learning_rate": 7.374631268436578e-05,
+      "loss": 0.7558,
+      "num_input_tokens_seen": 2587272,
+      "step": 250
+    },
+    {
+      "epoch": 0.07385524372230429,
+      "eval_loss": 0.810763955116272,
+      "eval_runtime": 19.1642,
+      "eval_samples_per_second": 3.131,
+      "eval_steps_per_second": 0.783,
+      "num_input_tokens_seen": 2587272,
+      "step": 250
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 2071176,
+  "num_input_tokens_seen": 2587272,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -382,7 +471,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 136642668789760.0,
+  "total_flos": 170656960086016.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null