Training in progress, step 2400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e895c1cddd40eb0dbe387456240309b06d66fe014fd793905368c3b37bbbff4a
+oid sha256:724b21694b2108c1eedcec3aba788b5559aed494f8af9352c85fe7551b55f69c
 size 29034840

last-checkpoint/global_step2400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d6f8ac48914dc4e7a50fa8cbfe71666cbf805142b9aec062a3a058e0e40e6b4
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8447d79e1990e976bf439b347c7e02eaec26da92a21ead32bee85b8426c2fbe
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bce02bc6a8b1239b3d666121512b78ca7e2cf74a6eb51c2ee283f4bd9c6afcaa
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf5efa42fc216ceaf721bcd6bd557cebcfb1be06ad53cfcae73b2caa1ecea8aa
+size 43429616

last-checkpoint/global_step2400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb28cb80645e4e5b08d929feb6cb284b80b9374d0abb5cac7a034e04e045bf67
+size 637299

last-checkpoint/global_step2400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e18a2b0b8ad83bd9cd5698849f6aafc46f23435824e7200c2e85a9eb4e492a1
+size 637171

last-checkpoint/global_step2400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc63c2624e3a26f46cb4e76250e1daf40e4de23f6d0694e319d1c3d09e6246cc
+size 637171

last-checkpoint/global_step2400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9469a2da3dcd4ac7538c85ff2efc3e7edb38b7bb9ddacc4ebbcca32aabfa5fe
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2350
+global_step2400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f12bf3da75454e5aae4644f2a1d46fdf90f68e680dbf5bdaa86861f825d32d80
+oid sha256:681faefe4cf303ca7f9bc3073b09b166da4f558d55bee3d5eee90ba5d83159bb
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2faef1b8798e7516fd96ee7b3363866a8f97ca2d0ec5a8dd27bbfe70b0c6a733
+oid sha256:c0fb250f1d137fb55cef85743a342508178b4fe3a20c6793c82e279730ea280b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f35f6d27fbd414dd4285d91816b37fb6b97ff10fbba4c074d56ad2a7f723033
+oid sha256:34a0d7cfaa34f7e3738b4ef4989d693ed7864fed3b2a44ef1b6892fdcf026bb9
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea5c64dfc7e3b2729065483dfef8e4bb0af0d9bae32df888d258ee3c2859d676
+oid sha256:2a82aabe23bc62e289ef7d075c79f353bbc81286ec0f8964eabda4209d630e10
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8d0442374b2f8618569ef196bfe7955aa6b552a4eb29da5972e99b6fb6a7877
+oid sha256:9534ff0afb419b8df69ec6dd9eaf407a5fa510c1fc235b2fb5d794e6ea2faecd
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.6942392909896603,
+  "epoch": 0.7090103397341211,
   "eval_steps": 50,
-  "global_step": 2350,
+  "global_step": 2400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4190,11 +4190,100 @@
       "eval_steps_per_second": 0.769,
       "num_input_tokens_seen": 24401256,
       "step": 2350
+    },
+    {
+      "epoch": 0.6957163958641064,
+      "grad_norm": 1.2627287884293426,
+      "learning_rate": 7.765017944892514e-05,
+      "loss": 0.2499,
+      "num_input_tokens_seen": 24453384,
+      "step": 2355
+    },
+    {
+      "epoch": 0.6971935007385525,
+      "grad_norm": 1.476393664852338,
+      "learning_rate": 7.754834350812765e-05,
+      "loss": 0.2132,
+      "num_input_tokens_seen": 24505960,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6986706056129985,
+      "grad_norm": 3.9395522365245226,
+      "learning_rate": 7.744634321418906e-05,
+      "loss": 0.2049,
+      "num_input_tokens_seen": 24559008,
+      "step": 2365
+    },
+    {
+      "epoch": 0.7001477104874446,
+      "grad_norm": 2.1895254551218115,
+      "learning_rate": 7.734417917564211e-05,
+      "loss": 0.2222,
+      "num_input_tokens_seen": 24611128,
+      "step": 2370
+    },
+    {
+      "epoch": 0.7016248153618907,
+      "grad_norm": 2.3733905521598353,
+      "learning_rate": 7.724185200199643e-05,
+      "loss": 0.2678,
+      "num_input_tokens_seen": 24662336,
+      "step": 2375
+    },
+    {
+      "epoch": 0.7031019202363368,
+      "grad_norm": 6.930201726148201,
+      "learning_rate": 7.713936230373491e-05,
+      "loss": 0.2888,
+      "num_input_tokens_seen": 24714032,
+      "step": 2380
+    },
+    {
+      "epoch": 0.7045790251107829,
+      "grad_norm": 1.2305357870388025,
+      "learning_rate": 7.703671069231007e-05,
+      "loss": 0.2609,
+      "num_input_tokens_seen": 24765296,
+      "step": 2385
+    },
+    {
+      "epoch": 0.7060561299852289,
+      "grad_norm": 1.3830979392967684,
+      "learning_rate": 7.693389778014037e-05,
+      "loss": 0.2654,
+      "num_input_tokens_seen": 24816744,
+      "step": 2390
+    },
+    {
+      "epoch": 0.707533234859675,
+      "grad_norm": 1.641476112098168,
+      "learning_rate": 7.683092418060664e-05,
+      "loss": 0.2231,
+      "num_input_tokens_seen": 24869320,
+      "step": 2395
+    },
+    {
+      "epoch": 0.7090103397341211,
+      "grad_norm": 9.651892844655563,
+      "learning_rate": 7.672779050804834e-05,
+      "loss": 0.262,
+      "num_input_tokens_seen": 24921872,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7090103397341211,
+      "eval_loss": 0.34713664650917053,
+      "eval_runtime": 18.9635,
+      "eval_samples_per_second": 3.164,
+      "eval_steps_per_second": 0.791,
+      "num_input_tokens_seen": 24921872,
+      "step": 2400
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 24401256,
+  "num_input_tokens_seen": 24921872,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4209,7 +4298,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1609869937344512.0,
+  "total_flos": 1644216510578688.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null