Training in progress, step 1500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e95045ab82949901554f82423b881753088f0bf319f5b2eb855236faa8d4d90c
+oid sha256:6289479e907da413d0601e67681f7846ac9804602b3a5c4cb53469f8420fd060
 size 29034840

last-checkpoint/global_step1500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d32c3db9bd47d3fd301d19afb9e65be568f162a529e5b395d56b562bff90f3fa
+size 43429616

last-checkpoint/global_step1500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e95b6c551561f8e28d2d5cae23096c9d3ed6b6aa830cd5d690389680532849
+size 43429616

last-checkpoint/global_step1500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7ecafa7b19a9899877a2ee1cf62b924940aea99ddbe0a2e41c1d2da49d51d50
+size 43429616

last-checkpoint/global_step1500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:999e1eaa71432819202d4d7c2c587bfe6e38a266275c2be56de7c4e6781a2677
+size 43429616

last-checkpoint/global_step1500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e26a054806a08fcc4699bcbb4ba16ad8b8ce9bd251f3f4f2ddea66428a6514f3
+size 637299

last-checkpoint/global_step1500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e6e56cb084d84e264a8def4093445029bf6f220c440facaf6f823c4ff482e1
+size 637171

last-checkpoint/global_step1500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd13d26848f42e1fe0b5c0bc25ada85bb5a6562b98a05e03d77b9a1bb5695c65
+size 637171

last-checkpoint/global_step1500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dca9d87505fd347b5f4b1f42a713081819b869c0819eaac51f4120208f87027
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1450
+global_step1500

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd4f3298d54e9509917373bcd359e11c92a8e0aa77b2cc0825602efd186ad77e
+oid sha256:dd3566049ad1f65d2f434d990deb65584d2b2dcb1aac8e89c68ea37dc533eab7
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fdab02a791039ff46df83a272c972ab0f366fcea91338790dc016dbbbf6de80
+oid sha256:2f96a47dc4176412bc893ccb49c004c8fa1cc8c306d67689d87ed20944233c62
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2808762a5f80587f05c1e2e8de26b5aef1010331cea366649319d593dbb0e66
+oid sha256:ebd9fa3db1079ccf750b71f4eeedbf1f04422fc748026a4b866afe133f9fbfd1
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c052112cbe3f325fd4543a02558091b80c9c8abcd8db8822be9a8306eab6f2b7
+oid sha256:383e9e252cc8292eef0120c964bdc8033972e800c085c97c42af97379e6b4b5c
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9dc71bb3c767f5227b4aaf564240b2395077bbb61533f057d89b79d0ef0db35
+oid sha256:f31de0059cfffab00767f1a3eadee4b6350e0f39836feff5d5d9263e375ce9aa
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2966395914554596,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1200",
-  "epoch": 0.42836041358936483,
+  "epoch": 0.4431314623338257,
   "eval_steps": 50,
-  "global_step": 1450,
+  "global_step": 1500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2588,11 +2588,100 @@
       "eval_steps_per_second": 0.775,
       "num_input_tokens_seen": 15050992,
       "step": 1450
+    },
+    {
+      "epoch": 0.4298375184638109,
+      "grad_norm": 5.756692171427544,
+      "learning_rate": 9.275183991651558e-05,
+      "loss": 0.3253,
+      "num_input_tokens_seen": 15103328,
+      "step": 1455
+    },
+    {
+      "epoch": 0.43131462333825704,
+      "grad_norm": 48.99329970671894,
+      "learning_rate": 9.268838142516943e-05,
+      "loss": 0.3999,
+      "num_input_tokens_seen": 15154640,
+      "step": 1460
+    },
+    {
+      "epoch": 0.4327917282127031,
+      "grad_norm": 4.8123322290098764,
+      "learning_rate": 9.262466825536782e-05,
+      "loss": 0.3529,
+      "num_input_tokens_seen": 15206264,
+      "step": 1465
+    },
+    {
+      "epoch": 0.4342688330871492,
+      "grad_norm": 8.626202826826654,
+      "learning_rate": 9.256070078722287e-05,
+      "loss": 0.3363,
+      "num_input_tokens_seen": 15258160,
+      "step": 1470
+    },
+    {
+      "epoch": 0.4357459379615953,
+      "grad_norm": 24.191365994287917,
+      "learning_rate": 9.249647940236385e-05,
+      "loss": 0.4133,
+      "num_input_tokens_seen": 15309224,
+      "step": 1475
+    },
+    {
+      "epoch": 0.43722304283604135,
+      "grad_norm": 15.447487032462668,
+      "learning_rate": 9.243200448393492e-05,
+      "loss": 0.3306,
+      "num_input_tokens_seen": 15361480,
+      "step": 1480
+    },
+    {
+      "epoch": 0.43870014771048743,
+      "grad_norm": 2.714376790067382,
+      "learning_rate": 9.236727641659277e-05,
+      "loss": 0.2993,
+      "num_input_tokens_seen": 15414680,
+      "step": 1485
+    },
+    {
+      "epoch": 0.4401772525849335,
+      "grad_norm": 11.85875780519428,
+      "learning_rate": 9.230229558650442e-05,
+      "loss": 0.3324,
+      "num_input_tokens_seen": 15466552,
+      "step": 1490
+    },
+    {
+      "epoch": 0.44165435745937964,
+      "grad_norm": 7.338287245679936,
+      "learning_rate": 9.223706238134485e-05,
+      "loss": 0.2615,
+      "num_input_tokens_seen": 15519472,
+      "step": 1495
+    },
+    {
+      "epoch": 0.4431314623338257,
+      "grad_norm": 11.54360838796349,
+      "learning_rate": 9.217157719029469e-05,
+      "loss": 0.2928,
+      "num_input_tokens_seen": 15572048,
+      "step": 1500
+    },
+    {
+      "epoch": 0.4431314623338257,
+      "eval_loss": 0.40494996309280396,
+      "eval_runtime": 19.2568,
+      "eval_samples_per_second": 3.116,
+      "eval_steps_per_second": 0.779,
+      "num_input_tokens_seen": 15572048,
+      "step": 1500
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 15050992,
+  "num_input_tokens_seen": 15572048,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2607,7 +2696,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 992909032161280.0,
+  "total_flos": 1027290982514688.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null