Training in progress, step 1650, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73467ea1ed97e925adf2917879e548677b9d96e9d65d80d0ea25c65250831ebe
+oid sha256:1a1114471ad61d66a88788a2005b7be221cdbe9e2fd3f43e9a2185e89b6fad6f
 size 29034840

last-checkpoint/global_step1650/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5add4929f3324c01b1475ea9b18eccfde6d5e29582e4ea048971d60f47d99b0d
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e859eadc0d763a55600f40d12de6bd95bd61cccbe158c79627dbe4cc63ceabb
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f3696c6a828086b5118a7bd50fee1f0dbb10b579094d2cc5ae42f2a94ccadd
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3ccf96f842c5cf5843174e479e5dcff218568e900811adafc4334b0a0b5b143
+size 43429616

last-checkpoint/global_step1650/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c79466c87dc7a7bb2278fbdb456e69e6c6b0a01149fe0952fd362a1881b0c3
+size 637299

last-checkpoint/global_step1650/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4009b72c3e7da18b4a07fc0d3d8de5ac6d846834d93ff9946ac1044932cdd63
+size 637171

last-checkpoint/global_step1650/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a7cde2a2fb17884fd88468a6046779089bf8060a77027c24f561b10b27cca30
+size 637171

last-checkpoint/global_step1650/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9f660827bea0980ccb09f606308a98aeb8817e9ee50af4a17e22e579fd1e42d
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1600
+global_step1650

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
+oid sha256:a90384755f5b036b42b1a465b39dbf24a925a02c04294f9d684bc1de7f4db1e5
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
+oid sha256:7621b41e55056218f97d5b32ae116de3304a677b9f27b6a62170d83a2bbff176
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
+oid sha256:997e9debadfd125b5c8b66ee6dd79ced3d40d353ff9250475f3814fd950012a6
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
+oid sha256:0f0256032419959580948d742425f66782bc8eb029126a091669a42c6ee0eba4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62282c037a983d19544e509ed880c9744baa4fc67a0800fdb043c257f3c8ac9a
+oid sha256:40442cc35519e6454a07ba94c151e6fd6557a2a4ddd9970fc5771865c93790aa
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.28714123368263245,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1550",
-  "epoch": 0.4726735598227474,
+  "epoch": 0.4874446085672083,
   "eval_steps": 50,
-  "global_step": 1600,
+  "global_step": 1650,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2855,11 +2855,100 @@
       "eval_steps_per_second": 0.777,
       "num_input_tokens_seen": 16609960,
       "step": 1600
+    },
+    {
+      "epoch": 0.4741506646971935,
+      "grad_norm": 11.101854016846067,
+      "learning_rate": 9.07388845507224e-05,
+      "loss": 0.3644,
+      "num_input_tokens_seen": 16661440,
+      "step": 1605
+    },
+    {
+      "epoch": 0.4756277695716396,
+      "grad_norm": 12.638527847469565,
+      "learning_rate": 9.066795715040825e-05,
+      "loss": 0.2733,
+      "num_input_tokens_seen": 16714200,
+      "step": 1610
+    },
+    {
+      "epoch": 0.4771048744460857,
+      "grad_norm": 2.7966971298188694,
+      "learning_rate": 9.059678712546963e-05,
+      "loss": 0.3063,
+      "num_input_tokens_seen": 16766904,
+      "step": 1615
+    },
+    {
+      "epoch": 0.47858197932053176,
+      "grad_norm": 8.210735089414575,
+      "learning_rate": 9.052537490050614e-05,
+      "loss": 0.3769,
+      "num_input_tokens_seen": 16818168,
+      "step": 1620
+    },
+    {
+      "epoch": 0.48005908419497784,
+      "grad_norm": 8.942406690882898,
+      "learning_rate": 9.045372090156243e-05,
+      "loss": 0.3089,
+      "num_input_tokens_seen": 16869952,
+      "step": 1625
+    },
+    {
+      "epoch": 0.4815361890694239,
+      "grad_norm": 11.68569569670719,
+      "learning_rate": 9.038182555612551e-05,
+      "loss": 0.2953,
+      "num_input_tokens_seen": 16922608,
+      "step": 1630
+    },
+    {
+      "epoch": 0.48301329394387,
+      "grad_norm": 17.89504289247946,
+      "learning_rate": 9.030968929312231e-05,
+      "loss": 0.3286,
+      "num_input_tokens_seen": 16974824,
+      "step": 1635
+    },
+    {
+      "epoch": 0.4844903988183161,
+      "grad_norm": 1.1612341646192372,
+      "learning_rate": 9.023731254291705e-05,
+      "loss": 0.3552,
+      "num_input_tokens_seen": 17026088,
+      "step": 1640
+    },
+    {
+      "epoch": 0.4859675036927622,
+      "grad_norm": 4.624302477495485,
+      "learning_rate": 9.016469573730869e-05,
+      "loss": 0.326,
+      "num_input_tokens_seen": 17077904,
+      "step": 1645
+    },
+    {
+      "epoch": 0.4874446085672083,
+      "grad_norm": 1.126074328380494,
+      "learning_rate": 9.009183930952836e-05,
+      "loss": 0.2698,
+      "num_input_tokens_seen": 17130896,
+      "step": 1650
+    },
+    {
+      "epoch": 0.4874446085672083,
+      "eval_loss": 0.40201568603515625,
+      "eval_runtime": 19.4814,
+      "eval_samples_per_second": 3.08,
+      "eval_steps_per_second": 0.77,
+      "num_input_tokens_seen": 17130896,
+      "step": 1650
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 16609960,
+  "num_input_tokens_seen": 17130896,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2874,7 +2963,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1095801178423296.0,
+  "total_flos": 1130167189372928.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null