Training in progress, step 2850, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a4775218e4d9519c7e7224a3c5f1838a68e0cfaca11bd2bcf9f97934c96fd4c
+oid sha256:ab786f741c7a478f92cbba3fdd1ddd3a418946d605848f775e0f6c469c96daf4
 size 29034840

last-checkpoint/global_step2850/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b5b453f38fcf173d68fbe320a1a7bf275430b7d48755f860ca69f16bc356f95
+size 43429616

last-checkpoint/global_step2850/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a7160fa6a1407580521ec65a1b29324097b978a3c27d363c96c593a0daa14d6
+size 43429616

last-checkpoint/global_step2850/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04f16f8be40bf5eb2c746a7889a4ed4d5cf4c745979d0ca3973891ffbb59c98
+size 43429616

last-checkpoint/global_step2850/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b5f386bfaa75a642ab7a72875028cfc9b4828e33a2fc974caa8432fd25f912
+size 43429616

last-checkpoint/global_step2850/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6812b284fbf3417e77fd86a27d65a3311729d5aa0ad1760be1c080039b6a22c
+size 637299

last-checkpoint/global_step2850/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04a961275b937b710afe74a7a3b9fd6a401f9924641e566c39654df096230565
+size 637171

last-checkpoint/global_step2850/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911e755a27628af9831ca453ecfd7f1f364d53b6ef98c992957c2fffb659a58c
+size 637171

last-checkpoint/global_step2850/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20554ce78011d819cef737623a9e66ee3291f99e51e1ff343f472c5d5f41a0bb
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2800
+global_step2850

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49db5a9fd0c84d580c671e52905ebeffc155b36537e76ff966d2e82906708999
+oid sha256:ff8dba2341c0517760edfde50521977f02a5bd982ffd3bc03de6109439c4f478
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8ca224562d8d97aaa131b3516288bb99f68d7dcf62170494326662bda0bb206
+oid sha256:d2bf831df9fbade9ac2a8db79798bc2a7b1afb85a78a6e463ec7a7db4acc0f8e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56e86a11c89dba78d60e1b2a1855a651b90a5a22ef131ce65d26af83668c154e
+oid sha256:8602ff0a0fa366d46b61c0ef2b23ce468387898cf2bc1027e5450de73ddf647f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3dd63019c923e9692431619aced46b91aaf3fd22e1c22ec0a64347f2fe635a0e
+oid sha256:4bb51d675cf23603b1b765cd645f53d6b66ddb104d56d48674e9c798e086f696
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0dc59065f970fd13c57a16e5aa2c2ec0e5dc6ba16189267486fbd8cc465a6fd
+oid sha256:8784a806a517b57d5b84ca561594528563d0fdcdc4d0513f5b313f6cc716b68e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.827178729689808,
+  "epoch": 0.8419497784342689,
   "eval_steps": 50,
-  "global_step": 2800,
+  "global_step": 2850,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4991,11 +4991,100 @@
       "eval_steps_per_second": 0.778,
       "num_input_tokens_seen": 29079576,
       "step": 2800
+    },
+    {
+      "epoch": 0.8286558345642541,
+      "grad_norm": 1.7749470743647537,
+      "learning_rate": 6.790053376980959e-05,
+      "loss": 0.1752,
+      "num_input_tokens_seen": 29131768,
+      "step": 2805
+    },
+    {
+      "epoch": 0.8301329394387001,
+      "grad_norm": 1.3850536432850993,
+      "learning_rate": 6.778644845303483e-05,
+      "loss": 0.2502,
+      "num_input_tokens_seen": 29183952,
+      "step": 2810
+    },
+    {
+      "epoch": 0.8316100443131462,
+      "grad_norm": 1.0103366085788665,
+      "learning_rate": 6.767225702248698e-05,
+      "loss": 0.2092,
+      "num_input_tokens_seen": 29236232,
+      "step": 2815
+    },
+    {
+      "epoch": 0.8330871491875923,
+      "grad_norm": 13.569757844040682,
+      "learning_rate": 6.755796015943097e-05,
+      "loss": 0.2492,
+      "num_input_tokens_seen": 29287672,
+      "step": 2820
+    },
+    {
+      "epoch": 0.8345642540620384,
+      "grad_norm": 1.23710967928213,
+      "learning_rate": 6.744355854576075e-05,
+      "loss": 0.2377,
+      "num_input_tokens_seen": 29339952,
+      "step": 2825
+    },
+    {
+      "epoch": 0.8360413589364845,
+      "grad_norm": 1.3865093164785327,
+      "learning_rate": 6.732905286399516e-05,
+      "loss": 0.168,
+      "num_input_tokens_seen": 29392128,
+      "step": 2830
+    },
+    {
+      "epoch": 0.8375184638109305,
+      "grad_norm": 1.585429725562894,
+      "learning_rate": 6.721444379727398e-05,
+      "loss": 0.1919,
+      "num_input_tokens_seen": 29444168,
+      "step": 2835
+    },
+    {
+      "epoch": 0.8389955686853766,
+      "grad_norm": 9.069319490379504,
+      "learning_rate": 6.709973202935374e-05,
+      "loss": 0.2225,
+      "num_input_tokens_seen": 29495592,
+      "step": 2840
+    },
+    {
+      "epoch": 0.8404726735598228,
+      "grad_norm": 0.9134374448072771,
+      "learning_rate": 6.698491824460371e-05,
+      "loss": 0.1715,
+      "num_input_tokens_seen": 29548008,
+      "step": 2845
+    },
+    {
+      "epoch": 0.8419497784342689,
+      "grad_norm": 1.4124132378382863,
+      "learning_rate": 6.687000312800178e-05,
+      "loss": 0.1928,
+      "num_input_tokens_seen": 29600536,
+      "step": 2850
+    },
+    {
+      "epoch": 0.8419497784342689,
+      "eval_loss": 0.28275948762893677,
+      "eval_runtime": 19.4837,
+      "eval_samples_per_second": 3.08,
+      "eval_steps_per_second": 0.77,
+      "num_input_tokens_seen": 29600536,
+      "step": 2850
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 29079576,
+  "num_input_tokens_seen": 29600536,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5010,7 +5099,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1918391455318016.0,
+  "total_flos": 1952800415416320.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null