Training in progress, step 3050, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75557f39c0933cfb737e5fea18b13474d2e22f2d9438f06b8e2ce77a22c9b865
+oid sha256:d5be8bc8351dafe40df5f56b88df2329a278b65ac350dc2b871125aa7c127df5
 size 29034840

last-checkpoint/global_step3050/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:408195cc2a75b9d9dec8c11f5a11495193a0ffc68118c11703359aa86321b503
+size 43429616

last-checkpoint/global_step3050/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a510fe5d1eabc5456190aeba27f6e572117e85b7af25c7cc138cc2192fb3ffe
+size 43429616

last-checkpoint/global_step3050/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be5c496609c0feb795d46f180675cc4ee4fbe1d9b8c53c57c1a515f05995e759
+size 43429616

last-checkpoint/global_step3050/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1018e102910e7c1651e3be7364ad138b52b3bd68573277eec5d851de006ace2
+size 43429616

last-checkpoint/global_step3050/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa48e566d49855033cca2fb11b808d251dc1cef85c4e14422e4b7323b7c5cedd
+size 637299

last-checkpoint/global_step3050/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a0b4aeba5ad023bbcf369d3255e9a14871d54ebfb11daaf1310b94fa4ee0c67
+size 637171

last-checkpoint/global_step3050/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bde9b336c46cc89d1b138ba52bd463c46440720d03ce038f35705b1d36df8f3
+size 637171

last-checkpoint/global_step3050/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93a81bec2d03d08f9cffb54b4dccc79722ba6644873d6d5b4d8c4bb8b402255b
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3000
+global_step3050

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8ac6af6ac2b73603409d1721537224310f2ce061bc8c1c1c6f959231ed2e31e
+oid sha256:6f30f7f3c3620ccd30020e1ea4b81d1a56ee511b742c91370577d55399c14412
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:693cfdc542e1c8c319f7052d02602310660cb04e6571aa78525e03834c8b9930
+oid sha256:440cc1f8e138e1a90606722bd350460b4460991a2f3671f46f880f5743522dca
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ca85a038ef549f3dd6ca18c0dadc240e57927cad9adbcbd9ff224f3da3ae003
+oid sha256:d78d5df466f5b34e85649b90825d4b168464f6d49c668313415473184409b799
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2a98b011c36f42f0d59506d21a897e487e2865babe045af4934af8b2748950
+oid sha256:6d8be6df9e66c8bba407df289d2aa6b4422668727a1e2419037de032213393a4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4494fc1ba6eef0a9379f34c371e4e34294ca2139350203f5fe5a9d90eff98549
+oid sha256:3da17ebe605411b58ea2b644f619dba9557cc5aeaf2c6d740ea8353d5ec2fab3
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.8862629246676514,
+  "epoch": 0.9010339734121122,
   "eval_steps": 50,
-  "global_step": 3000,
+  "global_step": 3050,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5347,11 +5347,100 @@
       "eval_steps_per_second": 0.774,
       "num_input_tokens_seen": 31159736,
       "step": 3000
+    },
+    {
+      "epoch": 0.8877400295420975,
+      "grad_norm": 1.2324370262682327,
+      "learning_rate": 6.326113804772157e-05,
+      "loss": 0.2011,
+      "num_input_tokens_seen": 31211632,
+      "step": 3005
+    },
+    {
+      "epoch": 0.8892171344165436,
+      "grad_norm": 1.7142403977850047,
+      "learning_rate": 6.314334540712983e-05,
+      "loss": 0.2197,
+      "num_input_tokens_seen": 31264376,
+      "step": 3010
+    },
+    {
+      "epoch": 0.8906942392909897,
+      "grad_norm": 1.50601762442169,
+      "learning_rate": 6.302547435347122e-05,
+      "loss": 0.1853,
+      "num_input_tokens_seen": 31316584,
+      "step": 3015
+    },
+    {
+      "epoch": 0.8921713441654358,
+      "grad_norm": 1.2451326697290692,
+      "learning_rate": 6.290752558996325e-05,
+      "loss": 0.2312,
+      "num_input_tokens_seen": 31367768,
+      "step": 3020
+    },
+    {
+      "epoch": 0.8936484490398818,
+      "grad_norm": 1.3816673419860452,
+      "learning_rate": 6.278949982028704e-05,
+      "loss": 0.2608,
+      "num_input_tokens_seen": 31419664,
+      "step": 3025
+    },
+    {
+      "epoch": 0.8951255539143279,
+      "grad_norm": 1.430852186731335,
+      "learning_rate": 6.267139774858318e-05,
+      "loss": 0.225,
+      "num_input_tokens_seen": 31471672,
+      "step": 3030
+    },
+    {
+      "epoch": 0.896602658788774,
+      "grad_norm": 22.943001206859204,
+      "learning_rate": 6.255322007944743e-05,
+      "loss": 0.225,
+      "num_input_tokens_seen": 31523888,
+      "step": 3035
+    },
+    {
+      "epoch": 0.8980797636632201,
+      "grad_norm": 0.8776969753848171,
+      "learning_rate": 6.243496751792658e-05,
+      "loss": 0.2291,
+      "num_input_tokens_seen": 31574992,
+      "step": 3040
+    },
+    {
+      "epoch": 0.8995568685376661,
+      "grad_norm": 0.8040234911638864,
+      "learning_rate": 6.231664076951421e-05,
+      "loss": 0.1926,
+      "num_input_tokens_seen": 31627608,
+      "step": 3045
+    },
+    {
+      "epoch": 0.9010339734121122,
+      "grad_norm": 1.336878225583922,
+      "learning_rate": 6.219824054014656e-05,
+      "loss": 0.2303,
+      "num_input_tokens_seen": 31679080,
+      "step": 3050
+    },
+    {
+      "epoch": 0.9010339734121122,
+      "eval_loss": 0.22528553009033203,
+      "eval_runtime": 19.2502,
+      "eval_samples_per_second": 3.117,
+      "eval_steps_per_second": 0.779,
+      "num_input_tokens_seen": 31679080,
+      "step": 3050
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 31159736,
+  "num_input_tokens_seen": 31679080,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5366,7 +5455,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2055677932208128.0,
+  "total_flos": 2089927830929408.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null