Training in progress, step 3100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5be8bc8351dafe40df5f56b88df2329a278b65ac350dc2b871125aa7c127df5
+oid sha256:4e16399a54feade86bfbf61a8556cd130e7f0cb7bcc816ba08b4856dddb7606a
 size 29034840

last-checkpoint/global_step3100/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6442e5ae9de93ac10a49957a278a8a63d68a76b5e1df84a2a5207fcdfdfe5303
+size 43429616

last-checkpoint/global_step3100/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6148ab1901e304f398bc54c299448042794f7778437012ec25bea1e0a271e0e
+size 43429616

last-checkpoint/global_step3100/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e45c316e0222d4db8f427c991e41ed7b9093eaa68eabafbe2505a1c9e031d0f
+size 43429616

last-checkpoint/global_step3100/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd92104d8c6c8c164703d77b771b654c64c485a242b292c5c9903c253fface5
+size 43429616

last-checkpoint/global_step3100/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:769a527c4e6f510809f5b2b13928892b9b7024384bc00772661a2cccde5e4fa4
+size 637299

last-checkpoint/global_step3100/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05589d1f79cab339f6f64907a5dbf1bd44b1e8d65a9a1f019d2ec8e2b3cd2a6
+size 637171

last-checkpoint/global_step3100/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e28ad7a834b66d1ee2fe12359362f2fdb764eaf9b451b44e1d6df40abfd8079c
+size 637171

last-checkpoint/global_step3100/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:226452aea7fb29b0cd6c8e9e340122bb65353ec4a0df00019fe9614dc4195bc6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3050
+global_step3100

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f30f7f3c3620ccd30020e1ea4b81d1a56ee511b742c91370577d55399c14412
+oid sha256:381f090b12cbb3fcce976bd2e72d07a7786154c2848cc881d75715648e7c4dc2
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:440cc1f8e138e1a90606722bd350460b4460991a2f3671f46f880f5743522dca
+oid sha256:4321dc1beb6c245d4a16a8fb2bed2ce2a40d89e5a9c611c4572f63a09523846c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d78d5df466f5b34e85649b90825d4b168464f6d49c668313415473184409b799
+oid sha256:5b79020117e301cd96518c9d1e3eb43a609bc85799f14bac63d9c572fc04cb89
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d8be6df9e66c8bba407df289d2aa6b4422668727a1e2419037de032213393a4
+oid sha256:5429623c7323b4a820ea3d76194bc456c2affeeb1f3af8978aec5aee11b2d1ef
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3da17ebe605411b58ea2b644f619dba9557cc5aeaf2c6d740ea8353d5ec2fab3
+oid sha256:e3b3f02f00a06f9630f63af0f9d2eac4db52dcd348e24cd0a92f7c194ed5add6
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.9010339734121122,
+  "epoch": 0.9158050221565731,
   "eval_steps": 50,
-  "global_step": 3050,
+  "global_step": 3100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5436,11 +5436,100 @@
       "eval_steps_per_second": 0.779,
       "num_input_tokens_seen": 31679080,
       "step": 3050
+    },
+    {
+      "epoch": 0.9025110782865583,
+      "grad_norm": 1.1996422945391199,
+      "learning_rate": 6.207976753619816e-05,
+      "loss": 0.1854,
+      "num_input_tokens_seen": 31731232,
+      "step": 3055
+    },
+    {
+      "epoch": 0.9039881831610044,
+      "grad_norm": 1.142249274568305,
+      "learning_rate": 6.196122246447779e-05,
+      "loss": 0.1813,
+      "num_input_tokens_seen": 31783480,
+      "step": 3060
+    },
+    {
+      "epoch": 0.9054652880354506,
+      "grad_norm": 1.8891844898246446,
+      "learning_rate": 6.184260603222416e-05,
+      "loss": 0.2099,
+      "num_input_tokens_seen": 31835864,
+      "step": 3065
+    },
+    {
+      "epoch": 0.9069423929098966,
+      "grad_norm": 1.1771283819772904,
+      "learning_rate": 6.17239189471017e-05,
+      "loss": 0.2158,
+      "num_input_tokens_seen": 31887120,
+      "step": 3070
+    },
+    {
+      "epoch": 0.9084194977843427,
+      "grad_norm": 1.000017408106927,
+      "learning_rate": 6.160516191719638e-05,
+      "loss": 0.1947,
+      "num_input_tokens_seen": 31938768,
+      "step": 3075
+    },
+    {
+      "epoch": 0.9098966026587888,
+      "grad_norm": 10.76780458746382,
+      "learning_rate": 6.148633565101145e-05,
+      "loss": 0.2058,
+      "num_input_tokens_seen": 31990800,
+      "step": 3080
+    },
+    {
+      "epoch": 0.9113737075332349,
+      "grad_norm": 1.5246138398849078,
+      "learning_rate": 6.136744085746322e-05,
+      "loss": 0.2366,
+      "num_input_tokens_seen": 32042096,
+      "step": 3085
+    },
+    {
+      "epoch": 0.912850812407681,
+      "grad_norm": 1.3169793573688027,
+      "learning_rate": 6.124847824587684e-05,
+      "loss": 0.2749,
+      "num_input_tokens_seen": 32092864,
+      "step": 3090
+    },
+    {
+      "epoch": 0.914327917282127,
+      "grad_norm": 1.0350686863717118,
+      "learning_rate": 6.112944852598205e-05,
+      "loss": 0.2242,
+      "num_input_tokens_seen": 32144288,
+      "step": 3095
+    },
+    {
+      "epoch": 0.9158050221565731,
+      "grad_norm": 0.8553950294390508,
+      "learning_rate": 6.1010352407908966e-05,
+      "loss": 0.1999,
+      "num_input_tokens_seen": 32196176,
+      "step": 3100
+    },
+    {
+      "epoch": 0.9158050221565731,
+      "eval_loss": 0.2710443437099457,
+      "eval_runtime": 19.0309,
+      "eval_samples_per_second": 3.153,
+      "eval_steps_per_second": 0.788,
+      "num_input_tokens_seen": 32196176,
+      "step": 3100
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 31679080,
+  "num_input_tokens_seen": 32196176,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5455,7 +5544,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2089927830929408.0,
+  "total_flos": 2124006796820480.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null