Training in progress, step 2750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1f0a1299753b652961a95b591427c87ce3772807fe346f4fffff8a783b0e00e
+oid sha256:71879fd91931f28264e3a813a60350149ee278572f6af60c14a93354f4b2e458
 size 29034840

last-checkpoint/global_step2750/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc0d37cf654c1249c009b871eed9237a0a0acf7b1358372d9251681491e7fa8e
+size 43429616

last-checkpoint/global_step2750/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85ead42a5e2129c0de701185d97072beb36c7a8b987c0270bd6b1014c6f6af8a
+size 43429616

last-checkpoint/global_step2750/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a218971b5aec39522cfc2c02fc622b2602a54d1fc85c124db943f9fe4c5d4317
+size 43429616

last-checkpoint/global_step2750/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67da6969facd5b755a2100a31775fd5fc5df0019c902e5fb142f6368a8b9141c
+size 43429616

last-checkpoint/global_step2750/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca9d7fe4f3a7ec00fdc1341eede29421baa46c4b3e5767e5648f785c7ab736f0
+size 637299

last-checkpoint/global_step2750/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6fc886de55064a42dd5e0e4609fa2286fb7e9e0903744251f646556db118519
+size 637171

last-checkpoint/global_step2750/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ca2ca95edc298786418965940b1985708efb5fbf41907c1149fe003030ad66
+size 637171

last-checkpoint/global_step2750/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d179ea4f34aafae9542a80db079ed838e02255e59a74f60c14d075aa5eb27444
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2700
+global_step2750

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e599331812a34463d102d64a4034a0b702a893f362f752003aa577fe71dcc1d
+oid sha256:5d8d3c7739f9787ea797b86ff1b3a51f9e68197835ba3178915a8a77558f67fc
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ed431e5e71393a0174ad2fd492755f8c1142596f1af3bfe7827c1f8f815dd80
+oid sha256:a22a57799bc43e59db67d9a787ed73040020c5f35990602033f4dab1318787d7
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2e70789f26a9f56b6b779e87cb1a405615af81562a256e5afe579f40972e827
+oid sha256:29a624b936b77a04d6bfb6940acdd65a710bf39452e419e7ddb5c40fb2261072
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c8c18bc74d5211e761da269c814d7da0687633993838ec22e81ac939a14e91b
+oid sha256:3a79306817d4440cd621149537e8cf216b60f847fc6f9531a6147426aa02bb07
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:375d770aab1911f1464e2da712c8440a47a0edbad49710a4097aa7592f9fdb3e
+oid sha256:fc99fc5a48a169bebd6bda86672afa6c721f544602a3586f7782ae7070075fc7
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.7976366322008862,
+  "epoch": 0.8124076809453471,
   "eval_steps": 50,
-  "global_step": 2700,
+  "global_step": 2750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4813,11 +4813,100 @@
       "eval_steps_per_second": 0.791,
       "num_input_tokens_seen": 28042992,
       "step": 2700
+    },
+    {
+      "epoch": 0.7991137370753324,
+      "grad_norm": 1.2540711450587154,
+      "learning_rate": 7.01589128070782e-05,
+      "loss": 0.2279,
+      "num_input_tokens_seen": 28094200,
+      "step": 2705
+    },
+    {
+      "epoch": 0.8005908419497785,
+      "grad_norm": 1.1647829910589267,
+      "learning_rate": 7.004709182686531e-05,
+      "loss": 0.2307,
+      "num_input_tokens_seen": 28146144,
+      "step": 2710
+    },
+    {
+      "epoch": 0.8020679468242246,
+      "grad_norm": 1.911821987729577,
+      "learning_rate": 6.993515124590362e-05,
+      "loss": 0.2025,
+      "num_input_tokens_seen": 28198600,
+      "step": 2715
+    },
+    {
+      "epoch": 0.8035450516986706,
+      "grad_norm": 3.9104392867751003,
+      "learning_rate": 6.982309173202951e-05,
+      "loss": 0.2318,
+      "num_input_tokens_seen": 28249928,
+      "step": 2720
+    },
+    {
+      "epoch": 0.8050221565731167,
+      "grad_norm": 3.023688281243673,
+      "learning_rate": 6.971091395378895e-05,
+      "loss": 0.2074,
+      "num_input_tokens_seen": 28301928,
+      "step": 2725
+    },
+    {
+      "epoch": 0.8064992614475628,
+      "grad_norm": 1.4517739851550877,
+      "learning_rate": 6.95986185804334e-05,
+      "loss": 0.1935,
+      "num_input_tokens_seen": 28354256,
+      "step": 2730
+    },
+    {
+      "epoch": 0.8079763663220089,
+      "grad_norm": 1.273143787849072,
+      "learning_rate": 6.948620628191595e-05,
+      "loss": 0.2457,
+      "num_input_tokens_seen": 28405800,
+      "step": 2735
+    },
+    {
+      "epoch": 0.8094534711964549,
+      "grad_norm": 7.171460586418023,
+      "learning_rate": 6.937367772888725e-05,
+      "loss": 0.2021,
+      "num_input_tokens_seen": 28457664,
+      "step": 2740
+    },
+    {
+      "epoch": 0.810930576070901,
+      "grad_norm": 4.016097948044553,
+      "learning_rate": 6.926103359269152e-05,
+      "loss": 0.2323,
+      "num_input_tokens_seen": 28509944,
+      "step": 2745
+    },
+    {
+      "epoch": 0.8124076809453471,
+      "grad_norm": 1.7260282738026802,
+      "learning_rate": 6.914827454536254e-05,
+      "loss": 0.2501,
+      "num_input_tokens_seen": 28561248,
+      "step": 2750
+    },
+    {
+      "epoch": 0.8124076809453471,
+      "eval_loss": 0.41148969531059265,
+      "eval_runtime": 19.291,
+      "eval_samples_per_second": 3.11,
+      "eval_steps_per_second": 0.778,
+      "num_input_tokens_seen": 28561248,
+      "step": 2750
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 28042992,
+  "num_input_tokens_seen": 28561248,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4832,7 +4921,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1850037045297152.0,
+  "total_flos": 1884238220361728.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null