Training in progress, step 2350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee4d20084c020812db6dbd79fbf4828c8db14c421e8619ec1b2d21ffca81b370
+oid sha256:e895c1cddd40eb0dbe387456240309b06d66fe014fd793905368c3b37bbbff4a
 size 29034840

last-checkpoint/global_step2350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e64eef91b1c41414b86c1c79785e71b08863e11060ff8cc3ef7864e7ebdfd9ea
+size 43429616

last-checkpoint/global_step2350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5375885f162d12b5216a73c6bb52338439850ad1e62fb1c7c0c1edab8bc406
+size 43429616

last-checkpoint/global_step2350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9591fd0143ecbd5381bf4045248702d04a33777473a65e109ac0ff4e0b72032
+size 43429616

last-checkpoint/global_step2350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f2bea725eb09303fe4967379db3c2f97314752353eb0873dd28f22e85954261
+size 43429616

last-checkpoint/global_step2350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77ea2f1422947988eee2012dbb023c20b562004f04b0280a6b2e02eb0104df6c
+size 637299

last-checkpoint/global_step2350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d2de4b133830a36a27f5675bcfb3196e62844d0ea6515c1da4410138b5a0d55
+size 637171

last-checkpoint/global_step2350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb4f0a348ba8ed1a0f899c6bd11b00341b3cb75bef3c71ead62d8cdd2a38d6d
+size 637171

last-checkpoint/global_step2350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddee7cdbd3744f566b1d3de4032edc8a6be338074790be0b4e6dd344c41978e2
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2300
+global_step2350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d25cbcbbaa0866ea9c7365cb49b84e805db119693e615f5a1898a6ebfe997e8
+oid sha256:f12bf3da75454e5aae4644f2a1d46fdf90f68e680dbf5bdaa86861f825d32d80
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a883389afac12125c2c6bf62631b7de0220fdb0020d24cd0c6e8f8858dd3b362
+oid sha256:2faef1b8798e7516fd96ee7b3363866a8f97ca2d0ec5a8dd27bbfe70b0c6a733
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:577d49de6d60035e159d9ebb1e6eabef79a55787b14ecea93a6a93c242661779
+oid sha256:7f35f6d27fbd414dd4285d91816b37fb6b97ff10fbba4c074d56ad2a7f723033
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5f2dd1c21e06806a9ce39eeab45734dfb8a62b829f91a86d1f65f13102d6242
+oid sha256:ea5c64dfc7e3b2729065483dfef8e4bb0af0d9bae32df888d258ee3c2859d676
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4affa863c39f4a87a7aa62bb6d8ea5d9ce72773cba2b4ccffc763273823a37b1
+oid sha256:b8d0442374b2f8618569ef196bfe7955aa6b552a4eb29da5972e99b6fb6a7877
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.24180778861045837,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2300",
-  "epoch": 0.6794682422451994,
+  "best_metric": 0.1869634985923767,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
+  "epoch": 0.6942392909896603,
   "eval_steps": 50,
-  "global_step": 2300,
+  "global_step": 2350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4101,11 +4101,100 @@
       "eval_steps_per_second": 0.791,
       "num_input_tokens_seen": 23883032,
       "step": 2300
+    },
+    {
+      "epoch": 0.6809453471196455,
+      "grad_norm": 15.468575933907964,
+      "learning_rate": 7.865936626267243e-05,
+      "loss": 0.2474,
+      "num_input_tokens_seen": 23934880,
+      "step": 2305
+    },
+    {
+      "epoch": 0.6824224519940916,
+      "grad_norm": 3.9259269918244932,
+      "learning_rate": 7.855920710508009e-05,
+      "loss": 0.2613,
+      "num_input_tokens_seen": 23986160,
+      "step": 2310
+    },
+    {
+      "epoch": 0.6838995568685377,
+      "grad_norm": 1.443222642921112,
+      "learning_rate": 7.845887756354458e-05,
+      "loss": 0.1957,
+      "num_input_tokens_seen": 24038984,
+      "step": 2315
+    },
+    {
+      "epoch": 0.6853766617429837,
+      "grad_norm": 2.418773014069429,
+      "learning_rate": 7.835837823663092e-05,
+      "loss": 0.2709,
+      "num_input_tokens_seen": 24090648,
+      "step": 2320
+    },
+    {
+      "epoch": 0.6868537666174298,
+      "grad_norm": 1.2678547091245644,
+      "learning_rate": 7.825770972391712e-05,
+      "loss": 0.2873,
+      "num_input_tokens_seen": 24142200,
+      "step": 2325
+    },
+    {
+      "epoch": 0.6883308714918759,
+      "grad_norm": 1.5899740505737827,
+      "learning_rate": 7.81568726259905e-05,
+      "loss": 0.2378,
+      "num_input_tokens_seen": 24194400,
+      "step": 2330
+    },
+    {
+      "epoch": 0.689807976366322,
+      "grad_norm": 1.6209911075093861,
+      "learning_rate": 7.805586754444416e-05,
+      "loss": 0.2615,
+      "num_input_tokens_seen": 24245328,
+      "step": 2335
+    },
+    {
+      "epoch": 0.691285081240768,
+      "grad_norm": 16.06254938606901,
+      "learning_rate": 7.795469508187343e-05,
+      "loss": 0.2234,
+      "num_input_tokens_seen": 24297400,
+      "step": 2340
+    },
+    {
+      "epoch": 0.6927621861152142,
+      "grad_norm": 2.2353075576651085,
+      "learning_rate": 7.785335584187219e-05,
+      "loss": 0.2833,
+      "num_input_tokens_seen": 24348536,
+      "step": 2345
+    },
+    {
+      "epoch": 0.6942392909896603,
+      "grad_norm": 11.321777956495533,
+      "learning_rate": 7.775185042902933e-05,
+      "loss": 0.2313,
+      "num_input_tokens_seen": 24401256,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6942392909896603,
+      "eval_loss": 0.1869634985923767,
+      "eval_runtime": 19.499,
+      "eval_samples_per_second": 3.077,
+      "eval_steps_per_second": 0.769,
+      "num_input_tokens_seen": 24401256,
+      "step": 2350
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 23883032,
+  "num_input_tokens_seen": 24401256,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4120,7 +4209,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1575723931533312.0,
+  "total_flos": 1609869937344512.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null