Training in progress, step 250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:381d89dd7c1135d0334349b9f49a436d03516c244b2ccee7000cd0fdc6d88805
+oid sha256:ea0e59edded26f764540e30e37224e6309622e9f0d8713fac942b2956b39deb3
 size 29034840

last-checkpoint/global_step250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8521d89f5d951cd266e5a0ad2c17de7b69aaa3e87e321317eea95aa55a91603
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:238b9e16d8923db2d6c50269fdc6c44d42412382c52e25374eec633be975b99d
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13d3ffa108ad1c07e8075652d5b94d4eedcb26e11c0d50073c8aa775a0789168
+size 43429616

last-checkpoint/global_step250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e9f9a85f8e966a5ca76f86099edc9d961561b0203f32181b5c873488556f321
+size 43429616

last-checkpoint/global_step250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e80b196c408fdbb93be47104d2e2ebb39806a8e2f671eb97b21520c1760537b
+size 637299

last-checkpoint/global_step250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50783d840386ee88fb64fd6d7bb5cc3c88c65818b6030a5cb92b901026aa8a7b
+size 637171

last-checkpoint/global_step250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a9975bb0ef92a03508b158d7fea0b886e853ff0adb21cca084bd76368bdf9a
+size 637171

last-checkpoint/global_step250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76693721fccb62522fa35aaa24f5d6f23796a4391cc1cbb825c3e42813ab3ab8
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step200
+global_step250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
+oid sha256:418a5f105ae834c3075024076916b2a9475918fe034c12d0dd5b6d91f1aba467
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
+oid sha256:6e07ace389d24bc1307b74f42a1e7b8f0117b0db853e2df64ff3f15cb92916a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
+oid sha256:da6a990f346d7014dffb28fa2bc7d3b890bd3c53712503fce3656da48d3d6e50
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
+oid sha256:e95f356ca38179b05993f55daece0223e96fa10b9a1b9ea2102a739211333f63
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdde69e82f45c9368fe627c2082d010eb77a8ceb0b2354f60caa089407a428a6
+oid sha256:4d7e2043e09ec2b328d02f0638a57759f62a1b72350c1a7738b78953d31c6142
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.09665286540985107,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-150",
-  "epoch": 0.10301313417460727,
+  "best_metric": 0.055875860154628754,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-250",
+  "epoch": 0.12876641771825909,
   "eval_steps": 50,
-  "global_step": 200,
+  "global_step": 250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -363,11 +363,100 @@
       "eval_steps_per_second": 0.779,
       "num_input_tokens_seen": 1996800,
       "step": 200
+    },
+    {
+      "epoch": 0.10558846252897244,
+      "grad_norm": 1.0969393658164421,
+      "learning_rate": 9.997103130994296e-05,
+      "loss": 0.0539,
+      "num_input_tokens_seen": 2046720,
+      "step": 205
+    },
+    {
+      "epoch": 0.10816379088333762,
+      "grad_norm": 2.885869194934028,
+      "learning_rate": 9.996216446216267e-05,
+      "loss": 0.0654,
+      "num_input_tokens_seen": 2096640,
+      "step": 210
+    },
+    {
+      "epoch": 0.11073911923770281,
+      "grad_norm": 0.5225257245731217,
+      "learning_rate": 9.995211600182397e-05,
+      "loss": 0.0316,
+      "num_input_tokens_seen": 2146560,
+      "step": 215
+    },
+    {
+      "epoch": 0.11331444759206799,
+      "grad_norm": 2.1553510734212797,
+      "learning_rate": 9.994088616657444e-05,
+      "loss": 0.1169,
+      "num_input_tokens_seen": 2196480,
+      "step": 220
+    },
+    {
+      "epoch": 0.11588977594643317,
+      "grad_norm": 1.1133884703723633,
+      "learning_rate": 9.992847522200133e-05,
+      "loss": 0.0382,
+      "num_input_tokens_seen": 2246400,
+      "step": 225
+    },
+    {
+      "epoch": 0.11846510430079835,
+      "grad_norm": 0.8875243341616034,
+      "learning_rate": 9.99148834616253e-05,
+      "loss": 0.0406,
+      "num_input_tokens_seen": 2296320,
+      "step": 230
+    },
+    {
+      "epoch": 0.12104043265516354,
+      "grad_norm": 1.81283533812695,
+      "learning_rate": 9.990011120689351e-05,
+      "loss": 0.0182,
+      "num_input_tokens_seen": 2346240,
+      "step": 235
+    },
+    {
+      "epoch": 0.12361576100952872,
+      "grad_norm": 3.873083258671571,
+      "learning_rate": 9.988415880717194e-05,
+      "loss": 0.0881,
+      "num_input_tokens_seen": 2396160,
+      "step": 240
+    },
+    {
+      "epoch": 0.1261910893638939,
+      "grad_norm": 3.427761103620865,
+      "learning_rate": 9.986702663973722e-05,
+      "loss": 0.0565,
+      "num_input_tokens_seen": 2446080,
+      "step": 245
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "grad_norm": 1.531943599765959,
+      "learning_rate": 9.98487151097676e-05,
+      "loss": 0.0805,
+      "num_input_tokens_seen": 2496000,
+      "step": 250
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "eval_loss": 0.055875860154628754,
+      "eval_runtime": 19.5106,
+      "eval_samples_per_second": 3.075,
+      "eval_steps_per_second": 0.769,
+      "num_input_tokens_seen": 2496000,
+      "step": 250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 1996800,
+  "num_input_tokens_seen": 2496000,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -382,7 +471,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 131704746672128.0,
+  "total_flos": 164644726243328.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null