Training in progress, step 300, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea0e59edded26f764540e30e37224e6309622e9f0d8713fac942b2956b39deb3
+oid sha256:8c6b6b2163a165d1469f79ac83287cf58b9febbee9aa4bb628b13cd978a905d2
 size 29034840

last-checkpoint/global_step300/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a7f8fb47b4fd856f28f876db8029858e01b075a9e06df2ffd855ab80a387321
+size 43429616

last-checkpoint/global_step300/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22bf056a4791dde098eda0a6ad1295601781e91929ca95a093f811f735f7d95b
+size 43429616

last-checkpoint/global_step300/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6649a4d4e087da0239b97d58a774f6fe70b7e8c18097da2c986f6a5761d088e6
+size 43429616

last-checkpoint/global_step300/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:297d7acb22e1081cf2fc6bad98f49dcbf4b535e6b990f560f28bd2873baf11dc
+size 43429616

last-checkpoint/global_step300/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4de5206222f5f6bc48ade0dd830e1dc6401703feaf7328bb17787793a784ed8
+size 637299

last-checkpoint/global_step300/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9c15f3de1191eba8ee412eb11cf43d8297d842d97f58ff3416540e6e60132b4
+size 637171

last-checkpoint/global_step300/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3363de2e039012fbc15be9a390355ad5f78df5709c5a29467c717bbc3d0644c
+size 637171

last-checkpoint/global_step300/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78419129614e77d4f1d73fe2039db0059edfa88fe000e6d37b57f43d85197257
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step250
+global_step300

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:418a5f105ae834c3075024076916b2a9475918fe034c12d0dd5b6d91f1aba467
+oid sha256:ef002048764051a71fb00f8f978e9ec32b780dc850bdb059af362cc56494234b
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e07ace389d24bc1307b74f42a1e7b8f0117b0db853e2df64ff3f15cb92916a2
+oid sha256:37194a6d48612e1a46a2d5d317ead97c70d9fc4569b0118fcd5f84c3dc9daa5a
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da6a990f346d7014dffb28fa2bc7d3b890bd3c53712503fce3656da48d3d6e50
+oid sha256:17c179483659a784aa1ace2427daff48c556a6bcc3c330e6f3274e4dc95e4b49
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e95f356ca38179b05993f55daece0223e96fa10b9a1b9ea2102a739211333f63
+oid sha256:b56857c9b117629f35af2c3d64f522d33a9d8aa94faa81ec6956380a895118c4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d7e2043e09ec2b328d02f0638a57759f62a1b72350c1a7738b78953d31c6142
+oid sha256:75979a52de119ed6c9cd6063ded2b7d6cb6c4870db4a05a6cc6a997d1fa9b653
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.055875860154628754,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-250",
-  "epoch": 0.12876641771825909,
+  "best_metric": 0.030908752232789993,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-300",
+  "epoch": 0.1545197012619109,
   "eval_steps": 50,
-  "global_step": 250,
+  "global_step": 300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -452,11 +452,100 @@
       "eval_steps_per_second": 0.769,
       "num_input_tokens_seen": 2496000,
       "step": 250
+    },
+    {
+      "epoch": 0.13134174607262425,
+      "grad_norm": 0.845514751132877,
+      "learning_rate": 9.98292246503335e-05,
+      "loss": 0.0235,
+      "num_input_tokens_seen": 2545920,
+      "step": 255
+    },
+    {
+      "epoch": 0.13391707442698944,
+      "grad_norm": 1.7441538057875345,
+      "learning_rate": 9.980855572238714e-05,
+      "loss": 0.0137,
+      "num_input_tokens_seen": 2595840,
+      "step": 260
+    },
+    {
+      "epoch": 0.13649240278135463,
+      "grad_norm": 0.6628270146260998,
+      "learning_rate": 9.978670881475172e-05,
+      "loss": 0.0605,
+      "num_input_tokens_seen": 2645760,
+      "step": 265
+    },
+    {
+      "epoch": 0.1390677311357198,
+      "grad_norm": 4.49926360488578,
+      "learning_rate": 9.976368444410985e-05,
+      "loss": 0.0585,
+      "num_input_tokens_seen": 2695680,
+      "step": 270
+    },
+    {
+      "epoch": 0.141643059490085,
+      "grad_norm": 0.7159498856832776,
+      "learning_rate": 9.973948315499126e-05,
+      "loss": 0.0096,
+      "num_input_tokens_seen": 2745600,
+      "step": 275
+    },
+    {
+      "epoch": 0.14421838784445018,
+      "grad_norm": 4.09842502301765,
+      "learning_rate": 9.971410551976002e-05,
+      "loss": 0.1383,
+      "num_input_tokens_seen": 2795520,
+      "step": 280
+    },
+    {
+      "epoch": 0.14679371619881534,
+      "grad_norm": 1.0773897808674016,
+      "learning_rate": 9.968755213860094e-05,
+      "loss": 0.049,
+      "num_input_tokens_seen": 2845440,
+      "step": 285
+    },
+    {
+      "epoch": 0.14936904455318054,
+      "grad_norm": 1.7224091076306651,
+      "learning_rate": 9.96598236395054e-05,
+      "loss": 0.0336,
+      "num_input_tokens_seen": 2895360,
+      "step": 290
+    },
+    {
+      "epoch": 0.1519443729075457,
+      "grad_norm": 0.24158350585209873,
+      "learning_rate": 9.96309206782565e-05,
+      "loss": 0.019,
+      "num_input_tokens_seen": 2945280,
+      "step": 295
+    },
+    {
+      "epoch": 0.1545197012619109,
+      "grad_norm": 0.6614271496466305,
+      "learning_rate": 9.960084393841355e-05,
+      "loss": 0.0381,
+      "num_input_tokens_seen": 2995200,
+      "step": 300
+    },
+    {
+      "epoch": 0.1545197012619109,
+      "eval_loss": 0.030908752232789993,
+      "eval_runtime": 19.8702,
+      "eval_samples_per_second": 3.02,
+      "eval_steps_per_second": 0.755,
+      "num_input_tokens_seen": 2995200,
+      "step": 300
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 2496000,
+  "num_input_tokens_seen": 2995200,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -471,7 +560,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 164644726243328.0,
+  "total_flos": 197584705814528.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null