Training in progress, step 200, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1db5746fd65250f2bf6817d2de9266703177b7d0c3decef80aefa9773e3ca7a
+oid sha256:381d89dd7c1135d0334349b9f49a436d03516c244b2ccee7000cd0fdc6d88805
 size 29034840

last-checkpoint/global_step200/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f73787e5bfc6d17b10cffea4f53d9c90b9a745d99e305b6688a004652359d006
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e8ea00d4eb5c712165d93ec8855b66f058a6461bf0b071821ec85f133579d6
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a35a8adb32924b5460da62aff07373df5d13887e2b01f2bb77544ef5250fc9b
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6cff643243e7c2ce43ae1a8c95695a24e43038a042e26a11a9d99cb3ee00f4
+size 43429616

last-checkpoint/global_step200/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9c8aed679b2d5c060599974f1ab45390fb83c42d37e4ed63003977054cda112
+size 637299

last-checkpoint/global_step200/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7ceb1154d5065a4632b1731df2acb5e51080a1f0388344212eaec5524452314
+size 637171

last-checkpoint/global_step200/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30465b90efa0eb73fc80dc870a336c0cc691a34eaf088821a381a9f71ee7d652
+size 637171

last-checkpoint/global_step200/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab524daca08828de12f8863c12ac5a32a74f179700a4a2850b77f7debcf459b3
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step150
+global_step200

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70cc56408014c410353d4dd58ae9b03f4be043f5f800324f66fd8e20e99b840e
+oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49d1438e98cc9c53a6852464635ce62e9788e61eb3646b73e33813f487c4b6ae
+oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4388add9cec90932f8ff0100d27a0574d98e1bad52ff89d44e31967d2b4fbfde
+oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a705d6dfaae4f2c1b4b2be6b25a6eb521ffae6fcba21cc1531e97b60037ed079
+oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c587ef55825cdfdbac47c29c3bd8a2996263c16833a5bd6aaa2fb014bea1e9d1
+oid sha256:fdde69e82f45c9368fe627c2082d010eb77a8ceb0b2354f60caa089407a428a6
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.09665286540985107,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-150",
-  "epoch": 0.07725985063095545,
+  "epoch": 0.10301313417460727,
   "eval_steps": 50,
-  "global_step": 150,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -274,11 +274,100 @@
       "eval_steps_per_second": 0.778,
       "num_input_tokens_seen": 1497600,
       "step": 150
+    },
+    {
+      "epoch": 0.07983517898532062,
+      "grad_norm": 6.096163706748617,
+      "learning_rate": 9.11764705882353e-05,
+      "loss": 0.06,
+      "num_input_tokens_seen": 1547520,
+      "step": 155
+    },
+    {
+      "epoch": 0.08241050733968582,
+      "grad_norm": 4.278069142242893,
+      "learning_rate": 9.411764705882353e-05,
+      "loss": 0.0265,
+      "num_input_tokens_seen": 1597440,
+      "step": 160
+    },
+    {
+      "epoch": 0.08498583569405099,
+      "grad_norm": 1.7183097652953412,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.1743,
+      "num_input_tokens_seen": 1647360,
+      "step": 165
+    },
+    {
+      "epoch": 0.08756116404841617,
+      "grad_norm": 4.95401899568707,
+      "learning_rate": 0.0001,
+      "loss": 0.051,
+      "num_input_tokens_seen": 1697280,
+      "step": 170
+    },
+    {
+      "epoch": 0.09013649240278135,
+      "grad_norm": 0.34558354886099124,
+      "learning_rate": 9.999940874631277e-05,
+      "loss": 0.0584,
+      "num_input_tokens_seen": 1747200,
+      "step": 175
+    },
+    {
+      "epoch": 0.09271182075714654,
+      "grad_norm": 3.6326401692458146,
+      "learning_rate": 9.999763499923432e-05,
+      "loss": 0.0704,
+      "num_input_tokens_seen": 1797120,
+      "step": 180
+    },
+    {
+      "epoch": 0.09528714911151172,
+      "grad_norm": 0.4999720825488852,
+      "learning_rate": 9.999467880071402e-05,
+      "loss": 0.0278,
+      "num_input_tokens_seen": 1847040,
+      "step": 185
+    },
+    {
+      "epoch": 0.0978624774658769,
+      "grad_norm": 0.5211964505880501,
+      "learning_rate": 9.999054022066641e-05,
+      "loss": 0.0862,
+      "num_input_tokens_seen": 1896960,
+      "step": 190
+    },
+    {
+      "epoch": 0.10043780582024209,
+      "grad_norm": 0.8767038751832389,
+      "learning_rate": 9.998521935696953e-05,
+      "loss": 0.0565,
+      "num_input_tokens_seen": 1946880,
+      "step": 195
+    },
+    {
+      "epoch": 0.10301313417460727,
+      "grad_norm": 0.1848827361202722,
+      "learning_rate": 9.997871633546257e-05,
+      "loss": 0.0459,
+      "num_input_tokens_seen": 1996800,
+      "step": 200
+    },
+    {
+      "epoch": 0.10301313417460727,
+      "eval_loss": 0.09957947582006454,
+      "eval_runtime": 19.2651,
+      "eval_samples_per_second": 3.114,
+      "eval_steps_per_second": 0.779,
+      "num_input_tokens_seen": 1996800,
+      "step": 200
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 1497600,
+  "num_input_tokens_seen": 1996800,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -293,7 +382,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 98764767100928.0,
+  "total_flos": 131704746672128.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null