Training in progress, step 3400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8815c16681432a11b5ac188380ad3a07078a835a470d216013977c67742fdcc
+oid sha256:2e3dd7488301b3ff8268dcffb6f575eefe718b69eb21c58090d96141890a4fd7
 size 18516456

last-checkpoint/global_step3399/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5377ce9ec944960b3451a8668ed2930b7d08a3218dc6c36c7af91660799a564
+size 27700976

last-checkpoint/global_step3399/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642f46e60fe60bc51e4297db96d429c44bcbb7de7c17fa6343f398bc77a5f7b7
+size 27700976

last-checkpoint/global_step3399/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0986456f1240e9a82a370b55c1b4f69c9fe5a39987a6cbd36d5459de46bc50a1
+size 27700976

last-checkpoint/global_step3399/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428d4ad8a46b8657df7181502a012cc6ed010aac1422fd4407a9f3b90c5f2245
+size 27700976

last-checkpoint/global_step3399/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8be0604bac04683caeabc3351d36a81d128263368a7e48830a613677261d0f5
+size 411571

last-checkpoint/global_step3399/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ceae8833e74f99b4451636773e39fc36c4f0ae5015d18206d304968d397d04a
+size 411507

last-checkpoint/global_step3399/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa075ddbef52cce7c5a92e4246c0d6329b0fde68c384da28862b974f36cbb9c
+size 411507

last-checkpoint/global_step3399/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecfe6be46982c753e1f7606377f3951ed6043daec06db0db276ef338ee316b03
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step3349
+global_step3399

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abe9163f042a56ab41ea5c2436dff084d8a4a6358e7f4cb1f18e04cb69810300
+oid sha256:a1b33fbd97eb762e874f342b555135e0178fee9c63fa00114395986bd49c7d6c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c621ead8d06a0f1d00c5217cc2cfdc90c8c62fa1cb0da0986461ec51fd1766b
+oid sha256:3ea9cac9af94198fada9ef3d4fae4312ce5ac99a95501a7745aeb7f91fcb6b08
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eff8dbefa4ff395a5376144d756cbe824baaab98a892f200d30b7916c24d27cf
+oid sha256:07e9f829a7622427f225a6c2d17e591979f9a3ce0b403f5f12527ef6cddec21d
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd67eb847a256b4f0de5857c5e8a43697485d1a0f6032004d0bc19149d77879c
+oid sha256:db61901b1f811673403412b38c3433989e32dba9ce91026522943ae0e96f1d82
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df3be552cf2524f9ece2b6a286f0ce246d18d14d42f9b8c771a555e051bcee33
+oid sha256:e5fc2897164e26dbf189cf39613143884cb612b3f808a6a18c481ece64d73bc7
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.725212464589235,
+  "epoch": 1.750965748132887,
   "eval_steps": 50,
-  "global_step": 3350,
+  "global_step": 3400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5970,11 +5970,100 @@
       "eval_steps_per_second": 0.93,
       "num_input_tokens_seen": 39175888,
       "step": 3350
+    },
+    {
+      "epoch": 1.7277877929436003,
+      "grad_norm": 10.968051828666288,
+      "learning_rate": 4.788399817602929e-08,
+      "loss": 0.2565,
+      "num_input_tokens_seen": 39234336,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7303631212979655,
+      "grad_norm": 5.1159559645491335,
+      "learning_rate": 3.7835537837338506e-08,
+      "loss": 0.2762,
+      "num_input_tokens_seen": 39292800,
+      "step": 3360
+    },
+    {
+      "epoch": 1.7329384496523308,
+      "grad_norm": 6.735859744015271,
+      "learning_rate": 2.8968690057051828e-08,
+      "loss": 0.2196,
+      "num_input_tokens_seen": 39351272,
+      "step": 3365
+    },
+    {
+      "epoch": 1.7355137780066958,
+      "grad_norm": 3.989003741597172,
+      "learning_rate": 2.128366453743591e-08,
+      "loss": 0.2482,
+      "num_input_tokens_seen": 39409736,
+      "step": 3370
+    },
+    {
+      "epoch": 1.738089106361061,
+      "grad_norm": 5.083412307953648,
+      "learning_rate": 1.4780643030476438e-08,
+      "loss": 0.2778,
+      "num_input_tokens_seen": 39468176,
+      "step": 3375
+    },
+    {
+      "epoch": 1.740664434715426,
+      "grad_norm": 7.4306605849577565,
+      "learning_rate": 9.459779333587104e-09,
+      "loss": 0.2048,
+      "num_input_tokens_seen": 39526688,
+      "step": 3380
+    },
+    {
+      "epoch": 1.7432397630697913,
+      "grad_norm": 4.202839419581782,
+      "learning_rate": 5.3211992859791835e-09,
+      "loss": 0.2296,
+      "num_input_tokens_seen": 39585152,
+      "step": 3385
+    },
+    {
+      "epoch": 1.7458150914241566,
+      "grad_norm": 7.909317855624412,
+      "learning_rate": 2.3650007656805806e-09,
+      "loss": 0.2713,
+      "num_input_tokens_seen": 39643640,
+      "step": 3390
+    },
+    {
+      "epoch": 1.7483904197785218,
+      "grad_norm": 7.880795429819755,
+      "learning_rate": 5.912536872321184e-10,
+      "loss": 0.2964,
+      "num_input_tokens_seen": 39702144,
+      "step": 3395
+    },
+    {
+      "epoch": 1.750965748132887,
+      "grad_norm": 4.00234080349809,
+      "learning_rate": 0.0,
+      "loss": 0.1797,
+      "num_input_tokens_seen": 39760664,
+      "step": 3400
+    },
+    {
+      "epoch": 1.750965748132887,
+      "eval_loss": 0.8603056073188782,
+      "eval_runtime": 16.2474,
+      "eval_samples_per_second": 3.693,
+      "eval_steps_per_second": 0.923,
+      "num_input_tokens_seen": 39760664,
+      "step": 3400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 39175888,
+  "num_input_tokens_seen": 39760664,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5984,12 +6073,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2199919870083072.0,
+  "total_flos": 2232757993603072.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null