Training in progress, step 2700, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:827ad766e3a13984fdb182a9e1ff3663479a4a48e1ebea8bd0fa17625232d440
+oid sha256:87c5131642e0134183eead6ee8a652dcec18d08a73880b46e923d3ba034a1f2c
 size 18516456

last-checkpoint/global_step2699/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c1953dd55a71d816604c9d88f5dfaf3de7cd1ff785816c4692c773827c2ad9
+size 27700976

last-checkpoint/global_step2699/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0a9c2679718849ab4b5d4b2f1d614cc1567d24c74158b6955041b79af75809
+size 27700976

last-checkpoint/global_step2699/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523ee62505a7c73d100c34d0a02ed8a16d6b35bf45179043c6a4cdeee2af7e77
+size 27700976

last-checkpoint/global_step2699/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e9918d979a0f9cb8ee62ff3c09dca43ec4f7d49cdd8ad9a2db3d4dca84f22d
+size 27700976

last-checkpoint/global_step2699/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc5e000dda8c9fd913e37f23f94be6c564e7be0729e32e4803afcd7fa6884f4
+size 411571

last-checkpoint/global_step2699/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b602913ea9e3a7f86da6723379d18b2b0e40385954be7b0ae584fc9967b2fb4a
+size 411507

last-checkpoint/global_step2699/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0485fe4c0ce08f8978f2d62d20e978fb3e99467ef8df597291bd1c294084aaae
+size 411507

last-checkpoint/global_step2699/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8408e6b1fca8b179b6e186db6945579ebf2fb33f98daf209b41784641dad2483
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2649
+global_step2699

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e599331812a34463d102d64a4034a0b702a893f362f752003aa577fe71dcc1d
+oid sha256:5d8d3c7739f9787ea797b86ff1b3a51f9e68197835ba3178915a8a77558f67fc
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ed431e5e71393a0174ad2fd492755f8c1142596f1af3bfe7827c1f8f815dd80
+oid sha256:a22a57799bc43e59db67d9a787ed73040020c5f35990602033f4dab1318787d7
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2e70789f26a9f56b6b779e87cb1a405615af81562a256e5afe579f40972e827
+oid sha256:29a624b936b77a04d6bfb6940acdd65a710bf39452e419e7ddb5c40fb2261072
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c8c18bc74d5211e761da269c814d7da0687633993838ec22e81ac939a14e91b
+oid sha256:3a79306817d4440cd621149537e8cf216b60f847fc6f9531a6147426aa02bb07
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfcf85c736e61a53c653b2d1f3342ce104fa9dc3f2c57b7753ecb2c25635e267
+oid sha256:5e5dcca4048a125fff8fd284657b0498882f3efcb97d36e331842fc3d6d7b6e6
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.3646664949781098,
+  "epoch": 1.3904197785217616,
   "eval_steps": 50,
-  "global_step": 2650,
+  "global_step": 2700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4724,11 +4724,100 @@
       "eval_steps_per_second": 0.933,
       "num_input_tokens_seen": 30988344,
       "step": 2650
+    },
+    {
+      "epoch": 1.367241823332475,
+      "grad_norm": 5.445887797084714,
+      "learning_rate": 1.2562061892553473e-05,
+      "loss": 0.3207,
+      "num_input_tokens_seen": 31046848,
+      "step": 2655
+    },
+    {
+      "epoch": 1.36981715168684,
+      "grad_norm": 8.28343197617098,
+      "learning_rate": 1.2401330199290367e-05,
+      "loss": 0.3001,
+      "num_input_tokens_seen": 31105352,
+      "step": 2660
+    },
+    {
+      "epoch": 1.3723924800412053,
+      "grad_norm": 6.0349779847885054,
+      "learning_rate": 1.224148772011346e-05,
+      "loss": 0.2858,
+      "num_input_tokens_seen": 31163848,
+      "step": 2665
+    },
+    {
+      "epoch": 1.3749678083955703,
+      "grad_norm": 6.430225669948217,
+      "learning_rate": 1.2082538235320929e-05,
+      "loss": 0.2338,
+      "num_input_tokens_seen": 31222360,
+      "step": 2670
+    },
+    {
+      "epoch": 1.3775431367499356,
+      "grad_norm": 7.550675916086161,
+      "learning_rate": 1.1924485504091565e-05,
+      "loss": 0.2212,
+      "num_input_tokens_seen": 31280840,
+      "step": 2675
+    },
+    {
+      "epoch": 1.3801184651043008,
+      "grad_norm": 9.927835245980713,
+      "learning_rate": 1.1767333264395736e-05,
+      "loss": 0.3131,
+      "num_input_tokens_seen": 31339264,
+      "step": 2680
+    },
+    {
+      "epoch": 1.382693793458666,
+      "grad_norm": 6.940248775417007,
+      "learning_rate": 1.1611085232907132e-05,
+      "loss": 0.3616,
+      "num_input_tokens_seen": 31397744,
+      "step": 2685
+    },
+    {
+      "epoch": 1.385269121813031,
+      "grad_norm": 13.50108715364713,
+      "learning_rate": 1.14557451049147e-05,
+      "loss": 0.3153,
+      "num_input_tokens_seen": 31456240,
+      "step": 2690
+    },
+    {
+      "epoch": 1.3878444501673963,
+      "grad_norm": 5.379761157260886,
+      "learning_rate": 1.1301316554235397e-05,
+      "loss": 0.3044,
+      "num_input_tokens_seen": 31514744,
+      "step": 2695
+    },
+    {
+      "epoch": 1.3904197785217616,
+      "grad_norm": 6.480605347127299,
+      "learning_rate": 1.114780323312724e-05,
+      "loss": 0.3163,
+      "num_input_tokens_seen": 31573240,
+      "step": 2700
+    },
+    {
+      "epoch": 1.3904197785217616,
+      "eval_loss": 0.7473158240318298,
+      "eval_runtime": 16.166,
+      "eval_samples_per_second": 3.711,
+      "eval_steps_per_second": 0.928,
+      "num_input_tokens_seen": 31573240,
+      "step": 2700
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 30988344,
+  "num_input_tokens_seen": 31573240,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4743,7 +4832,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1740143807954944.0,
+  "total_flos": 1772988198813696.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null