Training in progress, step 2250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76b20ab8c2a7403c32454801b8a1cf7e477efa58783a51bc7e3abf420b274c08
+oid sha256:fef821b7ed03f4855afe73a282dfe1eaf891fe7aec11dbae66d77bf1e75802cb
 size 18516456

last-checkpoint/global_step2249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4e0e15067425dd19e23913af5cadb71f7c0ce8e8e4aa92ebeb26f3b70866e88
+size 27700976

last-checkpoint/global_step2249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65544af251b9481248990fe36ee789bde1d9e030cee2b6d5d7e33daf332b7c3e
+size 27700976

last-checkpoint/global_step2249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80689559aea7baa93195feb4e4c40c89aff9367fd678be7d88a6c66663b38ed3
+size 27700976

last-checkpoint/global_step2249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c4abb06aa79e8fe0825c5ad8ff77c8bdc03a26fc1a37371992d32f819b63e2f
+size 27700976

last-checkpoint/global_step2249/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd223d5575f1d4caa0358b7085f8743404ec64526534fd4e13f730325601388
+size 411571

last-checkpoint/global_step2249/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:299465c58a3533bd287fde01ba5114f5a6421d904e8715e5cac80ccde8331aff
+size 411507

last-checkpoint/global_step2249/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6de7e2553a35e5a51c605253c768cde53f5c97a670676902669fa198173d4cd
+size 411507

last-checkpoint/global_step2249/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94c37d01755a8d68a4d6b72f9db4088b481a4af1fcfe1e3f6f7bc555daa306d4
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2199
+global_step2249

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3187a61ccc2722c440dc24ae4a6eefe6b9e5daccf9e92473bbb4483c7751ea77
+oid sha256:1d25cbcbbaa0866ea9c7365cb49b84e805db119693e615f5a1898a6ebfe997e8
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0f2a0df922fb3337cf2562745ebe8d5adf433ca45cb4e3da33a21b48183c000
+oid sha256:a883389afac12125c2c6bf62631b7de0220fdb0020d24cd0c6e8f8858dd3b362
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4d84b5276f687f44c9af60b1e41cd7b93a6d1659e36831a7bc021b5635d663b
+oid sha256:577d49de6d60035e159d9ebb1e6eabef79a55787b14ecea93a6a93c242661779
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d69159433c88b97106cf21b92eb5a3f66f0c826aa268d82a47b3faed1ac86cd
+oid sha256:b5f2dd1c21e06806a9ce39eeab45734dfb8a62b829f91a86d1f65f13102d6242
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6225488c9a450b7edfa6b28ac40ecd217bccdb84073c98a64aefcefa7ee337d2
+oid sha256:6909ad505e808e7099dbcdd8062e5535575cbfa3b4d3a7b7d3390e6a93ed3b49
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.1328869430852433,
+  "epoch": 1.158640226628895,
   "eval_steps": 50,
-  "global_step": 2200,
+  "global_step": 2250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3923,11 +3923,100 @@
       "eval_steps_per_second": 0.939,
       "num_input_tokens_seen": 25725560,
       "step": 2200
+    },
+    {
+      "epoch": 1.1354622714396085,
+      "grad_norm": 4.616007617470174,
+      "learning_rate": 3.0138142386977787e-05,
+      "loss": 0.3465,
+      "num_input_tokens_seen": 25784048,
+      "step": 2205
+    },
+    {
+      "epoch": 1.1380375997939738,
+      "grad_norm": 4.752551024155875,
+      "learning_rate": 2.991522876735154e-05,
+      "loss": 0.3077,
+      "num_input_tokens_seen": 25842512,
+      "step": 2210
+    },
+    {
+      "epoch": 1.140612928148339,
+      "grad_norm": 6.021213921198953,
+      "learning_rate": 2.9692790155527227e-05,
+      "loss": 0.4497,
+      "num_input_tokens_seen": 25900992,
+      "step": 2215
+    },
+    {
+      "epoch": 1.143188256502704,
+      "grad_norm": 8.098592782255322,
+      "learning_rate": 2.9470831812210837e-05,
+      "loss": 0.3811,
+      "num_input_tokens_seen": 25959448,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1457635848570693,
+      "grad_norm": 6.108837560432838,
+      "learning_rate": 2.924935898674992e-05,
+      "loss": 0.4053,
+      "num_input_tokens_seen": 26017936,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1483389132114346,
+      "grad_norm": 7.709937017464705,
+      "learning_rate": 2.902837691700945e-05,
+      "loss": 0.3421,
+      "num_input_tokens_seen": 26076440,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1509142415657996,
+      "grad_norm": 3.840146275079161,
+      "learning_rate": 2.880789082924798e-05,
+      "loss": 0.3228,
+      "num_input_tokens_seen": 26134896,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1534895699201648,
+      "grad_norm": 6.088757703790803,
+      "learning_rate": 2.858790593799405e-05,
+      "loss": 0.3695,
+      "num_input_tokens_seen": 26193368,
+      "step": 2240
+    },
+    {
+      "epoch": 1.15606489827453,
+      "grad_norm": 3.8647543120940844,
+      "learning_rate": 2.8368427445922696e-05,
+      "loss": 0.3463,
+      "num_input_tokens_seen": 26251848,
+      "step": 2245
+    },
+    {
+      "epoch": 1.158640226628895,
+      "grad_norm": 4.425454601086007,
+      "learning_rate": 2.8149460543732664e-05,
+      "loss": 0.3442,
+      "num_input_tokens_seen": 26310336,
+      "step": 2250
+    },
+    {
+      "epoch": 1.158640226628895,
+      "eval_loss": 0.7066138386726379,
+      "eval_runtime": 15.9558,
+      "eval_samples_per_second": 3.76,
+      "eval_steps_per_second": 0.94,
+      "num_input_tokens_seen": 26310336,
+      "step": 2250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 25725560,
+  "num_input_tokens_seen": 26310336,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3942,7 +4031,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1444609144651776.0,
+  "total_flos": 1477446954123264.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null