Training in progress, step 2300, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fef821b7ed03f4855afe73a282dfe1eaf891fe7aec11dbae66d77bf1e75802cb
+oid sha256:81cfb0ee5389022e6fe86935c49629e5dcdacdc2bddc774ca113221a60166576
 size 18516456

last-checkpoint/global_step2299/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82c5963b53be7bf4ac988682be435545c879b80467ee44486db35514a3dd16f8
+size 27700976

last-checkpoint/global_step2299/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44aa3615153bf253d897162ea20869121b8ae8927d27c350e7a308f12dc94a1a
+size 27700976

last-checkpoint/global_step2299/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6224f246a16fea820cc08f89f1f4e6de627075ef496f2a9c1e3b0e8061dec8b8
+size 27700976

last-checkpoint/global_step2299/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e229d62f02874b000b920fdd497f10658e226f07fa6e27065139834506b0a58
+size 27700976

last-checkpoint/global_step2299/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19668b184fd53a9ee5458aa56ddc85a6ca6e8304d2bff2460c472b9f14503bb3
+size 411571

last-checkpoint/global_step2299/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b400e32c99c430b7030ac0d8014b994f40222986d788094945a538a49140891
+size 411507

last-checkpoint/global_step2299/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21ee002c79d745c31b162ae6ceed53b333aa893fdbbdc34482e6406dbdd3c64f
+size 411507

last-checkpoint/global_step2299/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5a1e8275ab5d8b357ecea79a83241cf6e62a645d321f2a23a6ae5efec49b3bc
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2249
+global_step2299

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d25cbcbbaa0866ea9c7365cb49b84e805db119693e615f5a1898a6ebfe997e8
+oid sha256:f12bf3da75454e5aae4644f2a1d46fdf90f68e680dbf5bdaa86861f825d32d80
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a883389afac12125c2c6bf62631b7de0220fdb0020d24cd0c6e8f8858dd3b362
+oid sha256:2faef1b8798e7516fd96ee7b3363866a8f97ca2d0ec5a8dd27bbfe70b0c6a733
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:577d49de6d60035e159d9ebb1e6eabef79a55787b14ecea93a6a93c242661779
+oid sha256:7f35f6d27fbd414dd4285d91816b37fb6b97ff10fbba4c074d56ad2a7f723033
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5f2dd1c21e06806a9ce39eeab45734dfb8a62b829f91a86d1f65f13102d6242
+oid sha256:ea5c64dfc7e3b2729065483dfef8e4bb0af0d9bae32df888d258ee3c2859d676
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6909ad505e808e7099dbcdd8062e5535575cbfa3b4d3a7b7d3390e6a93ed3b49
+oid sha256:25aeb77fef385fcfd5658b6aaea11fb8aa276239b7ba19a4f3504b86f45fbcc9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.158640226628895,
+  "epoch": 1.184393510172547,
   "eval_steps": 50,
-  "global_step": 2250,
+  "global_step": 2300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4012,11 +4012,100 @@
       "eval_steps_per_second": 0.94,
       "num_input_tokens_seen": 26310336,
       "step": 2250
+    },
+    {
+      "epoch": 1.1612155549832603,
+      "grad_norm": 6.312367706992343,
+      "learning_rate": 2.7931010410023518e-05,
+      "loss": 0.3547,
+      "num_input_tokens_seen": 26368840,
+      "step": 2255
+    },
+    {
+      "epoch": 1.1637908833376256,
+      "grad_norm": 6.429493717694784,
+      "learning_rate": 2.771308221117309e-05,
+      "loss": 0.3125,
+      "num_input_tokens_seen": 26427280,
+      "step": 2260
+    },
+    {
+      "epoch": 1.1663662116919906,
+      "grad_norm": 6.993677707266103,
+      "learning_rate": 2.749568110121545e-05,
+      "loss": 0.3521,
+      "num_input_tokens_seen": 26485760,
+      "step": 2265
+    },
+    {
+      "epoch": 1.1689415400463559,
+      "grad_norm": 5.03743116566882,
+      "learning_rate": 2.7278812221718924e-05,
+      "loss": 0.281,
+      "num_input_tokens_seen": 26544224,
+      "step": 2270
+    },
+    {
+      "epoch": 1.1715168684007211,
+      "grad_norm": 5.828198718501714,
+      "learning_rate": 2.7062480701664488e-05,
+      "loss": 0.3653,
+      "num_input_tokens_seen": 26602712,
+      "step": 2275
+    },
+    {
+      "epoch": 1.1740921967550864,
+      "grad_norm": 6.1247491578050655,
+      "learning_rate": 2.6846691657324473e-05,
+      "loss": 0.3964,
+      "num_input_tokens_seen": 26661160,
+      "step": 2280
+    },
+    {
+      "epoch": 1.1766675251094514,
+      "grad_norm": 6.231155247277189,
+      "learning_rate": 2.663145019214163e-05,
+      "loss": 0.3119,
+      "num_input_tokens_seen": 26719648,
+      "step": 2285
+    },
+    {
+      "epoch": 1.1792428534638166,
+      "grad_norm": 6.501604840456734,
+      "learning_rate": 2.6416761396608362e-05,
+      "loss": 0.3832,
+      "num_input_tokens_seen": 26778112,
+      "step": 2290
+    },
+    {
+      "epoch": 1.1818181818181819,
+      "grad_norm": 5.377003761278013,
+      "learning_rate": 2.6202630348146324e-05,
+      "loss": 0.3277,
+      "num_input_tokens_seen": 26836592,
+      "step": 2295
+    },
+    {
+      "epoch": 1.184393510172547,
+      "grad_norm": 4.826044073542379,
+      "learning_rate": 2.598906211098643e-05,
+      "loss": 0.3877,
+      "num_input_tokens_seen": 26895096,
+      "step": 2300
+    },
+    {
+      "epoch": 1.184393510172547,
+      "eval_loss": 0.727741539478302,
+      "eval_runtime": 15.9289,
+      "eval_samples_per_second": 3.767,
+      "eval_steps_per_second": 0.942,
+      "num_input_tokens_seen": 26895096,
+      "step": 2300
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 26310336,
+  "num_input_tokens_seen": 26895096,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4031,7 +4120,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1477446954123264.0,
+  "total_flos": 1510284389515264.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null