Training in progress, step 1850, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80957d4d61fd32f325d1e1389afae72c016da18ced07ab4e480f80b2f80d1f1d
+oid sha256:4c71b1c1f3ec0ef5bc6d227c526efd5bb13dac7486cb5bdfe9bfe2263bb8ea0a
 size 18516456

last-checkpoint/global_step1850/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c20314ab90c7fcc9493bb96dc34f6d1cab223e0e759ea6347d160cf5b6e48b2
+size 27700976

last-checkpoint/global_step1850/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4db5c1f94bc37f96cf991b2b15040cc901616d5a2f0c0d9643be014c655c9b4
+size 27700976

last-checkpoint/global_step1850/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66bc6d83d2c439bfa7a45d8a556a1505ce0a192b0cf9487989dec9695e21fc7d
+size 27700976

last-checkpoint/global_step1850/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4552d96bcb9f541aa2f8d385e40d13ecc88071867822e0cd3ba052ba9358c742
+size 27700976

last-checkpoint/global_step1850/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e5bd675da6558cb9f8ed14d9838777f3ce1ac5d82fe11ebd683c0980e0ee45e
+size 411571

last-checkpoint/global_step1850/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc848d833b2f5b030f1e91a948e662a05534a308372acf444d61ab5ac50223c9
+size 411507

last-checkpoint/global_step1850/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1319dfc3bea59ba69f20016a69e27f88a7b5075437b00b6f4c88fba1bcf81e91
+size 411507

last-checkpoint/global_step1850/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:349232bd30fc6a46ae552dedd71ad73c666806d695405c0acde12e60c3b7c2a0
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1800
+global_step1850

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d76f093328c54f2b94a10b8b50dc92fc99ceff9e3949d050a70042526f1d0eb
+oid sha256:7cc13c69d2b97530a3e18634e2f473678ea6880a6b34244c9c86a457f70137e1
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54b172dd00cba9f005761d85fb0804f751caf6e1cc8294d1873354a890cb9909
+oid sha256:fcfd90610e35a8f40098e1413d3e3a1658276c859045d3450f227cab64c9081a
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec6adef733bd7630aa48eff1a6edaabc275d67293dcb0b7a64d71451405d489d
+oid sha256:663ae184d05dd950d27e1f419e0306ca3b141b9d7f075a80804cada62f64a363
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ce1c4ba1932c0f698bef90e0e4e74ded1e0db5fc35282a0815899b8be759e67
+oid sha256:ec4f0a72927b3837a0e73cb5612acf4318c503f3ba2108b3925f750332a49b60
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:586e8022fc0be60f50aebd58453b59ef73708ef9c9ee879c0bef0b407e787025
+oid sha256:5ab5a19cec978f6a84414cd7b1ae1d269e97d77f6437dc3a15a341f43865650e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 0.9271182075714653,
+  "epoch": 0.9528714911151172,
   "eval_steps": 50,
-  "global_step": 1800,
+  "global_step": 1850,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3211,11 +3211,100 @@
       "eval_steps_per_second": 0.939,
       "num_input_tokens_seen": 21053080,
       "step": 1800
+    },
+    {
+      "epoch": 0.9296935359258306,
+      "grad_norm": 6.328873193178562,
+      "learning_rate": 4.902743203531405e-05,
+      "loss": 0.4845,
+      "num_input_tokens_seen": 21111592,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9322688642801957,
+      "grad_norm": 4.7019594666508215,
+      "learning_rate": 4.8784333172436206e-05,
+      "loss": 0.441,
+      "num_input_tokens_seen": 21170024,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9348441926345609,
+      "grad_norm": 4.545287749618146,
+      "learning_rate": 4.854126306025812e-05,
+      "loss": 0.545,
+      "num_input_tokens_seen": 21228480,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9374195209889261,
+      "grad_norm": 7.047942469299444,
+      "learning_rate": 4.829822744742383e-05,
+      "loss": 0.4697,
+      "num_input_tokens_seen": 21286944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9399948493432912,
+      "grad_norm": 3.917758669787159,
+      "learning_rate": 4.8055232081761395e-05,
+      "loss": 0.423,
+      "num_input_tokens_seen": 21345456,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9425701776976565,
+      "grad_norm": 3.442911876713947,
+      "learning_rate": 4.781228271014704e-05,
+      "loss": 0.4715,
+      "num_input_tokens_seen": 21403896,
+      "step": 1830
+    },
+    {
+      "epoch": 0.9451455060520216,
+      "grad_norm": 4.755237925353789,
+      "learning_rate": 4.756938507836929e-05,
+      "loss": 0.5149,
+      "num_input_tokens_seen": 21462360,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9477208344063868,
+      "grad_norm": 5.3552741805060275,
+      "learning_rate": 4.732654493099291e-05,
+      "loss": 0.5403,
+      "num_input_tokens_seen": 21520864,
+      "step": 1840
+    },
+    {
+      "epoch": 0.950296162760752,
+      "grad_norm": 3.417134377266731,
+      "learning_rate": 4.708376801122321e-05,
+      "loss": 0.4757,
+      "num_input_tokens_seen": 21579376,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9528714911151172,
+      "grad_norm": 4.6802756294331855,
+      "learning_rate": 4.6841060060770154e-05,
+      "loss": 0.4347,
+      "num_input_tokens_seen": 21637848,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9528714911151172,
+      "eval_loss": 0.6808218359947205,
+      "eval_runtime": 16.1166,
+      "eval_samples_per_second": 3.723,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 21637848,
+      "step": 1850
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 21053080,
+  "num_input_tokens_seen": 21637848,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3230,7 +3319,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1182185020981248.0,
+  "total_flos": 1215022643281920.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null