Training in progress, step 1700, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:919717a8f83cc7597c4b3c1c6f8d6f3d9da7bb13e3737cb159de4e4e604b9ef9
+oid sha256:ba9914360b2dead740bb4f8973f7876b182880fd878912194cb890333ef7f606
 size 18516456

last-checkpoint/global_step1700/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f013102d025a0d93efddb53355cd46d133d540c05efd52903ac0e7df2b3a5a7
+size 27700976

last-checkpoint/global_step1700/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b67057336dd5fc1d2e096cb5948311380f08eab0b592030c98e1c23d06e4cf18
+size 27700976

last-checkpoint/global_step1700/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd4069f79d7af41073bdc6230a9f76d524e8139a7cbbcad75ec9bc78f29fb83
+size 27700976

last-checkpoint/global_step1700/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0900113e897cbbde7284a3e85c986e82d7ea60e287a384a35faa4c82fe9e5ca6
+size 27700976

last-checkpoint/global_step1700/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90ad7fce713cc4c6aac6ee2e3650f41ff5283d99963b83f91237e69637d182eb
+size 411571

last-checkpoint/global_step1700/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6513101107ca1980d33a5e4d6a1aa48fa89f014eb205a44405a520e8f4cf57ea
+size 411507

last-checkpoint/global_step1700/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0de1889354594f3a0eb87a002836aa15e9e2142ec0caadfd90235a3df462fae3
+size 411507

last-checkpoint/global_step1700/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9a67e6701e53e7386b0e2c7217ddd342c00b1cb8adc55c4fd155a01cf21c088
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1650
+global_step1700

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a90384755f5b036b42b1a465b39dbf24a925a02c04294f9d684bc1de7f4db1e5
+oid sha256:9e7c17922709137dd2f358be87cc431f7959a56821e2d051582e81679e2d388e
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7621b41e55056218f97d5b32ae116de3304a677b9f27b6a62170d83a2bbff176
+oid sha256:eaf0ec3cc431efdc1cb595b27e74fc020021d53f5c11850d7490a79bf42b71d6
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:997e9debadfd125b5c8b66ee6dd79ced3d40d353ff9250475f3814fd950012a6
+oid sha256:148fd0a663c121acf74d815964bc0d08d07065e8503b03adfd967fdaf63a6abc
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f0256032419959580948d742425f66782bc8eb029126a091669a42c6ee0eba4
+oid sha256:56589f6c34527656fe0dc752874cfc1460efbb5c0c3f874c4cd09d6415dd16c1
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a67704d17501733446a3916f5d09739e8293f473d2f1a5f63158f87200fdaa72
+oid sha256:dcafe96450339d5b6e4ff3a8d909c473a6afaab9bb34bc798805cded569190ca
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 0.8498583569405099,
+  "epoch": 0.8756116404841617,
   "eval_steps": 50,
-  "global_step": 1650,
+  "global_step": 1700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2944,11 +2944,100 @@
       "eval_steps_per_second": 0.936,
       "num_input_tokens_seen": 19298848,
       "step": 1650
+    },
+    {
+      "epoch": 0.8524336852948751,
+      "grad_norm": 3.2430676664218496,
+      "learning_rate": 5.6305258170538676e-05,
+      "loss": 0.584,
+      "num_input_tokens_seen": 19357304,
+      "step": 1655
+    },
+    {
+      "epoch": 0.8550090136492403,
+      "grad_norm": 3.140559424454581,
+      "learning_rate": 5.606396838947988e-05,
+      "loss": 0.5544,
+      "num_input_tokens_seen": 19415800,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8575843420036054,
+      "grad_norm": 3.993528386539066,
+      "learning_rate": 5.582253519467432e-05,
+      "loss": 0.6269,
+      "num_input_tokens_seen": 19474256,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8601596703579707,
+      "grad_norm": 2.202747116085024,
+      "learning_rate": 5.558096429605263e-05,
+      "loss": 0.5073,
+      "num_input_tokens_seen": 19532736,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8627349987123358,
+      "grad_norm": 4.4094334133851625,
+      "learning_rate": 5.533926140680221e-05,
+      "loss": 0.5319,
+      "num_input_tokens_seen": 19591184,
+      "step": 1675
+    },
+    {
+      "epoch": 0.865310327066701,
+      "grad_norm": 4.01821546567579,
+      "learning_rate": 5.509743224323203e-05,
+      "loss": 0.4525,
+      "num_input_tokens_seen": 19649656,
+      "step": 1680
+    },
+    {
+      "epoch": 0.8678856554210662,
+      "grad_norm": 5.3033277992950385,
+      "learning_rate": 5.485548252463749e-05,
+      "loss": 0.5276,
+      "num_input_tokens_seen": 19708144,
+      "step": 1685
+    },
+    {
+      "epoch": 0.8704609837754314,
+      "grad_norm": 5.124737819396939,
+      "learning_rate": 5.4613417973165106e-05,
+      "loss": 0.5482,
+      "num_input_tokens_seen": 19766592,
+      "step": 1690
+    },
+    {
+      "epoch": 0.8730363121297966,
+      "grad_norm": 3.47304956996904,
+      "learning_rate": 5.4371244313677225e-05,
+      "loss": 0.4656,
+      "num_input_tokens_seen": 19825064,
+      "step": 1695
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "grad_norm": 6.394279811127835,
+      "learning_rate": 5.4128967273616625e-05,
+      "loss": 0.5701,
+      "num_input_tokens_seen": 19883504,
+      "step": 1700
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "eval_loss": 0.7144017815589905,
+      "eval_runtime": 16.1358,
+      "eval_samples_per_second": 3.718,
+      "eval_steps_per_second": 0.93,
+      "num_input_tokens_seen": 19883504,
+      "step": 1700
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 19298848,
+  "num_input_tokens_seen": 19883504,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2963,7 +3052,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1083674466975744.0,
+  "total_flos": 1116505381011456.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null