Training in progress, step 3100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddb314047f6c123f2abaf4b7b141070b2299dee4fc65b9c0ec7277494419776e
+oid sha256:fae7873da4e864e2fb488bd3cadfc022d2333da4131c0ae5cf85dc328c20163a
 size 18516456

last-checkpoint/global_step3099/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08687bc050847df5b1d4953be6a89f6d3c396827e885d0bcc904a774af2dc598
+size 27700976

last-checkpoint/global_step3099/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4733e82a0123279183a91682295ed830955b7d6a11f3d2816718531939fa5868
+size 27700976

last-checkpoint/global_step3099/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5273cdd4fd6a02955cb6ea2f065a5478b96897779f3d7bb6e2ebab4cecf11343
+size 27700976

last-checkpoint/global_step3099/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2969f9e18a587de6f0a1fdb86ec2ca507f578ef8bc2927cb7a07238006f0da
+size 27700976

last-checkpoint/global_step3099/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf33710d8a0a286c6b871306b7dff759032c69cf094b7661792c302c4d2260c1
+size 411571

last-checkpoint/global_step3099/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:294962ac09933e5d95bed7e9e297e544fca51d19b97d63abe7746a3b01897f44
+size 411507

last-checkpoint/global_step3099/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fa8a8916e57a2a21340ba8575e5706ead69bca64e1c2ba188b4657c59b2a10b
+size 411507

last-checkpoint/global_step3099/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4695607ab9f4215123045a20add0b458cc604e18f82c0e0f93ebcb35cdd09a88
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step3049
+global_step3099

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:381f090b12cbb3fcce976bd2e72d07a7786154c2848cc881d75715648e7c4dc2
+oid sha256:f819cf2c7fe8719d427eaa6ec9775983252517a512a5eb46040621049bb676c6
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4321dc1beb6c245d4a16a8fb2bed2ce2a40d89e5a9c611c4572f63a09523846c
+oid sha256:2a78180f5e29cd98437c35591ab402d42bd98c8a12a813c23dfdfe59b5a850b5
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b79020117e301cd96518c9d1e3eb43a609bc85799f14bac63d9c572fc04cb89
+oid sha256:5f43520b5104ee75a59749a66f5367557a7f191243b389309b6acad668af81d2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5429623c7323b4a820ea3d76194bc456c2affeeb1f3af8978aec5aee11b2d1ef
+oid sha256:2c502c4157180ff49243cd3eeb13c063be673f3bf094a73b62e8315ca6f65442
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65132092aa084390575a45bea99d0dcabb8005a8fae760edf38c6251571f2afa
+oid sha256:d44cc93262eaa36be09e5c05cb4a41bcbfdd9c2c86797c6c680d23dd8657e6ca
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.5706927633273242,
+  "epoch": 1.596446046870976,
   "eval_steps": 50,
-  "global_step": 3050,
+  "global_step": 3100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5436,11 +5436,100 @@
       "eval_steps_per_second": 0.936,
       "num_input_tokens_seen": 35666976,
       "step": 3050
+    },
+    {
+      "epoch": 1.5732680916816895,
+      "grad_norm": 5.787936921221981,
+      "learning_rate": 2.7886498764184588e-06,
+      "loss": 0.2514,
+      "num_input_tokens_seen": 35725456,
+      "step": 3055
+    },
+    {
+      "epoch": 1.5758434200360547,
+      "grad_norm": 7.052716790363759,
+      "learning_rate": 2.7091379149682685e-06,
+      "loss": 0.3091,
+      "num_input_tokens_seen": 35783912,
+      "step": 3060
+    },
+    {
+      "epoch": 1.5784187483904197,
+      "grad_norm": 4.6737853290480915,
+      "learning_rate": 2.6307443893812843e-06,
+      "loss": 0.2629,
+      "num_input_tokens_seen": 35842376,
+      "step": 3065
+    },
+    {
+      "epoch": 1.580994076744785,
+      "grad_norm": 8.400296818269052,
+      "learning_rate": 2.5534711536759404e-06,
+      "loss": 0.3065,
+      "num_input_tokens_seen": 35900824,
+      "step": 3070
+    },
+    {
+      "epoch": 1.58356940509915,
+      "grad_norm": 3.909241159865706,
+      "learning_rate": 2.4773200353756798e-06,
+      "loss": 0.2577,
+      "num_input_tokens_seen": 35959264,
+      "step": 3075
+    },
+    {
+      "epoch": 1.5861447334535153,
+      "grad_norm": 5.227660314173737,
+      "learning_rate": 2.4022928354656473e-06,
+      "loss": 0.2359,
+      "num_input_tokens_seen": 36017760,
+      "step": 3080
+    },
+    {
+      "epoch": 1.5887200618078805,
+      "grad_norm": 5.407491053931616,
+      "learning_rate": 2.3283913283502044e-06,
+      "loss": 0.1897,
+      "num_input_tokens_seen": 36076280,
+      "step": 3085
+    },
+    {
+      "epoch": 1.5912953901622457,
+      "grad_norm": 5.771594174948701,
+      "learning_rate": 2.2556172618108997e-06,
+      "loss": 0.286,
+      "num_input_tokens_seen": 36134784,
+      "step": 3090
+    },
+    {
+      "epoch": 1.593870718516611,
+      "grad_norm": 5.508770087080472,
+      "learning_rate": 2.183972356965125e-06,
+      "loss": 0.2733,
+      "num_input_tokens_seen": 36193288,
+      "step": 3095
+    },
+    {
+      "epoch": 1.596446046870976,
+      "grad_norm": 6.343942326218544,
+      "learning_rate": 2.113458308225458e-06,
+      "loss": 0.2351,
+      "num_input_tokens_seen": 36251744,
+      "step": 3100
+    },
+    {
+      "epoch": 1.596446046870976,
+      "eval_loss": 0.8650907874107361,
+      "eval_runtime": 16.0989,
+      "eval_samples_per_second": 3.727,
+      "eval_steps_per_second": 0.932,
+      "num_input_tokens_seen": 36251744,
+      "step": 3100
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 35666976,
+  "num_input_tokens_seen": 36251744,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5455,7 +5544,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2002870983983104.0,
+  "total_flos": 2035710493982720.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null