Training in progress, step 1000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c25a3efd75d542958c5e34b7273f618978b6b36bd238438b10b03a0bfabd174
+oid sha256:29643b889b27240ceabef62c2230355b76d0a5ae5c9a27da11cbaeefa32d3892
 size 18516456

last-checkpoint/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef963570eb1ba4df539bddf25b054f5a48483f8f19ecf88b558127c936ca3d27
+size 27700976

last-checkpoint/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff4137c7eb7fcf0069f76df3f74f2fa406e147818355e76440597bbdca0e68f9
+size 27700976

last-checkpoint/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a052a8567494ae21ca0adf956ed248fc314e5e9f033dad61c8f5649874273a5
+size 27700976

last-checkpoint/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ceb885a88d4743d1d8c86c993b612c8ec02a15c07858e037d4ec9a4b390d5f8
+size 27700976

last-checkpoint/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10205d764f4a82fccb26cf20b76ca5be802170731caa2972798ed6f18b776a40
+size 411571

last-checkpoint/global_step1000/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bb44baf193ce857784d6a1205848bb539aa13db12a0ea7fd7585d55958f884d
+size 411507

last-checkpoint/global_step1000/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7032497c258c6a252bb2ac5d2e037f5b6daa91946766482ca66010b516894b4
+size 411507

last-checkpoint/global_step1000/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78c1da31eca391f472c936cc9270e4abfc939bdd70512bb92816381789cfdcfc
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step950
+global_step1000

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f2439da621f14c22b4f733e91bfc9de6b506d28d7b8d6f3eaca2e0b4f24c078
+oid sha256:7d73dfcc09cf3d6f08149535e03920234febc15f7e9a166987f3bc01ee871abf
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9e3fb386557f376b8946af5b8c91f9418f374dddb2ad9da4868b1ef16778c32
+oid sha256:4396a64b6da4868d060d1e3c7c9ccb12c39d63bd0f7b146d2512400aff4c769c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc7774d06045635bece9e960378fdc6913bf7bbbc903444cc570d1ca6ac25645
+oid sha256:95877efc8fb5eb302819ee7effca4222569cdcfdebb9fa5d9846e68ed9e833fe
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d98c54a80a914fecf43d06ea81432499f46e70664f1d04651bf339163e30fa9e
+oid sha256:e9fa4f23377f00fdde731da68a8690098617a1fdd912e03cdaa8bde87c493179
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9f224baf5bd2044314606c1d88f84cce32f1b37c43c15835b14e72f6a72a4fc
+oid sha256:bce7739c5bb5cf50e8f1c942e662e33e6aa589036d55e6fddd63bdf3171c1cae
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.8779178261756897,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-850",
-  "epoch": 0.4893123873293845,
+  "best_metric": 0.860200047492981,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1000",
+  "epoch": 0.5150656708730363,
   "eval_steps": 50,
-  "global_step": 950,
+  "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1698,11 +1698,100 @@
       "eval_steps_per_second": 0.923,
       "num_input_tokens_seen": 11111520,
       "step": 950
+    },
+    {
+      "epoch": 0.49188771568374967,
+      "grad_norm": 0.3904846319143667,
+      "learning_rate": 8.612053338817581e-05,
+      "loss": 0.9087,
+      "num_input_tokens_seen": 11170016,
+      "step": 955
+    },
+    {
+      "epoch": 0.49446304403811486,
+      "grad_norm": 0.44920450892911645,
+      "learning_rate": 8.595197232161824e-05,
+      "loss": 0.8915,
+      "num_input_tokens_seen": 11228496,
+      "step": 960
+    },
+    {
+      "epoch": 0.49703837239248005,
+      "grad_norm": 0.6093857047738649,
+      "learning_rate": 8.578256098561275e-05,
+      "loss": 0.8836,
+      "num_input_tokens_seen": 11286928,
+      "step": 965
+    },
+    {
+      "epoch": 0.49961370074684525,
+      "grad_norm": 0.6282945106836194,
+      "learning_rate": 8.561230338676239e-05,
+      "loss": 0.9116,
+      "num_input_tokens_seen": 11345400,
+      "step": 970
+    },
+    {
+      "epoch": 0.5021890291012104,
+      "grad_norm": 0.3187294296147391,
+      "learning_rate": 8.544120355168451e-05,
+      "loss": 0.8809,
+      "num_input_tokens_seen": 11403912,
+      "step": 975
+    },
+    {
+      "epoch": 0.5047643574555756,
+      "grad_norm": 0.4019889420836467,
+      "learning_rate": 8.526926552691544e-05,
+      "loss": 0.8895,
+      "num_input_tokens_seen": 11462344,
+      "step": 980
+    },
+    {
+      "epoch": 0.5073396858099408,
+      "grad_norm": 0.4762279449607594,
+      "learning_rate": 8.509649337881483e-05,
+      "loss": 0.8674,
+      "num_input_tokens_seen": 11520808,
+      "step": 985
+    },
+    {
+      "epoch": 0.509915014164306,
+      "grad_norm": 1.7062273050040726,
+      "learning_rate": 8.492289119346943e-05,
+      "loss": 0.8832,
+      "num_input_tokens_seen": 11579248,
+      "step": 990
+    },
+    {
+      "epoch": 0.5124903425186711,
+      "grad_norm": 0.7896696939552226,
+      "learning_rate": 8.474846307659658e-05,
+      "loss": 0.8581,
+      "num_input_tokens_seen": 11637712,
+      "step": 995
+    },
+    {
+      "epoch": 0.5150656708730363,
+      "grad_norm": 0.9287129351980297,
+      "learning_rate": 8.457321315344694e-05,
+      "loss": 0.8635,
+      "num_input_tokens_seen": 11696200,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5150656708730363,
+      "eval_loss": 0.860200047492981,
+      "eval_runtime": 16.1196,
+      "eval_samples_per_second": 3.722,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 11696200,
+      "step": 1000
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 11111520,
+  "num_input_tokens_seen": 11696200,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1717,7 +1806,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 623922645303296.0,
+  "total_flos": 656753805492224.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null