Training in progress, step 950, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd3d3749c1bd6d89c5172acc406cbf998cb265de2f8a059883bd34374666967b
+oid sha256:9c25a3efd75d542958c5e34b7273f618978b6b36bd238438b10b03a0bfabd174
 size 18516456

last-checkpoint/global_step950/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c881994b39776045ba6a3e0043760467d6a3dc65e4e3073384090d7079c6d4c
+size 27700976

last-checkpoint/global_step950/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3968dac460aefc7c50063d9956c7d24d784525ab3d59a329360070722b7b25
+size 27700976

last-checkpoint/global_step950/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ed45d8129e5b0647eb41f62ac33a693928c5530ed867e7b6c234e4b6738a4b0
+size 27700976

last-checkpoint/global_step950/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41ca8d8a60c28495df06f864662f9ca01a7cb07d2a2f0d696511f444452deeed
+size 27700976

last-checkpoint/global_step950/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2b6a70cd029279c6bbc6df4cd65e9a6e7015439bcb6f6e7752438a5ee7992e3
+size 411571

last-checkpoint/global_step950/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:866dbe7dbd6f8eb35527ac50f7a6b950a2bbd23f49dedcdfdc4ce44e4fab3ba7
+size 411507

last-checkpoint/global_step950/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad8e8dd84e1791fb37053ac9be587ac8a7e29d17344c17cbc1952436cdbd6f94
+size 411507

last-checkpoint/global_step950/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43aeac4610a7cebbf9e5ebc29d3145599901d1a133f08cd549a510e5383c512
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step900
+global_step950

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8f22ced19e790cc864cefe3b7c711d9ae631c44f95d42fb4829688cc3de0153
+oid sha256:7f2439da621f14c22b4f733e91bfc9de6b506d28d7b8d6f3eaca2e0b4f24c078
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e0407513eba77d34cbf3adf0e59a58bd80716f4f00f414854253637e82be43d
+oid sha256:c9e3fb386557f376b8946af5b8c91f9418f374dddb2ad9da4868b1ef16778c32
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6060636c023258ce9b965e244b8a58b4c99d5784dde4405b39737550ef50cd4f
+oid sha256:dc7774d06045635bece9e960378fdc6913bf7bbbc903444cc570d1ca6ac25645
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c24ccdfdcde39cb2265c82c50c36ffdfcc670f757aba4bcf4bb0fdc6d1373c4c
+oid sha256:d98c54a80a914fecf43d06ea81432499f46e70664f1d04651bf339163e30fa9e
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:feb4015894f59edc29c71bc4938b5d4ab98daad34a38a3d387b308b3b1d4b280
+oid sha256:b9f224baf5bd2044314606c1d88f84cce32f1b37c43c15835b14e72f6a72a4fc
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.8779178261756897,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-850",
-  "epoch": 0.4635591037857327,
+  "epoch": 0.4893123873293845,
   "eval_steps": 50,
-  "global_step": 900,
+  "global_step": 950,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1609,11 +1609,100 @@
       "eval_steps_per_second": 0.935,
       "num_input_tokens_seen": 10526712,
       "step": 900
+    },
+    {
+      "epoch": 0.46613443214009787,
+      "grad_norm": 0.18755280770432675,
+      "learning_rate": 8.775851227988656e-05,
+      "loss": 0.8955,
+      "num_input_tokens_seen": 10585232,
+      "step": 905
+    },
+    {
+      "epoch": 0.46870976049446306,
+      "grad_norm": 0.16684040416821233,
+      "learning_rate": 8.759866980070963e-05,
+      "loss": 0.8951,
+      "num_input_tokens_seen": 10643728,
+      "step": 910
+    },
+    {
+      "epoch": 0.47128508884882825,
+      "grad_norm": 0.33346521793095785,
+      "learning_rate": 8.743793810744654e-05,
+      "loss": 0.8951,
+      "num_input_tokens_seen": 10702240,
+      "step": 915
+    },
+    {
+      "epoch": 0.4738604172031934,
+      "grad_norm": 0.23650054707790025,
+      "learning_rate": 8.727632100142551e-05,
+      "loss": 0.9066,
+      "num_input_tokens_seen": 10760656,
+      "step": 920
+    },
+    {
+      "epoch": 0.4764357455575586,
+      "grad_norm": 0.20217442955339224,
+      "learning_rate": 8.711382230491493e-05,
+      "loss": 0.8953,
+      "num_input_tokens_seen": 10819128,
+      "step": 925
+    },
+    {
+      "epoch": 0.47901107391192377,
+      "grad_norm": 0.1648307621403396,
+      "learning_rate": 8.695044586103296e-05,
+      "loss": 0.8961,
+      "num_input_tokens_seen": 10877600,
+      "step": 930
+    },
+    {
+      "epoch": 0.48158640226628896,
+      "grad_norm": 0.25983065938238986,
+      "learning_rate": 8.678619553365659e-05,
+      "loss": 0.8965,
+      "num_input_tokens_seen": 10936088,
+      "step": 935
+    },
+    {
+      "epoch": 0.48416173062065415,
+      "grad_norm": 0.17882463002474594,
+      "learning_rate": 8.662107520733027e-05,
+      "loss": 0.9018,
+      "num_input_tokens_seen": 10994560,
+      "step": 940
+    },
+    {
+      "epoch": 0.4867370589750193,
+      "grad_norm": 0.14644012846994445,
+      "learning_rate": 8.64550887871741e-05,
+      "loss": 0.8944,
+      "num_input_tokens_seen": 11053016,
+      "step": 945
+    },
+    {
+      "epoch": 0.4893123873293845,
+      "grad_norm": 0.23751630760966444,
+      "learning_rate": 8.628824019879137e-05,
+      "loss": 0.8852,
+      "num_input_tokens_seen": 11111520,
+      "step": 950
+    },
+    {
+      "epoch": 0.4893123873293845,
+      "eval_loss": 0.8915690183639526,
+      "eval_runtime": 16.2589,
+      "eval_samples_per_second": 3.69,
+      "eval_steps_per_second": 0.923,
+      "num_input_tokens_seen": 11111520,
+      "step": 950
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 10526712,
+  "num_input_tokens_seen": 11111520,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1628,7 +1717,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 591081256517632.0,
+  "total_flos": 623922645303296.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null