Training in progress, step 1400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10b824529fcf1b1cc25b9074d2e972d61eae3cfa9d99a178c08c460258fd9cfb
+oid sha256:f2ea934ab728b1244cad904f211efc8dbea34bf120d0aaee69bc85b174ce604c
 size 18516456

last-checkpoint/global_step1400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65a32506fe9a075e8bb36535dc8dbcdb5b1ce6a1e0d1997f92b88f46d29d5dd8
+size 27700976

last-checkpoint/global_step1400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f4b2a134a207aac795fafdcc138df8fdf042c4133bd6620913e59b0d9c27457
+size 27700976

last-checkpoint/global_step1400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15dd8165e977edcbd6994c5489318245c9ad54541455e3124e9c073749122323
+size 27700976

last-checkpoint/global_step1400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ad19787732e8d8273fa1a687f45f2456da566d021f2d4115579ccd661a1028f
+size 27700976

last-checkpoint/global_step1400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec36d552bf6c107a3880a898ec20ae09651970c11b4a63ac3665580b5ffb88d
+size 411571

last-checkpoint/global_step1400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b9f5b60fd43cde7e00ab49e3793d827c723f499d77b9bf0439866dd5645ddd4
+size 411507

last-checkpoint/global_step1400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:def6be980722c6b0e706f08920073db9d89e86c803748b7f3de1245f5b6687a6
+size 411507

last-checkpoint/global_step1400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80cdb09ea542cc0993daee7484026b223c5352c21c5c12b0593dbecbee4ea110
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1350
+global_step1400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
+oid sha256:fe66a68e61de2221b30fd9749bc68b45a1474bb2cc95901bca9557ac87909355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
+oid sha256:0cd4f3162e46c3bb0f1fc4d3c52c7c33e60f56764458e0c8a73c3810b0a25f8c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
+oid sha256:185cc99aaa81b1b49b3ddc74aa6f97aa3036330983a7b69d52bd191057f9a5d5
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
+oid sha256:0e37403c30cb4309e54e5defdde1906486716fc859274035d44aaac5d48a97ba
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca5b091f7e9fe97f38f540a5a26edd08efe79a114c6bc8ff4d4c0d9acb5b20f2
+oid sha256:ba18a87eab7efca964506a3a18bdf9452eae09db2b3c77bd3a82db3283b5abc0
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.712917685508728,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1350",
-  "epoch": 0.695338655678599,
+  "best_metric": 0.6643603444099426,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1400",
+  "epoch": 0.7210919392222508,
   "eval_steps": 50,
-  "global_step": 1350,
+  "global_step": 1400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2410,11 +2410,100 @@
       "eval_steps_per_second": 0.933,
       "num_input_tokens_seen": 15789848,
       "step": 1350
+    },
+    {
+      "epoch": 0.6979139840329642,
+      "grad_norm": 2.128495144345323,
+      "learning_rate": 7.030720984447279e-05,
+      "loss": 0.7005,
+      "num_input_tokens_seen": 15848328,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7004893123873294,
+      "grad_norm": 1.9954206386005497,
+      "learning_rate": 7.008477123264848e-05,
+      "loss": 0.7406,
+      "num_input_tokens_seen": 15906824,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7030646407416946,
+      "grad_norm": 2.2104679425901397,
+      "learning_rate": 6.986185761302224e-05,
+      "loss": 0.73,
+      "num_input_tokens_seen": 15965312,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7056399690960597,
+      "grad_norm": 1.4881688553415275,
+      "learning_rate": 6.963847425753403e-05,
+      "loss": 0.7069,
+      "num_input_tokens_seen": 16023824,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7082152974504249,
+      "grad_norm": 1.7307886623214839,
+      "learning_rate": 6.941462644923318e-05,
+      "loss": 0.6859,
+      "num_input_tokens_seen": 16082280,
+      "step": 1375
+    },
+    {
+      "epoch": 0.7107906258047901,
+      "grad_norm": 1.996363722225207,
+      "learning_rate": 6.919031948215335e-05,
+      "loss": 0.7254,
+      "num_input_tokens_seen": 16140800,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7133659541591553,
+      "grad_norm": 1.9723274395570518,
+      "learning_rate": 6.896555866118741e-05,
+      "loss": 0.717,
+      "num_input_tokens_seen": 16199320,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7159412825135205,
+      "grad_norm": 1.741253496639104,
+      "learning_rate": 6.87403493019619e-05,
+      "loss": 0.7094,
+      "num_input_tokens_seen": 16257768,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7185166108678857,
+      "grad_norm": 1.6218002074106608,
+      "learning_rate": 6.851469673071143e-05,
+      "loss": 0.7862,
+      "num_input_tokens_seen": 16316264,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7210919392222508,
+      "grad_norm": 1.7586707307941614,
+      "learning_rate": 6.828860628415253e-05,
+      "loss": 0.7359,
+      "num_input_tokens_seen": 16374784,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7210919392222508,
+      "eval_loss": 0.6643603444099426,
+      "eval_runtime": 16.1894,
+      "eval_samples_per_second": 3.706,
+      "eval_steps_per_second": 0.927,
+      "num_input_tokens_seen": 16374784,
+      "step": 1400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 15789848,
+  "num_input_tokens_seen": 16374784,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2429,7 +2518,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 886628555423744.0,
+  "total_flos": 919473251942400.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null