Training in progress, step 1600, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c433a8aa8f1650b5f131001712aa61d13d2d36f41df4d65d21c83dee89eb91ab
+oid sha256:f11513d52bcbabbebdae5c22382754c386d07651a25539e1cdcc0f4b4982a0f7
 size 18516456

last-checkpoint/global_step1600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46463053469e5c128d18b59faffc7aa119512da4469fc23c9640b1c123d5397f
+size 27700976

last-checkpoint/global_step1600/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0004c0959e3595e03710b21e4bd4fa3ba2c02502ed83372000018de6f50e1d3
+size 27700976

last-checkpoint/global_step1600/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ee1ed10ee81a3036175bcbe2b80223be8b540d8636ada699171bfa75f66a1c2
+size 27700976

last-checkpoint/global_step1600/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b54a8cff3c72b28b47e16db62ced38ea1cdd159c1f590fed439d1fdc6f9878a
+size 27700976

last-checkpoint/global_step1600/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26acda4de0790975ebc9d4f0c12f3aebbcbccfe2cfd61f8c85e0ea9e9d99950a
+size 411571

last-checkpoint/global_step1600/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:760fc496088881394b803189c6aa215917ad5a1bb62ddd42914e38a5d2948529
+size 411507

last-checkpoint/global_step1600/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf8aa6a9173571e234a2795a5345c292d9c46961b42871d2a86e31748f61f13
+size 411507

last-checkpoint/global_step1600/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ebdb6d3a5f2a383b8517fde2dce340179a2adc82a0a53d88e7375c7669fd5e7
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1550
+global_step1600

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
+oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
+oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
+oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
+oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eb069683e7f84aa36296476346fc663361d9b05ad7b09b71f22f44afdb0ea48
+oid sha256:f2487a6c511ed8055eb0842d87966b09ae8b62c1b4514727282ca413d6e9c4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.632923424243927,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1500",
-  "epoch": 0.7983517898532063,
+  "best_metric": 0.6319106221199036,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
+  "epoch": 0.8241050733968581,
   "eval_steps": 50,
-  "global_step": 1550,
+  "global_step": 1600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2766,11 +2766,100 @@
       "eval_steps_per_second": 0.936,
       "num_input_tokens_seen": 18129304,
       "step": 1550
+    },
+    {
+      "epoch": 0.8009271182075715,
+      "grad_norm": 2.9735507158511987,
+      "learning_rate": 6.109217817775139e-05,
+      "loss": 0.5681,
+      "num_input_tokens_seen": 18187728,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8035024465619367,
+      "grad_norm": 3.6620315644598778,
+      "learning_rate": 6.085494954896156e-05,
+      "loss": 0.6292,
+      "num_input_tokens_seen": 18246192,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8060777749163018,
+      "grad_norm": 4.03631122919402,
+      "learning_rate": 6.061746419901388e-05,
+      "loss": 0.6512,
+      "num_input_tokens_seen": 18304632,
+      "step": 1565
+    },
+    {
+      "epoch": 0.808653103270667,
+      "grad_norm": 4.0040288177360805,
+      "learning_rate": 6.0379727744471936e-05,
+      "loss": 0.5476,
+      "num_input_tokens_seen": 18363136,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8112284316250322,
+      "grad_norm": 3.9448861517599996,
+      "learning_rate": 6.014174580783794e-05,
+      "loss": 0.5632,
+      "num_input_tokens_seen": 18421592,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8138037599793974,
+      "grad_norm": 3.8400680048739435,
+      "learning_rate": 5.990352401741981e-05,
+      "loss": 0.6225,
+      "num_input_tokens_seen": 18480104,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8163790883337626,
+      "grad_norm": 2.7981339113543284,
+      "learning_rate": 5.9665068007197976e-05,
+      "loss": 0.5801,
+      "num_input_tokens_seen": 18538600,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8189544166881277,
+      "grad_norm": 4.290843515697908,
+      "learning_rate": 5.94263834166923e-05,
+      "loss": 0.6364,
+      "num_input_tokens_seen": 18597104,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8215297450424929,
+      "grad_norm": 3.9001572117535566,
+      "learning_rate": 5.918747589082853e-05,
+      "loss": 0.6088,
+      "num_input_tokens_seen": 18655584,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8241050733968581,
+      "grad_norm": 3.5623412341260363,
+      "learning_rate": 5.8948351079804875e-05,
+      "loss": 0.6564,
+      "num_input_tokens_seen": 18714072,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8241050733968581,
+      "eval_loss": 0.6319106221199036,
+      "eval_runtime": 16.0199,
+      "eval_samples_per_second": 3.745,
+      "eval_steps_per_second": 0.936,
+      "num_input_tokens_seen": 18714072,
+      "step": 1600
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 18129304,
+  "num_input_tokens_seen": 18714072,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2785,7 +2874,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1017997776650240.0,
+  "total_flos": 1050836028358656.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null