Training in progress, step 550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ba83473ace8f2ecdbc048dafa2d00257b4fa3a981f66b8f547625be4d8d6a90
+oid sha256:ad0a41144b9d8eea3b3f1de8e9e5e7c14c303c31098798928859d47cbd861a53
 size 18516456

last-checkpoint/global_step550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2dd708bce8156f81e78f48af2f45249f50b48b1b10d3cccc754368c848a42c4
+size 27700976

last-checkpoint/global_step550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7707a634b8052289bb808cab58425d2c29bb9ab889ca015141f434440020c92
+size 27700976

last-checkpoint/global_step550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da189be86b251daa6a3d75c53724aa18a89d2bec0b692343472d0c9f34db7ca9
+size 27700976

last-checkpoint/global_step550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ddb526250554a510bf29668756d7e7bd859ccdb9723fc265e8b2c030c2bc23
+size 27700976

last-checkpoint/global_step550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5fae2e4f2b1854438b05b6f32d9c6d08952bdf18b07e6c8719824f3cbb0fa55
+size 411571

last-checkpoint/global_step550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbbae66b52b89cb6be888983982f3c0ca6e723bfd1f4512b76672b9dc82aa8e1
+size 411507

last-checkpoint/global_step550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37f690bd1d0259ae507a97d5795da4f0e813887d5a4e4978f511ea048053e38
+size 411507

last-checkpoint/global_step550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:059ca587c41d1e52a95054bc3e00214fd350ff19fc2a534776d29cee2872f2e5
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step500
+global_step550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdd1f02cb20d3f4f7e0dd26fea62af57e5e71316163f926a28ed6cf89a9f3777
+oid sha256:ae78313eb528c8d3695eebaf4de3539bd0a0bc6ee18c66af1ee183442f1758a0
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc6d54ba2aa85e2f895439a1b787ec947b848a1c34ea5a3a28821572bf2b9fec
+oid sha256:1b38031f60d9e88601d369ef46bcdcf2b5b03f2cb4ba93853bcb2328df7ebb7c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b6927d26551cddd8e35b34b43e79bd58f8b6027b6a481bb6a563a3652addeb4
+oid sha256:f58092375c93d237cd0e3149aecfbf83e2acdae46279e07a32920d01cb507e64
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8295b1be8e66b4b30cb905dc48cfc717c027e427937b8142d00ae9de8106c6a
+oid sha256:83cd4bbff9962da7ec6787fcea8d65df7096917f9a5902e249ba7aee8887fe5f
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab7567aeded1bd7ef9f3ba115e57865dd25bb569f9711f33170eb2a51540c216
+oid sha256:6e8b881f6464ee76e192f8a5dbebbec89a38d087d3502270b9c7e6038613f3b2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.8908902406692505,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-300",
-  "epoch": 0.25753283543651817,
+  "epoch": 0.28328611898017,
   "eval_steps": 50,
-  "global_step": 500,
+  "global_step": 550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -897,11 +897,100 @@
       "eval_steps_per_second": 0.874,
       "num_input_tokens_seen": 5848048,
       "step": 500
+    },
+    {
+      "epoch": 0.26010816379088336,
+      "grad_norm": 0.25461397268106634,
+      "learning_rate": 9.736925561061871e-05,
+      "loss": 0.8954,
+      "num_input_tokens_seen": 5906512,
+      "step": 505
+    },
+    {
+      "epoch": 0.2626834921452485,
+      "grad_norm": 0.38602603275675745,
+      "learning_rate": 9.729086208503174e-05,
+      "loss": 0.8927,
+      "num_input_tokens_seen": 5965024,
+      "step": 510
+    },
+    {
+      "epoch": 0.2652588204996137,
+      "grad_norm": 0.150082825225123,
+      "learning_rate": 9.721135012358156e-05,
+      "loss": 0.898,
+      "num_input_tokens_seen": 6023496,
+      "step": 515
+    },
+    {
+      "epoch": 0.2678341488539789,
+      "grad_norm": 0.26881662025899655,
+      "learning_rate": 9.713072160673777e-05,
+      "loss": 0.9016,
+      "num_input_tokens_seen": 6082000,
+      "step": 520
+    },
+    {
+      "epoch": 0.2704094772083441,
+      "grad_norm": 0.5039123575147229,
+      "learning_rate": 9.704897844137673e-05,
+      "loss": 0.8842,
+      "num_input_tokens_seen": 6140480,
+      "step": 525
+    },
+    {
+      "epoch": 0.27298480556270927,
+      "grad_norm": 0.27836945453098666,
+      "learning_rate": 9.696612256073633e-05,
+      "loss": 0.8921,
+      "num_input_tokens_seen": 6198968,
+      "step": 530
+    },
+    {
+      "epoch": 0.2755601339170744,
+      "grad_norm": 0.22936338891946384,
+      "learning_rate": 9.688215592437039e-05,
+      "loss": 0.8979,
+      "num_input_tokens_seen": 6257464,
+      "step": 535
+    },
+    {
+      "epoch": 0.2781354622714396,
+      "grad_norm": 0.396486857609105,
+      "learning_rate": 9.679708051810221e-05,
+      "loss": 0.8951,
+      "num_input_tokens_seen": 6315944,
+      "step": 540
+    },
+    {
+      "epoch": 0.2807107906258048,
+      "grad_norm": 0.4751226662261396,
+      "learning_rate": 9.67108983539777e-05,
+      "loss": 0.9149,
+      "num_input_tokens_seen": 6374408,
+      "step": 545
+    },
+    {
+      "epoch": 0.28328611898017,
+      "grad_norm": 0.26829103885131056,
+      "learning_rate": 9.662361147021779e-05,
+      "loss": 0.9013,
+      "num_input_tokens_seen": 6432936,
+      "step": 550
+    },
+    {
+      "epoch": 0.28328611898017,
+      "eval_loss": 0.9001271724700928,
+      "eval_runtime": 16.9878,
+      "eval_samples_per_second": 3.532,
+      "eval_steps_per_second": 0.883,
+      "num_input_tokens_seen": 6432936,
+      "step": 550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 5848048,
+  "num_input_tokens_seen": 6432936,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -916,7 +1005,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 328347984855040.0,
+  "total_flos": 361192817164288.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null