Training in progress, step 2500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcb059e4fc556edf0deec1236fbeb6b876aa7055fe07000f0612ab6a0a9c5f0c
+oid sha256:ad9c5a90f0316ee953467cae5ec9a849c3cc7b35324d075196476f5e2a6e857e
 size 18516456

last-checkpoint/global_step2499/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fcc3f5d47981eca52aac72c6f1923e8e0b1ac2c99f7de1fb71a1119a2fa16e3
+size 27700976

last-checkpoint/global_step2499/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89147f56d24767b4393162cef41e78d244f6f91fe989c692401bb8a24d7097f0
+size 27700976

last-checkpoint/global_step2499/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99ddc4d3b8340e811524cbed30fc7cc07ac1370e1b47a428e151bddffc516ac5
+size 27700976

last-checkpoint/global_step2499/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5997e47e1d70d8913e1277da8de8f8effff04d72f786314af60edca7b5e6f64
+size 27700976

last-checkpoint/global_step2499/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5439f58a29e0366a7f50646773fb60ae2a5bb18562e3e838d0c9c82494b2bccf
+size 411571

last-checkpoint/global_step2499/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4397da1199b3676b64dda73bd387563bfbc197a42301176b630448704ab76503
+size 411507

last-checkpoint/global_step2499/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5488ff43420c1760e1387d8e3f7bcb220f9b9293aebfa2a49ff4fb3705841c9c
+size 411507

last-checkpoint/global_step2499/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3836948132309c757f9c1b8d26443f387effa62d14f97840356a6b4d7ff376de
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2449
+global_step2499

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a97c73c15a2a5b2de7dc426a700b2053aee43809425431c513cc5e3aab6c2107
+oid sha256:0362dfd92e8da01e4a0deedcbd1c493b8162d5d1d84d5a4c1cd210c556f2cf9b
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1296b339c1b16ab7e14352a269004d20ede428aef748283fb0a6650d62f58129
+oid sha256:e65c5adee1a22c5343e38495a6905880496fb22d5e3ec5b16b87aadb731969d2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000b1637f5e73170f2337500a6a083df3a43d967d642b6c3a68f60deb6c3b960
+oid sha256:f8d8858483b0c6944d55621cc2633469e3e0d04c48b6671eee92d4abab2352c2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8bcb6e7802f6d888bc099642911087298cfb1adf7053a2d43a67192a53404ef
+oid sha256:ca42ef4f7a2f8c2285c4cf6cef585dcc0b132b21e8bb33d96d53b6db837f5e54
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf7df6dd05dd94a7aa9222f36c7d07fc54cb5fa02909dfb44c3de4cef2e774f9
+oid sha256:be1e9cd300c4f4c1fc9be3848ef7e995abd4a81c17c7a3b103813aaad4725565
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.2616533608035025,
+  "epoch": 1.2874066443471541,
   "eval_steps": 50,
-  "global_step": 2450,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4368,11 +4368,100 @@
       "eval_steps_per_second": 0.935,
       "num_input_tokens_seen": 28649256,
       "step": 2450
+    },
+    {
+      "epoch": 1.2642286891578676,
+      "grad_norm": 10.005201788297592,
+      "learning_rate": 1.967458152065857e-05,
+      "loss": 0.2664,
+      "num_input_tokens_seen": 28707736,
+      "step": 2455
+    },
+    {
+      "epoch": 1.2668040175122328,
+      "grad_norm": 4.744134155404128,
+      "learning_rate": 1.9481612060899646e-05,
+      "loss": 0.3692,
+      "num_input_tokens_seen": 28766232,
+      "step": 2460
+    },
+    {
+      "epoch": 1.269379345866598,
+      "grad_norm": 8.49200897563331,
+      "learning_rate": 1.928936436551661e-05,
+      "loss": 0.315,
+      "num_input_tokens_seen": 28824688,
+      "step": 2465
+    },
+    {
+      "epoch": 1.271954674220963,
+      "grad_norm": 5.112500789477909,
+      "learning_rate": 1.9097842981195834e-05,
+      "loss": 0.3536,
+      "num_input_tokens_seen": 28883176,
+      "step": 2470
+    },
+    {
+      "epoch": 1.2745300025753283,
+      "grad_norm": 4.93472430343828,
+      "learning_rate": 1.8907052437446272e-05,
+      "loss": 0.3143,
+      "num_input_tokens_seen": 28941592,
+      "step": 2475
+    },
+    {
+      "epoch": 1.2771053309296936,
+      "grad_norm": 4.6754631245280365,
+      "learning_rate": 1.871699724649244e-05,
+      "loss": 0.3114,
+      "num_input_tokens_seen": 29000064,
+      "step": 2480
+    },
+    {
+      "epoch": 1.2796806592840588,
+      "grad_norm": 7.198381813960669,
+      "learning_rate": 1.8527681903167644e-05,
+      "loss": 0.3327,
+      "num_input_tokens_seen": 29058496,
+      "step": 2485
+    },
+    {
+      "epoch": 1.2822559876384239,
+      "grad_norm": 9.221713217692685,
+      "learning_rate": 1.833911088480767e-05,
+      "loss": 0.2543,
+      "num_input_tokens_seen": 29116992,
+      "step": 2490
+    },
+    {
+      "epoch": 1.284831315992789,
+      "grad_norm": 8.499870267936974,
+      "learning_rate": 1.8151288651144893e-05,
+      "loss": 0.2854,
+      "num_input_tokens_seen": 29175496,
+      "step": 2495
+    },
+    {
+      "epoch": 1.2874066443471541,
+      "grad_norm": 4.289294450742717,
+      "learning_rate": 1.796421964420285e-05,
+      "loss": 0.3221,
+      "num_input_tokens_seen": 29233968,
+      "step": 2500
+    },
+    {
+      "epoch": 1.2874066443471541,
+      "eval_loss": 0.7222262620925903,
+      "eval_runtime": 16.106,
+      "eval_samples_per_second": 3.725,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 29233968,
+      "step": 2500
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 28649256,
+  "num_input_tokens_seen": 29233968,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4387,7 +4476,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1608790114041856.0,
+  "total_flos": 1641625797525504.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null