Training in progress, step 2450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdf5f54c0020018c5eca1b4c1055878f2c1e58bfa3f188c951ed0eed9b279f0d
+oid sha256:dcb059e4fc556edf0deec1236fbeb6b876aa7055fe07000f0612ab6a0a9c5f0c
 size 18516456

last-checkpoint/global_step2449/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312b7d9e865cdff8180b1853c9c534b21f97892cc57c756a034a07f820c4f138
+size 27700976

last-checkpoint/global_step2449/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2660c50e02c64468c5886292e901515341622cb1d48b1279b727eb768542d31b
+size 27700976

last-checkpoint/global_step2449/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f61170bae6fda22d97311adb07c4cbededa78a0be5aae1af0516b04b431c96
+size 27700976

last-checkpoint/global_step2449/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e46e477cc0d200ebca990c54a0ad533ed6fc4f3a80c19443626d49216d313f4
+size 27700976

last-checkpoint/global_step2449/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82025a80cba943dce5adc862e22781467fe53eb775fc39e338f3f094ddb3942f
+size 411571

last-checkpoint/global_step2449/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9920568a21c644da1ac7c1ca0dde0003a400ad93c0d01d13e3c39d7e20b5ee
+size 411507

last-checkpoint/global_step2449/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae1f288118cb19b70ec47c255f31f11cb69aa9c13edae83cb39473e89ab93d88
+size 411507

last-checkpoint/global_step2449/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657f34b262e72c913843d6673a0971a8c8f1cd47a8a9ec7dfa23ec74383242c5
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2399
+global_step2449

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b8aff7a1897a7eaf48c78ea1f8115c061edfa2b6fa42280e2c1c58fe66b1f8a
+oid sha256:a97c73c15a2a5b2de7dc426a700b2053aee43809425431c513cc5e3aab6c2107
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d73a7524f07999ef35d5d9b107dcc1678eae2ada841644e1bd00ec0734368c2
+oid sha256:1296b339c1b16ab7e14352a269004d20ede428aef748283fb0a6650d62f58129
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:149a2ed30e88bf94d622f8d7693f382286a49ac536a3f63efc50cab63f6b9f39
+oid sha256:000b1637f5e73170f2337500a6a083df3a43d967d642b6c3a68f60deb6c3b960
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18f12c1b5aae2b7d4bb968649839fc7ff1ce6131508baad4b633693b04cee910
+oid sha256:b8bcb6e7802f6d888bc099642911087298cfb1adf7053a2d43a67192a53404ef
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33f198cd3e0b21f350098b16849fc1ced7c1d5261e89a1c503f4a0d3ce688f30
+oid sha256:bf7df6dd05dd94a7aa9222f36c7d07fc54cb5fa02909dfb44c3de4cef2e774f9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.2359000772598505,
+  "epoch": 1.2616533608035025,
   "eval_steps": 50,
-  "global_step": 2400,
+  "global_step": 2450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4279,11 +4279,100 @@
       "eval_steps_per_second": 0.941,
       "num_input_tokens_seen": 28064552,
       "step": 2400
+    },
+    {
+      "epoch": 1.2384754056142158,
+      "grad_norm": 4.430659190759614,
+      "learning_rate": 2.1642960822587878e-05,
+      "loss": 0.2416,
+      "num_input_tokens_seen": 28123016,
+      "step": 2405
+    },
+    {
+      "epoch": 1.241050733968581,
+      "grad_norm": 4.985077238748084,
+      "learning_rate": 2.1443027488440338e-05,
+      "loss": 0.3007,
+      "num_input_tokens_seen": 28181464,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2436260623229463,
+      "grad_norm": 11.21074775906945,
+      "learning_rate": 2.124376953090456e-05,
+      "loss": 0.2655,
+      "num_input_tokens_seen": 28239920,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2462013906773113,
+      "grad_norm": 6.8116545197169724,
+      "learning_rate": 2.104519166246059e-05,
+      "loss": 0.3075,
+      "num_input_tokens_seen": 28298432,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2487767190316765,
+      "grad_norm": 10.87615610006345,
+      "learning_rate": 2.0847298579504344e-05,
+      "loss": 0.3537,
+      "num_input_tokens_seen": 28356904,
+      "step": 2425
+    },
+    {
+      "epoch": 1.2513520473860418,
+      "grad_norm": 3.9413743825159133,
+      "learning_rate": 2.065009496223638e-05,
+      "loss": 0.2993,
+      "num_input_tokens_seen": 28415384,
+      "step": 2430
+    },
+    {
+      "epoch": 1.2539273757404068,
+      "grad_norm": 3.3043013555966407,
+      "learning_rate": 2.045358547455138e-05,
+      "loss": 0.2752,
+      "num_input_tokens_seen": 28473848,
+      "step": 2435
+    },
+    {
+      "epoch": 1.256502704094772,
+      "grad_norm": 3.6641007142438338,
+      "learning_rate": 2.0257774763927655e-05,
+      "loss": 0.2975,
+      "num_input_tokens_seen": 28532312,
+      "step": 2440
+    },
+    {
+      "epoch": 1.2590780324491373,
+      "grad_norm": 6.306122720573227,
+      "learning_rate": 2.0062667461317426e-05,
+      "loss": 0.4051,
+      "num_input_tokens_seen": 28590784,
+      "step": 2445
+    },
+    {
+      "epoch": 1.2616533608035025,
+      "grad_norm": 4.823015256168698,
+      "learning_rate": 1.9868268181037185e-05,
+      "loss": 0.2966,
+      "num_input_tokens_seen": 28649256,
+      "step": 2450
+    },
+    {
+      "epoch": 1.2616533608035025,
+      "eval_loss": 0.7485548853874207,
+      "eval_runtime": 16.0437,
+      "eval_samples_per_second": 3.74,
+      "eval_steps_per_second": 0.935,
+      "num_input_tokens_seen": 28649256,
+      "step": 2450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 28064552,
+  "num_input_tokens_seen": 28649256,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4298,7 +4387,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1575955876020224.0,
+  "total_flos": 1608790114041856.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null