Training in progress, step 1450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2ea934ab728b1244cad904f211efc8dbea34bf120d0aaee69bc85b174ce604c
+oid sha256:c22365f9f1922257bce2b624ba131f902a872d3af6a64b3fc1e949516f00f5ac
 size 18516456

last-checkpoint/global_step1450/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c8959c735a1f2fc25bf1d4928b76efa4bb530eb830280ca22ca1633aa52d376
+size 27700976

last-checkpoint/global_step1450/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f85a672550122c013bfd5d71900a7266dd62f855af7c974a333f7a40dea95c
+size 27700976

last-checkpoint/global_step1450/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f6cefdf65440a82655cd36f4399caf641f34ba34512af3342227658a10dd026
+size 27700976

last-checkpoint/global_step1450/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9dbf6e1beb3b51e1eae928a14e8e1b7f54e2387d23032e91b3e09b668bb4f34
+size 27700976

last-checkpoint/global_step1450/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc9cf1986a83b78bb824f6697b3219dc70f628d204f6308b3f5c0110c8ec35a
+size 411571

last-checkpoint/global_step1450/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa0aa15ab897602cd8484da3aeaeb273f468cc1ae87ee3cad1a86b4b8ac57b22
+size 411507

last-checkpoint/global_step1450/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:223d06577f0627ab6d548c0af2814bcdad88b0a8c0c2157d9f9f6b236f33679a
+size 411507

last-checkpoint/global_step1450/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39561dbca5b73760a417b8083b6ccb8825ee6b9f832540b6a585f4a97eef0a5a
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1400
+global_step1450

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe66a68e61de2221b30fd9749bc68b45a1474bb2cc95901bca9557ac87909355
+oid sha256:dd4f3298d54e9509917373bcd359e11c92a8e0aa77b2cc0825602efd186ad77e
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cd4f3162e46c3bb0f1fc4d3c52c7c33e60f56764458e0c8a73c3810b0a25f8c
+oid sha256:5fdab02a791039ff46df83a272c972ab0f366fcea91338790dc016dbbbf6de80
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:185cc99aaa81b1b49b3ddc74aa6f97aa3036330983a7b69d52bd191057f9a5d5
+oid sha256:e2808762a5f80587f05c1e2e8de26b5aef1010331cea366649319d593dbb0e66
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e37403c30cb4309e54e5defdde1906486716fc859274035d44aaac5d48a97ba
+oid sha256:c052112cbe3f325fd4543a02558091b80c9c8abcd8db8822be9a8306eab6f2b7
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba18a87eab7efca964506a3a18bdf9452eae09db2b3c77bd3a82db3283b5abc0
+oid sha256:22e337e693b8d4ecad5d49a11b7f0649adec1748625a9390ba4ddabc0e901287
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6643603444099426,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1400",
-  "epoch": 0.7210919392222508,
+  "epoch": 0.7468452227659027,
   "eval_steps": 50,
-  "global_step": 1400,
+  "global_step": 1450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2499,11 +2499,100 @@
       "eval_steps_per_second": 0.927,
       "num_input_tokens_seen": 16374784,
       "step": 1400
+    },
+    {
+      "epoch": 0.723667267576616,
+      "grad_norm": 2.665622720042704,
+      "learning_rate": 6.806208330935766e-05,
+      "loss": 0.706,
+      "num_input_tokens_seen": 16433288,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7262425959309812,
+      "grad_norm": 2.123869663010538,
+      "learning_rate": 6.783513316362855e-05,
+      "loss": 0.6714,
+      "num_input_tokens_seen": 16491784,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7288179242853464,
+      "grad_norm": 1.584213945279146,
+      "learning_rate": 6.760776121436962e-05,
+      "loss": 0.693,
+      "num_input_tokens_seen": 16550272,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7313932526397116,
+      "grad_norm": 2.2481839233017764,
+      "learning_rate": 6.737997283896103e-05,
+      "loss": 0.7005,
+      "num_input_tokens_seen": 16608704,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7339685809940767,
+      "grad_norm": 2.4818230151927643,
+      "learning_rate": 6.715177342463145e-05,
+      "loss": 0.6573,
+      "num_input_tokens_seen": 16667200,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7365439093484419,
+      "grad_norm": 2.5398594354263486,
+      "learning_rate": 6.692316836833065e-05,
+      "loss": 0.6751,
+      "num_input_tokens_seen": 16725704,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7391192377028071,
+      "grad_norm": 2.7486055345229343,
+      "learning_rate": 6.6694163076602e-05,
+      "loss": 0.6173,
+      "num_input_tokens_seen": 16784192,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7416945660571723,
+      "grad_norm": 5.356237563459472,
+      "learning_rate": 6.646476296545434e-05,
+      "loss": 0.728,
+      "num_input_tokens_seen": 16842704,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7442698944115375,
+      "grad_norm": 2.088505948846248,
+      "learning_rate": 6.623497346023418e-05,
+      "loss": 0.743,
+      "num_input_tokens_seen": 16901176,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7468452227659027,
+      "grad_norm": 2.2198436340262,
+      "learning_rate": 6.60047999954972e-05,
+      "loss": 0.6291,
+      "num_input_tokens_seen": 16959632,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7468452227659027,
+      "eval_loss": 0.753077507019043,
+      "eval_runtime": 16.0383,
+      "eval_samples_per_second": 3.741,
+      "eval_steps_per_second": 0.935,
+      "num_input_tokens_seen": 16959632,
+      "step": 1450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 16374784,
+  "num_input_tokens_seen": 16959632,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2518,7 +2607,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 919473251942400.0,
+  "total_flos": 952313688883200.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null