Training in progress, step 3350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45971737e3e2100e724159295c32390e6a57ebf38af6100e1947303c16949ead
+oid sha256:d8815c16681432a11b5ac188380ad3a07078a835a470d216013977c67742fdcc
 size 18516456

last-checkpoint/global_step3349/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9714545356a6630ff7f755f70b5be0c5e7edf5d539477ee88ea052d1f5598ebd
+size 27700976

last-checkpoint/global_step3349/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d1579920fea0ef33bc5b33ce6cf141616f20b0269e8a7bdd9f0a90f26f6a2c9
+size 27700976

last-checkpoint/global_step3349/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd6db98ef211e2c56dd2d666af47364ceb1233cd2d1ee905f273134b0d08006
+size 27700976

last-checkpoint/global_step3349/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04701505f90deb228f105d51a68b8700254f1af158c358157a7f5164a6337094
+size 27700976

last-checkpoint/global_step3349/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa9df0b0da29b86c93aae4c46944940ad56253950056266aac238ca870ad70eb
+size 411571

last-checkpoint/global_step3349/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886be10efdee881980fd6ce46c1367af0920a4635a04619ca3555bda180eea9a
+size 411507

last-checkpoint/global_step3349/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ee14de80690392657e5287da53bf812aee796c74cd5fc6b8b0e06086a79e86e
+size 411507

last-checkpoint/global_step3349/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24dbd4e75dd77a6428d2b7668112b6f9f43420d98041e84c360bd2332d8b128c
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step3299
+global_step3349

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26d08ced4d0f6490515a22c9e9401cc7f71de8b6e2c1525e9c8dac221d4b80ab
+oid sha256:abe9163f042a56ab41ea5c2436dff084d8a4a6358e7f4cb1f18e04cb69810300
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:182d3e4bee7983edebdece2816547628c5ac3e14f5b53eac54c08ad9cbfbfac7
+oid sha256:3c621ead8d06a0f1d00c5217cc2cfdc90c8c62fa1cb0da0986461ec51fd1766b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:674c98433de6d8eccea8d9f711e54aa6a237220bb981e22133b7735c14dc835f
+oid sha256:eff8dbefa4ff395a5376144d756cbe824baaab98a892f200d30b7916c24d27cf
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e6ca2e2b0a88474561faf5e462ebee1de20672630f977a22533f633eb059d6d
+oid sha256:dd67eb847a256b4f0de5857c5e8a43697485d1a0f6032004d0bc19149d77879c
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecf2871d286bb06cddbfaac7d28300a567a2a36a68cd74d9cb437d09165d13f9
+oid sha256:df3be552cf2524f9ece2b6a286f0ce246d18d14d42f9b8c771a555e051bcee33
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.6994591810455835,
+  "epoch": 1.725212464589235,
   "eval_steps": 50,
-  "global_step": 3300,
+  "global_step": 3350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5881,11 +5881,100 @@
       "eval_steps_per_second": 0.929,
       "num_input_tokens_seen": 38591128,
       "step": 3300
+    },
+    {
+      "epoch": 1.7020345093999485,
+      "grad_norm": 4.935833215525081,
+      "learning_rate": 2.1329118524827662e-07,
+      "loss": 0.2337,
+      "num_input_tokens_seen": 38649640,
+      "step": 3305
+    },
+    {
+      "epoch": 1.7046098377543135,
+      "grad_norm": 5.746920185244728,
+      "learning_rate": 1.9144427761286222e-07,
+      "loss": 0.215,
+      "num_input_tokens_seen": 38708112,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7071851661086788,
+      "grad_norm": 6.501004359690972,
+      "learning_rate": 1.7077534966650766e-07,
+      "loss": 0.2871,
+      "num_input_tokens_seen": 38766624,
+      "step": 3315
+    },
+    {
+      "epoch": 1.709760494463044,
+      "grad_norm": 6.996403813160393,
+      "learning_rate": 1.51284890232406e-07,
+      "loss": 0.3478,
+      "num_input_tokens_seen": 38825104,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7123358228174093,
+      "grad_norm": 5.178545190033401,
+      "learning_rate": 1.3297336026280027e-07,
+      "loss": 0.2055,
+      "num_input_tokens_seen": 38883560,
+      "step": 3325
+    },
+    {
+      "epoch": 1.7149111511717745,
+      "grad_norm": 6.686144266429449,
+      "learning_rate": 1.158411928280645e-07,
+      "loss": 0.2992,
+      "num_input_tokens_seen": 38942040,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7174864795261395,
+      "grad_norm": 4.337439288142164,
+      "learning_rate": 9.988879310649513e-08,
+      "loss": 0.2302,
+      "num_input_tokens_seen": 39000488,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7200618078805048,
+      "grad_norm": 6.5240260149211755,
+      "learning_rate": 8.511653837470212e-08,
+      "loss": 0.265,
+      "num_input_tokens_seen": 39058960,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7226371362348698,
+      "grad_norm": 7.592689596688837,
+      "learning_rate": 7.152477799867719e-08,
+      "loss": 0.3147,
+      "num_input_tokens_seen": 39117416,
+      "step": 3345
+    },
+    {
+      "epoch": 1.725212464589235,
+      "grad_norm": 6.429413076205037,
+      "learning_rate": 5.911383342556143e-08,
+      "loss": 0.2674,
+      "num_input_tokens_seen": 39175888,
+      "step": 3350
+    },
+    {
+      "epoch": 1.725212464589235,
+      "eval_loss": 0.8666485548019409,
+      "eval_runtime": 16.1238,
+      "eval_samples_per_second": 3.721,
+      "eval_steps_per_second": 0.93,
+      "num_input_tokens_seen": 39175888,
+      "step": 3350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 38591128,
+  "num_input_tokens_seen": 39175888,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5900,7 +5989,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2167080547254272.0,
+  "total_flos": 2199919870083072.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null