Training in progress, step 3350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edfec1beb3478ee41909b5ce24ea198317f07af35edd0e3dbe2802570cfa4c2f
+oid sha256:1b41c1be8b974e693981390b4f8ea3183b990f4c6cbeb85e09c310968df2c797
 size 29034840

last-checkpoint/global_step3350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d20b600d06facb5d1a65de24264de7232da2e2cd6f969ab08631a24840a3405
+size 43429616

last-checkpoint/global_step3350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2b008e2b80435e1bd3bec146b2acf3573291d2d3f56e6e6243e3f4eb986ca65
+size 43429616

last-checkpoint/global_step3350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41228dce70389c5be02f1d9fd4103e635a6d41918c34dc3d3e6a22a9d2fbdf3c
+size 43429616

last-checkpoint/global_step3350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd214d9e5759263f6575a06133cc7923a0629469860b9537d1a52db881455bc5
+size 43429616

last-checkpoint/global_step3350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:073795d432f58b231e7787ae71bffd4b21abaa4a026ff18ff2e2ed8893c36861
+size 637299

last-checkpoint/global_step3350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:643c6a357e2e8c148a6add3e575bb83cf59b2d675d40d1e16b4a4d241c96db90
+size 637171

last-checkpoint/global_step3350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2f145fc244b1bf5a757743dfcb4b470273c31ce1d3708ccf6edfdc0edfd08ec
+size 637171

last-checkpoint/global_step3350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61ca17421513fa3bb768f1adbd5c20c63c1ca5515293cf864bfe65f82c03da1f
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3300
+global_step3350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d8a584abdea9bed1d2dc22d8c9034ba07ae85ad6668d6cd55747af816dcecc1
+oid sha256:26d08ced4d0f6490515a22c9e9401cc7f71de8b6e2c1525e9c8dac221d4b80ab
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7862a59bb6480df16b96e840172772c30d9d3037819d18063df33360e83b04a2
+oid sha256:182d3e4bee7983edebdece2816547628c5ac3e14f5b53eac54c08ad9cbfbfac7
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b509c445a6a6ed6d0215c0e232a66489dc20e39b5202d274561183fd43ef3e5
+oid sha256:674c98433de6d8eccea8d9f711e54aa6a237220bb981e22133b7735c14dc835f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1cf1a6a26872b9918c1086ace4fcda88a76d9b6eef501300fd67bf0cc8946d81
+oid sha256:4e6ca2e2b0a88474561faf5e462ebee1de20672630f977a22533f633eb059d6d
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecf2871d286bb06cddbfaac7d28300a567a2a36a68cd74d9cb437d09165d13f9
+oid sha256:df3be552cf2524f9ece2b6a286f0ce246d18d14d42f9b8c771a555e051bcee33
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.8498583569405099,
+  "epoch": 0.8627349987123358,
   "eval_steps": 50,
-  "global_step": 3300,
+  "global_step": 3350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5881,11 +5881,100 @@
       "eval_steps_per_second": 0.798,
       "num_input_tokens_seen": 34654480,
       "step": 3300
+    },
+    {
+      "epoch": 0.8511460211176926,
+      "grad_norm": 1.3186556509713656,
+      "learning_rate": 2.1329118524827662e-07,
+      "loss": 0.3195,
+      "num_input_tokens_seen": 34706600,
+      "step": 3305
+    },
+    {
+      "epoch": 0.8524336852948751,
+      "grad_norm": 5.6829477329970475,
+      "learning_rate": 1.9144427761286222e-07,
+      "loss": 0.2817,
+      "num_input_tokens_seen": 34759528,
+      "step": 3310
+    },
+    {
+      "epoch": 0.8537213494720577,
+      "grad_norm": 10.648820125115563,
+      "learning_rate": 1.7077534966650766e-07,
+      "loss": 0.3131,
+      "num_input_tokens_seen": 34811832,
+      "step": 3315
+    },
+    {
+      "epoch": 0.8550090136492403,
+      "grad_norm": 1.412998871252427,
+      "learning_rate": 1.51284890232406e-07,
+      "loss": 0.2926,
+      "num_input_tokens_seen": 34864696,
+      "step": 3320
+    },
+    {
+      "epoch": 0.8562966778264228,
+      "grad_norm": 2.3084307837646425,
+      "learning_rate": 1.3297336026280027e-07,
+      "loss": 0.2606,
+      "num_input_tokens_seen": 34917584,
+      "step": 3325
+    },
+    {
+      "epoch": 0.8575843420036054,
+      "grad_norm": 1.9582259889928806,
+      "learning_rate": 1.158411928280645e-07,
+      "loss": 0.3203,
+      "num_input_tokens_seen": 34969720,
+      "step": 3330
+    },
+    {
+      "epoch": 0.8588720061807881,
+      "grad_norm": 4.1936933517667825,
+      "learning_rate": 9.988879310649513e-08,
+      "loss": 0.3211,
+      "num_input_tokens_seen": 35021296,
+      "step": 3335
+    },
+    {
+      "epoch": 0.8601596703579707,
+      "grad_norm": 5.882661534247897,
+      "learning_rate": 8.511653837470212e-08,
+      "loss": 0.2923,
+      "num_input_tokens_seen": 35073120,
+      "step": 3340
+    },
+    {
+      "epoch": 0.8614473345351532,
+      "grad_norm": 6.708718647923996,
+      "learning_rate": 7.152477799867719e-08,
+      "loss": 0.289,
+      "num_input_tokens_seen": 35126296,
+      "step": 3345
+    },
+    {
+      "epoch": 0.8627349987123358,
+      "grad_norm": 2.5743366606144185,
+      "learning_rate": 5.911383342556143e-08,
+      "loss": 0.3253,
+      "num_input_tokens_seen": 35179104,
+      "step": 3350
+    },
+    {
+      "epoch": 0.8627349987123358,
+      "eval_loss": 0.4714648127555847,
+      "eval_runtime": 37.5951,
+      "eval_samples_per_second": 3.192,
+      "eval_steps_per_second": 0.798,
+      "num_input_tokens_seen": 35179104,
+      "step": 3350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 34654480,
+  "num_input_tokens_seen": 35179104,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5900,7 +5989,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2286412268568576.0,
+  "total_flos": 2321016547704832.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null