Training in progress, step 1400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5539aab9f718348f04ec41613d234e7bd892b381194398a0eadf2adcfde0ddb
+oid sha256:e0eaf3c3fbb7e51224bc7a40e841d98120e282fdcc24346a2251f02af1f31442
 size 29034840

last-checkpoint/global_step1400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e342082d4542a64d191aa926861afcf05b77eaceea1f30ac3fc9550d2dd105
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502251206a24a0a024f81c30de85799cd1c6b914f9e685359411506498a96167
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e212f331b2681aab216c5852744fb0215a5ced4ea0ede628b65d8da873c9ed62
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b3b2f0c4e540817d85bff704f6250658a8e37e6d5762cb79853b83c1eb15c0
+size 43429616

last-checkpoint/global_step1400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1906adf1b95058095138f9b57795e6e0826a899c9c7e5a525c8eae36ecaa15f4
+size 637299

last-checkpoint/global_step1400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a0bb562939d112dd834c43097bc2d1ef141d083d45e4138023cb41209e8c874
+size 637171

last-checkpoint/global_step1400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eb2bc0e5fa21a633543ff99a5a9ea2924b55031a219c0558ebd60e5ccef7d0e
+size 637171

last-checkpoint/global_step1400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:336e83dddf7ea2bb6e2ee90e33a484acff5b821f73e635c4aaaa44a24086519e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1350
+global_step1400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
+oid sha256:fe66a68e61de2221b30fd9749bc68b45a1474bb2cc95901bca9557ac87909355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
+oid sha256:0cd4f3162e46c3bb0f1fc4d3c52c7c33e60f56764458e0c8a73c3810b0a25f8c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
+oid sha256:185cc99aaa81b1b49b3ddc74aa6f97aa3036330983a7b69d52bd191057f9a5d5
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
+oid sha256:0e37403c30cb4309e54e5defdde1906486716fc859274035d44aaac5d48a97ba
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5473741dbc1be4510ceec76e5e3cd10aedf1c7667da998c551f8978b6c6c33d3
+oid sha256:9eb4a31de57d35a9d36a648c17d1ff1a00be01f744b83cf70f63cae24d418555
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2966395914554596,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1200",
-  "epoch": 0.3988183161004431,
+  "epoch": 0.413589364844904,
   "eval_steps": 50,
-  "global_step": 1350,
+  "global_step": 1400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2410,11 +2410,100 @@
       "eval_steps_per_second": 0.777,
       "num_input_tokens_seen": 14014736,
       "step": 1350
+    },
+    {
+      "epoch": 0.40029542097488924,
+      "grad_norm": 2.292705535408954,
+      "learning_rate": 9.396695542455704e-05,
+      "loss": 0.4115,
+      "num_input_tokens_seen": 14066880,
+      "step": 1355
+    },
+    {
+      "epoch": 0.4017725258493353,
+      "grad_norm": 10.68072230240614,
+      "learning_rate": 9.390866796969577e-05,
+      "loss": 0.365,
+      "num_input_tokens_seen": 14118320,
+      "step": 1360
+    },
+    {
+      "epoch": 0.4032496307237814,
+      "grad_norm": 28.45565288311722,
+      "learning_rate": 9.385011855616177e-05,
+      "loss": 0.3904,
+      "num_input_tokens_seen": 14169208,
+      "step": 1365
+    },
+    {
+      "epoch": 0.40472673559822747,
+      "grad_norm": 9.32794663574214,
+      "learning_rate": 9.379130753326021e-05,
+      "loss": 0.5425,
+      "num_input_tokens_seen": 14220632,
+      "step": 1370
+    },
+    {
+      "epoch": 0.40620384047267355,
+      "grad_norm": 4.737143544435888,
+      "learning_rate": 9.373223525185709e-05,
+      "loss": 0.3985,
+      "num_input_tokens_seen": 14272640,
+      "step": 1375
+    },
+    {
+      "epoch": 0.4076809453471196,
+      "grad_norm": 17.480173613134482,
+      "learning_rate": 9.367290206437702e-05,
+      "loss": 0.3528,
+      "num_input_tokens_seen": 14324960,
+      "step": 1380
+    },
+    {
+      "epoch": 0.4091580502215657,
+      "grad_norm": 4.40598964753602,
+      "learning_rate": 9.361330832480124e-05,
+      "loss": 0.3687,
+      "num_input_tokens_seen": 14376792,
+      "step": 1385
+    },
+    {
+      "epoch": 0.41063515509601184,
+      "grad_norm": 15.961709998187562,
+      "learning_rate": 9.355345438866538e-05,
+      "loss": 0.3552,
+      "num_input_tokens_seen": 14428192,
+      "step": 1390
+    },
+    {
+      "epoch": 0.4121122599704579,
+      "grad_norm": 4.033485652398453,
+      "learning_rate": 9.349334061305743e-05,
+      "loss": 0.3194,
+      "num_input_tokens_seen": 14480568,
+      "step": 1395
+    },
+    {
+      "epoch": 0.413589364844904,
+      "grad_norm": 9.187315388235644,
+      "learning_rate": 9.343296735661557e-05,
+      "loss": 0.388,
+      "num_input_tokens_seen": 14532288,
+      "step": 1400
+    },
+    {
+      "epoch": 0.413589364844904,
+      "eval_loss": 0.38656601309776306,
+      "eval_runtime": 19.1495,
+      "eval_samples_per_second": 3.133,
+      "eval_steps_per_second": 0.783,
+      "num_input_tokens_seen": 14532288,
+      "step": 1400
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 14014736,
+  "num_input_tokens_seen": 14532288,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2429,7 +2518,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 924522107633664.0,
+  "total_flos": 958658687795200.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null