Training in progress, step 1050, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a3ab6d644ac3cb3a77587a21e5db04187d88cc24b3b2be9b5e4e433e0add233
+oid sha256:4499fca7257a53d1a8959e9e138764d24facbc55103cd1d2e30b092c058fc6d9
 size 29034840

last-checkpoint/global_step1050/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dd37b9bb3c7ca4a2e1ac547a76cd314772017b0eee2e58e2c86444e0dfd52ed
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64f584e3a146e5d4835f3ddce8eac0097d6c92edb4539be3c5891333747fa4a2
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5db9da879469a496cdd9059d07e41a815553c093a28b1e5f757b50e9736d5819
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49f55ceac858d890c28903f7fa52e3dd5e171674ec8b674e1c236daced759375
+size 43429616

last-checkpoint/global_step1050/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5740907eeaf461a960ab6da055d43d89f41f6417899be566427552f76b967abe
+size 637299

last-checkpoint/global_step1050/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28dcc960a14c72ef47fb9722c4b98cf1679b58db19513a1020156883879227c7
+size 637171

last-checkpoint/global_step1050/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:055280373e3009fe56853d9b9ef4c853cc09e485fa66a9bd20cba222d17c0bf5
+size 637171

last-checkpoint/global_step1050/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:179e95a89c200a847bd6de0c47ca8b3d721ccf69a889371f83c8cb19695431d6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1000
+global_step1050

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d73dfcc09cf3d6f08149535e03920234febc15f7e9a166987f3bc01ee871abf
+oid sha256:67db742b8aa1744a8224bf2a1f79d89caff63b15f78a455d92bb666df82183ea
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4396a64b6da4868d060d1e3c7c9ccb12c39d63bd0f7b146d2512400aff4c769c
+oid sha256:c0f9acb7e6f8bbfb305c3601c71eb6189af24942fab5f99046412c03bb10c3eb
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95877efc8fb5eb302819ee7effca4222569cdcfdebb9fa5d9846e68ed9e833fe
+oid sha256:713783338342f7486f6f186abd03c5963a0d22368f403efb2bf903ed083d2b64
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9fa4f23377f00fdde731da68a8690098617a1fdd912e03cdaa8bde87c493179
+oid sha256:83aaec0b5b7d8a2da4577075066cf434ce6e9feb9327edbea6677a2e51d76466
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bce7739c5bb5cf50e8f1c942e662e33e6aa589036d55e6fddd63bdf3171c1cae
+oid sha256:ccde9da0d23fd32800f01283cdb6c677def9ab43edb7232a9c4c4e9101a14cc0
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.0007357922149822116,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-1000",
-  "epoch": 0.5150656708730363,
+  "epoch": 0.5408189544166881,
   "eval_steps": 50,
-  "global_step": 1000,
+  "global_step": 1050,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1787,11 +1787,100 @@
       "eval_steps_per_second": 0.782,
       "num_input_tokens_seen": 9984000,
       "step": 1000
+    },
+    {
+      "epoch": 0.5176409992274015,
+      "grad_norm": 0.06909777020752475,
+      "learning_rate": 8.439714556870704e-05,
+      "loss": 0.0042,
+      "num_input_tokens_seen": 10033920,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5202163275817667,
+      "grad_norm": 0.15306455110846962,
+      "learning_rate": 8.422026448640124e-05,
+      "loss": 0.0419,
+      "num_input_tokens_seen": 10083840,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5227916559361319,
+      "grad_norm": 0.06864588320363613,
+      "learning_rate": 8.40425740897932e-05,
+      "loss": 0.0313,
+      "num_input_tokens_seen": 10133760,
+      "step": 1015
+    },
+    {
+      "epoch": 0.525366984290497,
+      "grad_norm": 0.041539288845711536,
+      "learning_rate": 8.386407858128706e-05,
+      "loss": 0.005,
+      "num_input_tokens_seen": 10183680,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5279423126448622,
+      "grad_norm": 3.866462743446183,
+      "learning_rate": 8.368478218232787e-05,
+      "loss": 0.0171,
+      "num_input_tokens_seen": 10233600,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5305176409992274,
+      "grad_norm": 0.27036285469370147,
+      "learning_rate": 8.350468913330192e-05,
+      "loss": 0.0019,
+      "num_input_tokens_seen": 10283520,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5330929693535926,
+      "grad_norm": 0.12025617159059553,
+      "learning_rate": 8.33238036934364e-05,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 10333440,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5356682977079578,
+      "grad_norm": 0.013500323699360073,
+      "learning_rate": 8.31421301406986e-05,
+      "loss": 0.0019,
+      "num_input_tokens_seen": 10383360,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5382436260623229,
+      "grad_norm": 0.10118623585384687,
+      "learning_rate": 8.29596727716949e-05,
+      "loss": 0.018,
+      "num_input_tokens_seen": 10433280,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5408189544166881,
+      "grad_norm": 0.23565575572231393,
+      "learning_rate": 8.277643590156894e-05,
+      "loss": 0.0007,
+      "num_input_tokens_seen": 10483200,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5408189544166881,
+      "eval_loss": 0.05677948147058487,
+      "eval_runtime": 19.0806,
+      "eval_samples_per_second": 3.145,
+      "eval_steps_per_second": 0.786,
+      "num_input_tokens_seen": 10483200,
+      "step": 1050
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 9984000,
+  "num_input_tokens_seen": 10483200,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1806,7 +1895,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 658744419811328.0,
+  "total_flos": 691684399382528.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null