Training in progress, step 500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6563ebf68bb7cb8cee897a19128b74ef4208c6802d855d4f6562f7c26273f70
+oid sha256:6e25d499b4bbad9ee63daa7c21a747a882c092990720dbb0eeef7ae14ead09fd
 size 29034840

last-checkpoint/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8faa2348d84a3d617d8f16b6f97ef74fc9be90c1c0e88727d81375b2cdc1deae
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6847b112017e988b54fcd5f01c924f7701049acfed4a4cbac82789cede84343
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77be84932aeabc68bdb651479743b5e58e8a7f574c5184ffd59952dbeaf28da7
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:483e2fa99f325bd234c1ccd4c7ac7d64a95c47b5b8f22f8b71549fb8d3196661
+size 43429616

last-checkpoint/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:982b6b8112b1a68d2e38cae94a225f18bd9ea392e1654dbc7cc130776bff950b
+size 637299

last-checkpoint/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854e2832a5dae385b495dad0465c766c553cf722b470925f5286a7230114f652
+size 637171

last-checkpoint/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da17084af120106739c69dcfe393381f051b6adbcd329aae7c5a39f5b06e3345
+size 637171

last-checkpoint/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7066b428556bc0b3f73235845964ecb2a136bcb07c44570c346b66a0f2f5992
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step450
+global_step500

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbc6521b0b64cb12d818506108fcf257a4089ca8a9b1e453776ed3e032e7176
+oid sha256:bdd1f02cb20d3f4f7e0dd26fea62af57e5e71316163f926a28ed6cf89a9f3777
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b13e3da1b0679cab1bab94f893e385a9a224d3335b5a6f62602f33c2be88d03
+oid sha256:fc6d54ba2aa85e2f895439a1b787ec947b848a1c34ea5a3a28821572bf2b9fec
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a24f0e0f117b5a8236e0d12594c0c358f41ef00068d4460002e95ad1cc3cb1c
+oid sha256:8b6927d26551cddd8e35b34b43e79bd58f8b6027b6a481bb6a563a3652addeb4
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e46e4eab6c4a25d84ad36ddf1357401788adeeb6388c03cefa35a63b52ee7610
+oid sha256:f8295b1be8e66b4b30cb905dc48cfc717c027e427937b8142d00ae9de8106c6a
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:192829f095e6c906a86011515c9b1d243ad2d8f891793c9e500d2d433b84fc78
+oid sha256:ab7567aeded1bd7ef9f3ba115e57865dd25bb569f9711f33170eb2a51540c216
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.006690301466733217,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-450",
-  "epoch": 0.23177955189286634,
+  "epoch": 0.25753283543651817,
   "eval_steps": 50,
-  "global_step": 450,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -808,11 +808,100 @@
       "eval_steps_per_second": 0.799,
       "num_input_tokens_seen": 4492800,
       "step": 450
+    },
+    {
+      "epoch": 0.23435488024723153,
+      "grad_norm": 2.8137229294991064,
+      "learning_rate": 9.809128215864097e-05,
+      "loss": 0.0596,
+      "num_input_tokens_seen": 4542720,
+      "step": 455
+    },
+    {
+      "epoch": 0.2369302086015967,
+      "grad_norm": 0.045182426772616975,
+      "learning_rate": 9.802417057704931e-05,
+      "loss": 0.0238,
+      "num_input_tokens_seen": 4592640,
+      "step": 460
+    },
+    {
+      "epoch": 0.23950553695596188,
+      "grad_norm": 2.1659990503080424,
+      "learning_rate": 9.795592321674045e-05,
+      "loss": 0.0331,
+      "num_input_tokens_seen": 4642560,
+      "step": 465
+    },
+    {
+      "epoch": 0.24208086531032708,
+      "grad_norm": 0.3208179936186945,
+      "learning_rate": 9.788654169177453e-05,
+      "loss": 0.0177,
+      "num_input_tokens_seen": 4692480,
+      "step": 470
+    },
+    {
+      "epoch": 0.24465619366469224,
+      "grad_norm": 1.4150868022870595,
+      "learning_rate": 9.781602764303487e-05,
+      "loss": 0.028,
+      "num_input_tokens_seen": 4742400,
+      "step": 475
+    },
+    {
+      "epoch": 0.24723152201905743,
+      "grad_norm": 0.21015675310797993,
+      "learning_rate": 9.774438273818911e-05,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 4792320,
+      "step": 480
+    },
+    {
+      "epoch": 0.24980685037342262,
+      "grad_norm": 2.6788940269709287,
+      "learning_rate": 9.767160867164979e-05,
+      "loss": 0.0242,
+      "num_input_tokens_seen": 4842240,
+      "step": 485
+    },
+    {
+      "epoch": 0.2523821787277878,
+      "grad_norm": 0.03933517764434036,
+      "learning_rate": 9.759770716453436e-05,
+      "loss": 0.0286,
+      "num_input_tokens_seen": 4892160,
+      "step": 490
+    },
+    {
+      "epoch": 0.254957507082153,
+      "grad_norm": 2.640299598300008,
+      "learning_rate": 9.752267996462434e-05,
+      "loss": 0.0394,
+      "num_input_tokens_seen": 4942080,
+      "step": 495
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "grad_norm": 0.03143215150024672,
+      "learning_rate": 9.744652884632406e-05,
+      "loss": 0.0122,
+      "num_input_tokens_seen": 4992000,
+      "step": 500
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "eval_loss": 0.007972972467541695,
+      "eval_runtime": 18.8509,
+      "eval_samples_per_second": 3.183,
+      "eval_steps_per_second": 0.796,
+      "num_input_tokens_seen": 4992000,
+      "step": 500
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 4492800,
+  "num_input_tokens_seen": 4992000,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -827,7 +916,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 296404644528128.0,
+  "total_flos": 329344624099328.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null