Training in progress, step 450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8499cdded170833460860914d9976764a5d9563bba2689c4e8900b95bab5712a
+oid sha256:d6563ebf68bb7cb8cee897a19128b74ef4208c6802d855d4f6562f7c26273f70
 size 29034840

last-checkpoint/global_step450/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b55f4575188d91c76c82c6604d74d9072cb87f5a25c83a1422d18e9a3981296
+size 43429616

last-checkpoint/global_step450/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfb4cf2ed3c3207ed357bfcf90b6cf77a8d8516195639f4f2f43f533a4d5baaf
+size 43429616

last-checkpoint/global_step450/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6959253acb0958e24e4d551f262ea041f55f82cafaf798926ce915ed93b31e
+size 43429616

last-checkpoint/global_step450/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e2f83a95f144d26704aec59a04dedb11b8b66af0f86fe0761af35f1ea688f73
+size 43429616

last-checkpoint/global_step450/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2c43b6581640ec9140a8e671944ff69d96a0284c4069ddbe5a73547523d050e
+size 637299

last-checkpoint/global_step450/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e7b078653231149a903bc00280c61fc902a6ebe6078af02817b5efd13e9ef2b
+size 637171

last-checkpoint/global_step450/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3014bb4485c273ec85ee6adab304455dcd75b99ca668f9c10e04f1ebc443fec8
+size 637171

last-checkpoint/global_step450/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23e7a4eb8e027c3a1e9ed72ab3097610a2c4faa00790d4b003da7cdb06018e2
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step400
+global_step450

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:543ef05f530d40ee20b8d626b07a69b86597aca643e48897571062f973efe84f
+oid sha256:7dbc6521b0b64cb12d818506108fcf257a4089ca8a9b1e453776ed3e032e7176
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a23f732e43838ce0398d2636885ac16badbb9bcbc04d1406069ba3027bc5ae0
+oid sha256:2b13e3da1b0679cab1bab94f893e385a9a224d3335b5a6f62602f33c2be88d03
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e10cce960e7068b051c05e35ed6160656be9091c63f13796ac2ed7e9c84e5a72
+oid sha256:6a24f0e0f117b5a8236e0d12594c0c358f41ef00068d4460002e95ad1cc3cb1c
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6f6049e212b1df5cefc5d834afcd8cc052c73f1457449e9fe8a38d514f54078
+oid sha256:e46e4eab6c4a25d84ad36ddf1357401788adeeb6388c03cefa35a63b52ee7610
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32762cec872a9c6d771fb0f76b3f72991fda55aee1494a130c6c6b449c48a001
+oid sha256:192829f095e6c906a86011515c9b1d243ad2d8f891793c9e500d2d433b84fc78
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.024433813989162445,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-400",
-  "epoch": 0.20602626834921453,
+  "best_metric": 0.006690301466733217,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-450",
+  "epoch": 0.23177955189286634,
   "eval_steps": 50,
-  "global_step": 400,
+  "global_step": 450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -719,11 +719,100 @@
       "eval_steps_per_second": 0.797,
       "num_input_tokens_seen": 3993600,
       "step": 400
+    },
+    {
+      "epoch": 0.2086015967035797,
+      "grad_norm": 0.06416132017781931,
+      "learning_rate": 9.86995943049533e-05,
+      "loss": 0.0074,
+      "num_input_tokens_seen": 4043520,
+      "step": 405
+    },
+    {
+      "epoch": 0.2111769250579449,
+      "grad_norm": 0.04809215529224866,
+      "learning_rate": 9.864392335627117e-05,
+      "loss": 0.0375,
+      "num_input_tokens_seen": 4093440,
+      "step": 410
+    },
+    {
+      "epoch": 0.21375225341231008,
+      "grad_norm": 0.40943321947836153,
+      "learning_rate": 9.858710197162721e-05,
+      "loss": 0.0217,
+      "num_input_tokens_seen": 4143360,
+      "step": 415
+    },
+    {
+      "epoch": 0.21632758176667524,
+      "grad_norm": 0.08069011480339518,
+      "learning_rate": 9.852913149485556e-05,
+      "loss": 0.016,
+      "num_input_tokens_seen": 4193280,
+      "step": 420
+    },
+    {
+      "epoch": 0.21890291012104043,
+      "grad_norm": 1.806585526467194,
+      "learning_rate": 9.847001329696653e-05,
+      "loss": 0.0832,
+      "num_input_tokens_seen": 4243200,
+      "step": 425
+    },
+    {
+      "epoch": 0.22147823847540563,
+      "grad_norm": 3.122914329597603,
+      "learning_rate": 9.840974877611422e-05,
+      "loss": 0.0444,
+      "num_input_tokens_seen": 4293120,
+      "step": 430
+    },
+    {
+      "epoch": 0.2240535668297708,
+      "grad_norm": 1.0101510644418257,
+      "learning_rate": 9.834833935756344e-05,
+      "loss": 0.0465,
+      "num_input_tokens_seen": 4343040,
+      "step": 435
+    },
+    {
+      "epoch": 0.22662889518413598,
+      "grad_norm": 1.7770164472545809,
+      "learning_rate": 9.828578649365601e-05,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 4392960,
+      "step": 440
+    },
+    {
+      "epoch": 0.22920422353850115,
+      "grad_norm": 0.9798524063329249,
+      "learning_rate": 9.822209166377635e-05,
+      "loss": 0.02,
+      "num_input_tokens_seen": 4442880,
+      "step": 445
+    },
+    {
+      "epoch": 0.23177955189286634,
+      "grad_norm": 0.22305429096692395,
+      "learning_rate": 9.815725637431662e-05,
+      "loss": 0.0157,
+      "num_input_tokens_seen": 4492800,
+      "step": 450
+    },
+    {
+      "epoch": 0.23177955189286634,
+      "eval_loss": 0.006690301466733217,
+      "eval_runtime": 18.7725,
+      "eval_samples_per_second": 3.196,
+      "eval_steps_per_second": 0.799,
+      "num_input_tokens_seen": 4492800,
+      "step": 450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 3993600,
+  "num_input_tokens_seen": 4492800,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -738,7 +827,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 263464664956928.0,
+  "total_flos": 296404644528128.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null