Training in progress, step 400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eded4b34f6a51299f10b1a923e18a4a4f8fbfec56b877276cbf713599e577f0a
+oid sha256:8499cdded170833460860914d9976764a5d9563bba2689c4e8900b95bab5712a
 size 29034840

last-checkpoint/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ccbd5e79369eb0cb40a4cd80d2925323c3d4440ed4fd84c3fd5b3fb850db60c
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f96bf857531a2a8a932cb8430d8ea2ecefeec5bd1e4920cf514f3efabaedc69
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ef3b09d3ecff6f5923049ced04822e95598533c15c66484e3593ce2da4af7f4
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12a587d5c244da423186f73082962fac0478da902cd1ab891e76883d6cb2848e
+size 43429616

last-checkpoint/global_step400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5156b71863558eb940883c2a53abc6ea0ecf35171e1c6ef6fd48a17669031953
+size 637299

last-checkpoint/global_step400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c22d7604f5e9a963cc0beedf938177c8d8489af65df205ebac4c98a1b8ec451
+size 637171

last-checkpoint/global_step400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3741599f9c6a30d8f169f2bdf2dbef5196058896ed7f387c615c11e68675c1bf
+size 637171

last-checkpoint/global_step400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945fe57eb537c5ffd3d2e0d80ed5dfc750e49073d3d12e111bfb6c6bd93acce2
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step350
+global_step400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
+oid sha256:543ef05f530d40ee20b8d626b07a69b86597aca643e48897571062f973efe84f
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
+oid sha256:7a23f732e43838ce0398d2636885ac16badbb9bcbc04d1406069ba3027bc5ae0
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
+oid sha256:e10cce960e7068b051c05e35ed6160656be9091c63f13796ac2ed7e9c84e5a72
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
+oid sha256:b6f6049e212b1df5cefc5d834afcd8cc052c73f1457449e9fe8a38d514f54078
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff02966f5e9ff1944d3a06acaf420bd5bf127f85a91f6b1b2b1649b32fc1fd58
+oid sha256:32762cec872a9c6d771fb0f76b3f72991fda55aee1494a130c6c6b449c48a001
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.030908752232789993,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-300",
-  "epoch": 0.1802729848055627,
+  "best_metric": 0.024433813989162445,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-400",
+  "epoch": 0.20602626834921453,
   "eval_steps": 50,
-  "global_step": 350,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -630,11 +630,100 @@
       "eval_steps_per_second": 0.775,
       "num_input_tokens_seen": 3494400,
       "step": 350
+    },
+    {
+      "epoch": 0.1828483131599279,
+      "grad_norm": 0.3217035938609153,
+      "learning_rate": 9.91927536630402e-05,
+      "loss": 0.0267,
+      "num_input_tokens_seen": 3544320,
+      "step": 355
+    },
+    {
+      "epoch": 0.18542364151429308,
+      "grad_norm": 1.9146681345367045,
+      "learning_rate": 9.91486549841951e-05,
+      "loss": 0.0642,
+      "num_input_tokens_seen": 3594240,
+      "step": 360
+    },
+    {
+      "epoch": 0.18799896986865824,
+      "grad_norm": 0.053067921944857786,
+      "learning_rate": 9.91033939324107e-05,
+      "loss": 0.0302,
+      "num_input_tokens_seen": 3644160,
+      "step": 365
+    },
+    {
+      "epoch": 0.19057429822302344,
+      "grad_norm": 0.07277339857890938,
+      "learning_rate": 9.905697157811761e-05,
+      "loss": 0.0099,
+      "num_input_tokens_seen": 3694080,
+      "step": 370
+    },
+    {
+      "epoch": 0.19314962657738863,
+      "grad_norm": 4.117705571576552,
+      "learning_rate": 9.900938901921131e-05,
+      "loss": 0.0573,
+      "num_input_tokens_seen": 3744000,
+      "step": 375
+    },
+    {
+      "epoch": 0.1957249549317538,
+      "grad_norm": 2.4479374171708304,
+      "learning_rate": 9.896064738102635e-05,
+      "loss": 0.0302,
+      "num_input_tokens_seen": 3793920,
+      "step": 380
+    },
+    {
+      "epoch": 0.19830028328611898,
+      "grad_norm": 2.1751552571638153,
+      "learning_rate": 9.891074781630966e-05,
+      "loss": 0.0374,
+      "num_input_tokens_seen": 3843840,
+      "step": 385
+    },
+    {
+      "epoch": 0.20087561164048418,
+      "grad_norm": 1.4138134932494708,
+      "learning_rate": 9.885969150519331e-05,
+      "loss": 0.0227,
+      "num_input_tokens_seen": 3893760,
+      "step": 390
+    },
+    {
+      "epoch": 0.20345093999484934,
+      "grad_norm": 0.3312762825665537,
+      "learning_rate": 9.88074796551666e-05,
+      "loss": 0.0104,
+      "num_input_tokens_seen": 3943680,
+      "step": 395
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "grad_norm": 0.06774783590394957,
+      "learning_rate": 9.875411350104744e-05,
+      "loss": 0.0146,
+      "num_input_tokens_seen": 3993600,
+      "step": 400
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "eval_loss": 0.024433813989162445,
+      "eval_runtime": 18.8143,
+      "eval_samples_per_second": 3.189,
+      "eval_steps_per_second": 0.797,
+      "num_input_tokens_seen": 3993600,
+      "step": 400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 3494400,
+  "num_input_tokens_seen": 3993600,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -649,7 +738,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 230524685385728.0,
+  "total_flos": 263464664956928.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null