Training in progress, step 400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:886e83225731329ec8fecf2885b7ea9c6656cc91593fafb84eb477c2dda182a4
+oid sha256:5682b2eafe76e843b3c252d9efdff6d8e222ea2c09fcb985ed44476c6c49d3c0
 size 29034840

last-checkpoint/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a45cfdc87543c37602b2850a2daa8381c1c423e6271447006c77a8c50d1815e8
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ca19b4bd5d3c5fb71bcb136818805c6db46f6916dd983d744b9d1ae27ef437
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7d688deb0c4aae8479bc33e6f59c5ac798a1ba5ce1a590218ef0defa660653b
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383a4effbd1f408cef05361624a56f704544abceccb850128888bbc09af0004d
+size 43429616

last-checkpoint/global_step400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5156b71863558eb940883c2a53abc6ea0ecf35171e1c6ef6fd48a17669031953
+size 637299

last-checkpoint/global_step400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c22d7604f5e9a963cc0beedf938177c8d8489af65df205ebac4c98a1b8ec451
+size 637171

last-checkpoint/global_step400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3741599f9c6a30d8f169f2bdf2dbef5196058896ed7f387c615c11e68675c1bf
+size 637171

last-checkpoint/global_step400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945fe57eb537c5ffd3d2e0d80ed5dfc750e49073d3d12e111bfb6c6bd93acce2
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step350
+global_step400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
+oid sha256:543ef05f530d40ee20b8d626b07a69b86597aca643e48897571062f973efe84f
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
+oid sha256:7a23f732e43838ce0398d2636885ac16badbb9bcbc04d1406069ba3027bc5ae0
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
+oid sha256:e10cce960e7068b051c05e35ed6160656be9091c63f13796ac2ed7e9c84e5a72
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
+oid sha256:b6f6049e212b1df5cefc5d834afcd8cc052c73f1457449e9fe8a38d514f54078
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03940fe8f12e50ac0a3c905dfe63b68f75ffd42f00104f9fc8d680b7aa48b61a
+oid sha256:4fa1113b4807f273646ff4a5c640f26a2f072e0ac3eba31e530c029f936b8156
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7663387656211853,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-350",
-  "epoch": 0.103397341211226,
+  "best_metric": 0.7038857936859131,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-400",
+  "epoch": 0.11816838995568685,
   "eval_steps": 50,
-  "global_step": 350,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -630,11 +630,100 @@
       "eval_steps_per_second": 0.781,
       "num_input_tokens_seen": 3625752,
       "step": 350
+    },
+    {
+      "epoch": 0.10487444608567208,
+      "grad_norm": 21.440776172892136,
+      "learning_rate": 9.999847271358347e-05,
+      "loss": 0.7222,
+      "num_input_tokens_seen": 3676984,
+      "step": 355
+    },
+    {
+      "epoch": 0.10635155096011817,
+      "grad_norm": 3.9046938348237252,
+      "learning_rate": 9.99973690201885e-05,
+      "loss": 0.6639,
+      "num_input_tokens_seen": 3729168,
+      "step": 360
+    },
+    {
+      "epoch": 0.10782865583456426,
+      "grad_norm": 5.783088469074539,
+      "learning_rate": 9.999596704299139e-05,
+      "loss": 0.6501,
+      "num_input_tokens_seen": 3780672,
+      "step": 365
+    },
+    {
+      "epoch": 0.10930576070901034,
+      "grad_norm": 4.421078404189889,
+      "learning_rate": 9.999426679035628e-05,
+      "loss": 0.6871,
+      "num_input_tokens_seen": 3832328,
+      "step": 370
+    },
+    {
+      "epoch": 0.11078286558345643,
+      "grad_norm": 6.348668054760613,
+      "learning_rate": 9.99922682724269e-05,
+      "loss": 0.6621,
+      "num_input_tokens_seen": 3883112,
+      "step": 375
+    },
+    {
+      "epoch": 0.11225997045790251,
+      "grad_norm": 3.7958066942788573,
+      "learning_rate": 9.998997150112635e-05,
+      "loss": 0.7156,
+      "num_input_tokens_seen": 3934976,
+      "step": 380
+    },
+    {
+      "epoch": 0.1137370753323486,
+      "grad_norm": 2.694693287446712,
+      "learning_rate": 9.998737649015718e-05,
+      "loss": 0.6662,
+      "num_input_tokens_seen": 3986192,
+      "step": 385
+    },
+    {
+      "epoch": 0.11521418020679468,
+      "grad_norm": 6.428397441401454,
+      "learning_rate": 9.998448325500118e-05,
+      "loss": 0.682,
+      "num_input_tokens_seen": 4037760,
+      "step": 390
+    },
+    {
+      "epoch": 0.11669128508124077,
+      "grad_norm": 6.165388347571309,
+      "learning_rate": 9.998129181291936e-05,
+      "loss": 0.6137,
+      "num_input_tokens_seen": 4090872,
+      "step": 395
+    },
+    {
+      "epoch": 0.11816838995568685,
+      "grad_norm": 4.354814876263017,
+      "learning_rate": 9.997780218295185e-05,
+      "loss": 0.6739,
+      "num_input_tokens_seen": 4142592,
+      "step": 400
+    },
+    {
+      "epoch": 0.11816838995568685,
+      "eval_loss": 0.7038857936859131,
+      "eval_runtime": 19.0624,
+      "eval_samples_per_second": 3.148,
+      "eval_steps_per_second": 0.787,
+      "num_input_tokens_seen": 4142592,
+      "step": 400
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 3625752,
+  "num_input_tokens_seen": 4142592,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -649,7 +738,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 239163058946048.0,
+  "total_flos": 273245789749248.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null