Training in progress, step 2950, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f649bc97ca842140ab28c612e0e561ebbde8139ce799f23e401fad5ec0bf673a
+oid sha256:3b7a40fbb5d3ef8c199f94ef197e14510d714bce74ec2a360d2feaf74aeb1aee
 size 29034840

last-checkpoint/global_step2950/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9212095d1b2ad6dede71e3497e07f7008f864836ee652f7d721937f55416294
+size 43429616

last-checkpoint/global_step2950/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1e820af5f58539147abfb7908d919cd87d04b274cdf0246b111d819a373d5df
+size 43429616

last-checkpoint/global_step2950/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29fa9402ebb7e280c65f2f1cbd5d0e9207a9da3e0d25f466996ebe2889a99a3d
+size 43429616

last-checkpoint/global_step2950/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:602e6fda95ec4917ebd699b723c0a2bce3bd1b7ded81cd3fcb8bcb48477b055d
+size 43429616

last-checkpoint/global_step2950/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512575152585707305753bb11f9e58a6063a289cf07716493a20893fbd1142b9
+size 637299

last-checkpoint/global_step2950/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd557eb4b9edd0a1910792057338dca17083fa9a8f40f41e28c2c90ec22a93ae
+size 637171

last-checkpoint/global_step2950/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbd7c1168f13cf3fc959af479e28a5a467ab4dd67204cffc114ba3660c0517c0
+size 637171

last-checkpoint/global_step2950/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06808762d99a725be346c2151fb638282bfb9e97c3c8bcd58aaef62a9f5b3815
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2900
+global_step2950

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce92cea831a04716b4b472f1dad1cc986b2021dee9aac057217f5d455b27ec42
+oid sha256:bfc5d4e344535f1dd0ff5275633ec3d55bb6249e432442365ff24445d82ec35c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cddb73bbdf0f6f6a2c3182d70f7ad5d587353b164c08dd4f383b940d6b61e4e
+oid sha256:8a898928042c09dc123c1025557279997043b7f607bc91ee2ff2d4b4d2b9f1ba
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b24b508e466beb446d37377d2a04757d3bc2b4230de3ac56b25a65d7753a74c1
+oid sha256:6923d07d979aa78d66765208f598662fd5092b5227cd87920feedfb923fa375f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c6a18a7de8b25b21673ba2ff7efbaaae00ec8c453c7975b467c1df87b87022
+oid sha256:e9ba35b9b3c512fbe857d909557329ba47dbefe5f521014123c05901c32edb6d
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6207ec0a6d3415bd88090ab549215521abec8051b511fdc1a1cfc0484ab44197
+oid sha256:40ec7601b82ddf00dad2cf9910c40a0b3836b366525ddd347dc523d4b95c6345
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.1869634985923767,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-2350",
-  "epoch": 0.8567208271787297,
+  "epoch": 0.8714918759231906,
   "eval_steps": 50,
-  "global_step": 2900,
+  "global_step": 2950,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5169,11 +5169,100 @@
       "eval_steps_per_second": 0.784,
       "num_input_tokens_seen": 30118072,
       "step": 2900
+    },
+    {
+      "epoch": 0.8581979320531757,
+      "grad_norm": 1.2779653791980783,
+      "learning_rate": 6.559944544378072e-05,
+      "loss": 0.2241,
+      "num_input_tokens_seen": 30170248,
+      "step": 2905
+    },
+    {
+      "epoch": 0.8596750369276218,
+      "grad_norm": 1.6407050078852088,
+      "learning_rate": 6.548336798918411e-05,
+      "loss": 0.2298,
+      "num_input_tokens_seen": 30222016,
+      "step": 2910
+    },
+    {
+      "epoch": 0.8611521418020679,
+      "grad_norm": 1.5078723595666699,
+      "learning_rate": 6.536719816096935e-05,
+      "loss": 0.2396,
+      "num_input_tokens_seen": 30273312,
+      "step": 2915
+    },
+    {
+      "epoch": 0.8626292466765141,
+      "grad_norm": 1.852309531147588,
+      "learning_rate": 6.52509366522045e-05,
+      "loss": 0.2324,
+      "num_input_tokens_seen": 30324328,
+      "step": 2920
+    },
+    {
+      "epoch": 0.8641063515509602,
+      "grad_norm": 1.3789560520965807,
+      "learning_rate": 6.513458415650452e-05,
+      "loss": 0.2263,
+      "num_input_tokens_seen": 30376488,
+      "step": 2925
+    },
+    {
+      "epoch": 0.8655834564254062,
+      "grad_norm": 1.904793400524472,
+      "learning_rate": 6.501814136802725e-05,
+      "loss": 0.1734,
+      "num_input_tokens_seen": 30429504,
+      "step": 2930
+    },
+    {
+      "epoch": 0.8670605612998523,
+      "grad_norm": 1.15273615308065,
+      "learning_rate": 6.490160898146918e-05,
+      "loss": 0.2235,
+      "num_input_tokens_seen": 30480400,
+      "step": 2935
+    },
+    {
+      "epoch": 0.8685376661742984,
+      "grad_norm": 1.4070108528869274,
+      "learning_rate": 6.47849876920614e-05,
+      "loss": 0.2297,
+      "num_input_tokens_seen": 30531912,
+      "step": 2940
+    },
+    {
+      "epoch": 0.8700147710487445,
+      "grad_norm": 1.182373794273276,
+      "learning_rate": 6.46682781955653e-05,
+      "loss": 0.1764,
+      "num_input_tokens_seen": 30584688,
+      "step": 2945
+    },
+    {
+      "epoch": 0.8714918759231906,
+      "grad_norm": 17.302996120392294,
+      "learning_rate": 6.455148118826859e-05,
+      "loss": 0.1692,
+      "num_input_tokens_seen": 30637448,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8714918759231906,
+      "eval_loss": 0.20344533026218414,
+      "eval_runtime": 18.8463,
+      "eval_samples_per_second": 3.184,
+      "eval_steps_per_second": 0.796,
+      "num_input_tokens_seen": 30637448,
+      "step": 2950
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 30118072,
+  "num_input_tokens_seen": 30637448,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5188,7 +5277,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1986962888589312.0,
+  "total_flos": 2021200008314880.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null