Training in progress, step 1250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60c80d389637b5f5cae317955ea9368ed58131fb66260b7e896c23b167e9cf7b
+oid sha256:49efabf09713516088de3465883ff1d8943c4d5b7de6e932ffa50531df3a26bb
 size 29034840

last-checkpoint/global_step1250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2202ee0464fb513426f1e8722944a7fbaa2f8768d30031c50846421852b1d429
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:641d1317aa3075660cd3806c498ba14bd369852ef671a940ce6ce172f318abd5
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48595f3be6b89066149925a745daeb0e2d5a3235c77b8ee70a7e133101faf672
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29308d99f6727670fec02c2f18a390aa460c63a33d10b3341aaf14ca0cb533d2
+size 43429616

last-checkpoint/global_step1250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec0f2cde302545248de950463724f6d392acd94dddf06b2baba0a3d8f91b0e0
+size 637299

last-checkpoint/global_step1250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e30a5b6d29417f61fce028606e2a43dda69feda25ab401bf98a939ad68d0a488
+size 637171

last-checkpoint/global_step1250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55d5732bdcd7f622a6122c509cb64b5a03edeaa513ee805d437e1045732cd1d7
+size 637171

last-checkpoint/global_step1250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac58ba9b33d13c7004129934ac2b499707908f73809a09276a3945da3fd62d36
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1200
+global_step1250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a209a0c0025f9ce8e2beeba50c1f0828d5c34a2482310fcd0bf5fc24c2c67be2
+oid sha256:15bbef9a4c878a05b4f189e9f77701a153dc9faf093499714094cb36ac0ca030
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a67fb929b8c51f9b1c6ff9f11366e57e55128a1d36df85a9d37a008b49017a75
+oid sha256:7a7122760cfda3f3e13eac4f7e56d09b0fd3beac4bd49b2f0e58da9519469ff9
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9ef3b0c0978d0b611f4257c939f1c2c6f07e6227bfea6675532d285b0b64a7
+oid sha256:71edf044d4ea96e62ffecab515d42b58ae56cd64abd0092a897468e0e0d7c10f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0183d14c8ed52ee533139532e9bcf7bc34ec297a064845b35741cb501d92675f
+oid sha256:ef5ca30351deca9d83675a183b2186ef346093f56e0d5a6dacdf62ef921f12c2
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17a5556c8233d79e1be6279770d5e53a5fee5448790e76942f098adafb906464
+oid sha256:88c4e20232245763c3fa9c7256563263254aa7f4fcf1acb3e46226c0466eab5e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2966395914554596,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1200",
-  "epoch": 0.35450516986706054,
+  "epoch": 0.36927621861152143,
   "eval_steps": 50,
-  "global_step": 1200,
+  "global_step": 1250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2143,11 +2143,100 @@
       "eval_steps_per_second": 0.768,
       "num_input_tokens_seen": 12456040,
       "step": 1200
+    },
+    {
+      "epoch": 0.3559822747415066,
+      "grad_norm": 5.5028656713393245,
+      "learning_rate": 9.559210016722184e-05,
+      "loss": 0.3717,
+      "num_input_tokens_seen": 12507640,
+      "step": 1205
+    },
+    {
+      "epoch": 0.35745937961595275,
+      "grad_norm": 13.214008089689216,
+      "learning_rate": 9.554182607505484e-05,
+      "loss": 0.541,
+      "num_input_tokens_seen": 12559400,
+      "step": 1210
+    },
+    {
+      "epoch": 0.35893648449039883,
+      "grad_norm": 6.269664608708862,
+      "learning_rate": 9.54912802808102e-05,
+      "loss": 0.3965,
+      "num_input_tokens_seen": 12610992,
+      "step": 1215
+    },
+    {
+      "epoch": 0.3604135893648449,
+      "grad_norm": 21.247512275128738,
+      "learning_rate": 9.544046308604364e-05,
+      "loss": 0.4834,
+      "num_input_tokens_seen": 12662688,
+      "step": 1220
+    },
+    {
+      "epoch": 0.361890694239291,
+      "grad_norm": 4.14272563629135,
+      "learning_rate": 9.538937479393001e-05,
+      "loss": 0.4538,
+      "num_input_tokens_seen": 12713600,
+      "step": 1225
+    },
+    {
+      "epoch": 0.36336779911373707,
+      "grad_norm": 12.86150407455535,
+      "learning_rate": 9.533801570926157e-05,
+      "loss": 0.4226,
+      "num_input_tokens_seen": 12766360,
+      "step": 1230
+    },
+    {
+      "epoch": 0.36484490398818314,
+      "grad_norm": 4.36264988758363,
+      "learning_rate": 9.52863861384461e-05,
+      "loss": 0.4315,
+      "num_input_tokens_seen": 12817248,
+      "step": 1235
+    },
+    {
+      "epoch": 0.3663220088626292,
+      "grad_norm": 1.6622545996067835,
+      "learning_rate": 9.523448638950508e-05,
+      "loss": 0.3567,
+      "num_input_tokens_seen": 12868496,
+      "step": 1240
+    },
+    {
+      "epoch": 0.36779911373707536,
+      "grad_norm": 5.5241376090939065,
+      "learning_rate": 9.518231677207192e-05,
+      "loss": 0.3431,
+      "num_input_tokens_seen": 12920168,
+      "step": 1245
+    },
+    {
+      "epoch": 0.36927621861152143,
+      "grad_norm": 5.717434283790562,
+      "learning_rate": 9.512987759739003e-05,
+      "loss": 0.335,
+      "num_input_tokens_seen": 12972696,
+      "step": 1250
+    },
+    {
+      "epoch": 0.36927621861152143,
+      "eval_loss": 0.31846168637275696,
+      "eval_runtime": 18.9319,
+      "eval_samples_per_second": 3.169,
+      "eval_steps_per_second": 0.792,
+      "num_input_tokens_seen": 12972696,
+      "step": 1250
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 12456040,
+  "num_input_tokens_seen": 12972696,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2162,7 +2251,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 821733054480384.0,
+  "total_flos": 855825716609024.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null