Training in progress, step 350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b81846ac02e033886bed1edec96d0da70d2a41c811163f600e741ff029d3c0d0
+oid sha256:886e83225731329ec8fecf2885b7ea9c6656cc91593fafb84eb477c2dda182a4
 size 29034840

last-checkpoint/global_step350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79b6b307b01d3efde2c810c27deac3da1f68f59b2e7bc06c678e8049bbb464d7
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad2193f4466e52cf222439411745f1ef9f7dad08da561dad4ee778cf160c4c03
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27131004ad60a0e53ce4afbf6b5938b652d931a604c2ef9147081316bdd1cb58
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d9d9bf373bb9177c18083da7511a1cf27e7aec1b7409c02381a5c7ffaede440
+size 43429616

last-checkpoint/global_step350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81931d0b9384ae31c6dabf80c0501d96a39631493fc01d9768303da174b201d0
+size 637299

last-checkpoint/global_step350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f499324fd68027eacf299cbbe7bc3fdb8132535402762a6d42a52093afb2e644
+size 637171

last-checkpoint/global_step350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc84fd8164b463b5d2b20670849f7afe073adacfef5d35bdc708fd108503c2e5
+size 637171

last-checkpoint/global_step350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af47b1cce71f0f2679a390a0d88f58018746837e2d03a9ea7b758abb44feb7a6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step300
+global_step350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef002048764051a71fb00f8f978e9ec32b780dc850bdb059af362cc56494234b
+oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37194a6d48612e1a46a2d5d317ead97c70d9fc4569b0118fcd5f84c3dc9daa5a
+oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c179483659a784aa1ace2427daff48c556a6bcc3c330e6f3274e4dc95e4b49
+oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b56857c9b117629f35af2c3d64f522d33a9d8aa94faa81ec6956380a895118c4
+oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad5c396c3dfcdbd96b175132537b2bcb7ae04407121e3f88285feb248284457a
+oid sha256:03940fe8f12e50ac0a3c905dfe63b68f75ffd42f00104f9fc8d680b7aa48b61a
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.810763955116272,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-250",
-  "epoch": 0.08862629246676514,
+  "best_metric": 0.7663387656211853,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-350",
+  "epoch": 0.103397341211226,
   "eval_steps": 50,
-  "global_step": 300,
+  "global_step": 350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -541,11 +541,100 @@
       "eval_steps_per_second": 0.782,
       "num_input_tokens_seen": 3107200,
       "step": 300
+    },
+    {
+      "epoch": 0.09010339734121123,
+      "grad_norm": 2.2491377996087385,
+      "learning_rate": 8.997050147492626e-05,
+      "loss": 0.7394,
+      "num_input_tokens_seen": 3158648,
+      "step": 305
+    },
+    {
+      "epoch": 0.0915805022156573,
+      "grad_norm": 8.740989037358858,
+      "learning_rate": 9.144542772861357e-05,
+      "loss": 0.7371,
+      "num_input_tokens_seen": 3210560,
+      "step": 310
+    },
+    {
+      "epoch": 0.0930576070901034,
+      "grad_norm": 4.552322042735297,
+      "learning_rate": 9.29203539823009e-05,
+      "loss": 0.7622,
+      "num_input_tokens_seen": 3263664,
+      "step": 315
+    },
+    {
+      "epoch": 0.09453471196454949,
+      "grad_norm": 5.062297996808734,
+      "learning_rate": 9.43952802359882e-05,
+      "loss": 0.7214,
+      "num_input_tokens_seen": 3315520,
+      "step": 320
+    },
+    {
+      "epoch": 0.09601181683899557,
+      "grad_norm": 4.295724990139621,
+      "learning_rate": 9.587020648967551e-05,
+      "loss": 0.7078,
+      "num_input_tokens_seen": 3368088,
+      "step": 325
+    },
+    {
+      "epoch": 0.09748892171344166,
+      "grad_norm": 7.980776602247676,
+      "learning_rate": 9.734513274336283e-05,
+      "loss": 0.6852,
+      "num_input_tokens_seen": 3420176,
+      "step": 330
+    },
+    {
+      "epoch": 0.09896602658788774,
+      "grad_norm": 33.51326353666061,
+      "learning_rate": 9.882005899705014e-05,
+      "loss": 0.7557,
+      "num_input_tokens_seen": 3471184,
+      "step": 335
+    },
+    {
+      "epoch": 0.10044313146233383,
+      "grad_norm": 4.612370523858782,
+      "learning_rate": 9.99999940340072e-05,
+      "loss": 0.6709,
+      "num_input_tokens_seen": 3523008,
+      "step": 340
+    },
+    {
+      "epoch": 0.1019202363367799,
+      "grad_norm": 3.706129373980499,
+      "learning_rate": 9.999978522440803e-05,
+      "loss": 0.7252,
+      "num_input_tokens_seen": 3573880,
+      "step": 345
+    },
+    {
+      "epoch": 0.103397341211226,
+      "grad_norm": 4.907030070826967,
+      "learning_rate": 9.999927811659165e-05,
+      "loss": 0.6602,
+      "num_input_tokens_seen": 3625752,
+      "step": 350
+    },
+    {
+      "epoch": 0.103397341211226,
+      "eval_loss": 0.7663387656211853,
+      "eval_runtime": 19.2114,
+      "eval_samples_per_second": 3.123,
+      "eval_steps_per_second": 0.781,
+      "num_input_tokens_seen": 3625752,
+      "step": 350
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 3107200,
+  "num_input_tokens_seen": 3625752,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -560,7 +649,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 204907697143808.0,
+  "total_flos": 239163058946048.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null