Training in progress, step 1200, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:050dc1f1854d68a42fb308d9dd143ddc3d6fc0d86870ea1eac6bf90317df973a
+oid sha256:419197c7b895b88c7afacbe056bbf0e1d22ab3d4ba5caafffba674acfa991533
 size 29034840

last-checkpoint/global_step1200/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1aec5f88c70086eb769372a167d1660968748cc110327de5c320debe43f371
+size 43429616

last-checkpoint/global_step1200/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae158e05ebda4f41c8ce55623f0bddf3afa6bbc045c2ada936c7da24035799d3
+size 43429616

last-checkpoint/global_step1200/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26f6bf14fc9c2c40db373922da15d23c0d08a53dc9d0c74f5f03dc1437ca0fb7
+size 43429616

last-checkpoint/global_step1200/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31febf6ba053a4152cb24e518f2e806dda3c3010aa10d75d7e5a858d33d3eb36
+size 43429616

last-checkpoint/global_step1200/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d06f5911185a6ebb03a50cfa88a0b397d7dcb732800b879d25a0b602da1df67
+size 637299

last-checkpoint/global_step1200/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ddb6eccf101a9bfdd7a0e209ad8e34c5bd436c139a6cc24e86858cfb663649f
+size 637171

last-checkpoint/global_step1200/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63fa0ba1e22cd4330a732b53b080494fee16b561a8509fb226cb0abdd186e7ea
+size 637171

last-checkpoint/global_step1200/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a54361c50a0229f80835738434c247594a5157c91a95390f6abdef0e295dfb9d
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1150
+global_step1200

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:274dc3860ee0c7f4d5348f60910a4b568498c04adfefb89f905b1c78a82c1312
+oid sha256:a209a0c0025f9ce8e2beeba50c1f0828d5c34a2482310fcd0bf5fc24c2c67be2
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9aa441491b9ca89e796944520fa1db332a67c0a1a920be83edd2d96d741716d
+oid sha256:a67fb929b8c51f9b1c6ff9f11366e57e55128a1d36df85a9d37a008b49017a75
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1ee3434533b24fb771504fa8cceb5c2ea25fe0de1641128feaceccc65afe6ed
+oid sha256:1b9ef3b0c0978d0b611f4257c939f1c2c6f07e6227bfea6675532d285b0b64a7
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93b4a44be1335173d2e3120bd0d1e6346f3e832d8935752c70ce1e98f017fa87
+oid sha256:0183d14c8ed52ee533139532e9bcf7bc34ec297a064845b35741cb501d92675f
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d461c8d7517d4b88333bff7984fc3bfc149292198b04bbc18a49aee698ffb5c
+oid sha256:16e0cffc6b063574ed312ee2198c86d3dddf2450d0400f042fdc08bd22dcbc7f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 1.5566551155643538e-05,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-1150",
-  "epoch": 0.5923255215039918,
+  "epoch": 0.6180788050476436,
   "eval_steps": 50,
-  "global_step": 1150,
+  "global_step": 1200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2054,11 +2054,100 @@
       "eval_steps_per_second": 0.775,
       "num_input_tokens_seen": 11481600,
       "step": 1150
+    },
+    {
+      "epoch": 0.5949008498583569,
+      "grad_norm": 0.00046149068264744525,
+      "learning_rate": 7.875623046909544e-05,
+      "loss": 0.0003,
+      "num_input_tokens_seen": 11531520,
+      "step": 1155
+    },
+    {
+      "epoch": 0.5974761782127221,
+      "grad_norm": 0.0007891686440962277,
+      "learning_rate": 7.855697251155967e-05,
+      "loss": 0.0,
+      "num_input_tokens_seen": 11581440,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6000515065670873,
+      "grad_norm": 6.055667685356653,
+      "learning_rate": 7.835703917741212e-05,
+      "loss": 0.0414,
+      "num_input_tokens_seen": 11631360,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6026268349214525,
+      "grad_norm": 3.9367530795344696,
+      "learning_rate": 7.81564351951057e-05,
+      "loss": 0.02,
+      "num_input_tokens_seen": 11681280,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6052021632758177,
+      "grad_norm": 0.014148361508830113,
+      "learning_rate": 7.795516530895414e-05,
+      "loss": 0.0002,
+      "num_input_tokens_seen": 11731200,
+      "step": 1175
+    },
+    {
+      "epoch": 0.6077774916301828,
+      "grad_norm": 0.12219412030414617,
+      "learning_rate": 7.775323427901993e-05,
+      "loss": 0.0026,
+      "num_input_tokens_seen": 11781120,
+      "step": 1180
+    },
+    {
+      "epoch": 0.610352819984548,
+      "grad_norm": 5.108798582105785,
+      "learning_rate": 7.755064688100171e-05,
+      "loss": 0.0168,
+      "num_input_tokens_seen": 11831040,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6129281483389132,
+      "grad_norm": 0.004932933139070266,
+      "learning_rate": 7.734740790612136e-05,
+      "loss": 0.0037,
+      "num_input_tokens_seen": 11880960,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6155034766932784,
+      "grad_norm": 0.022109282268939556,
+      "learning_rate": 7.714352216101055e-05,
+      "loss": 0.0007,
+      "num_input_tokens_seen": 11930880,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6180788050476436,
+      "grad_norm": 0.037426371340479145,
+      "learning_rate": 7.693899446759727e-05,
+      "loss": 0.0038,
+      "num_input_tokens_seen": 11980800,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6180788050476436,
+      "eval_loss": 0.0022160401567816734,
+      "eval_runtime": 18.725,
+      "eval_samples_per_second": 3.204,
+      "eval_steps_per_second": 0.801,
+      "num_input_tokens_seen": 11980800,
+      "step": 1200
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 11481600,
+  "num_input_tokens_seen": 11980800,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2073,7 +2162,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 757564358524928.0,
+  "total_flos": 790504338096128.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null