Training in progress, step 350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c6b6b2163a165d1469f79ac83287cf58b9febbee9aa4bb628b13cd978a905d2
+oid sha256:eded4b34f6a51299f10b1a923e18a4a4f8fbfec56b877276cbf713599e577f0a
 size 29034840

last-checkpoint/global_step350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90a33b14a32671efa5d30ed3f1ddb28ac96d5eb2ccb153161ff5f64cb2ae3488
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51eb4e3fe6fcc6573dc4ade1a280ae414713b4dd43a2d6c4f1364deb54e80cf5
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec2712b5b58a827a32066d1db92f270e2c65f19f2c6fd75b579c617d1ae47aac
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2116ce77930c2cde11d3b0741e8598f567e99af5ec50646068d8605078381c
+size 43429616

last-checkpoint/global_step350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81931d0b9384ae31c6dabf80c0501d96a39631493fc01d9768303da174b201d0
+size 637299

last-checkpoint/global_step350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f499324fd68027eacf299cbbe7bc3fdb8132535402762a6d42a52093afb2e644
+size 637171

last-checkpoint/global_step350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc84fd8164b463b5d2b20670849f7afe073adacfef5d35bdc708fd108503c2e5
+size 637171

last-checkpoint/global_step350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af47b1cce71f0f2679a390a0d88f58018746837e2d03a9ea7b758abb44feb7a6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step300
+global_step350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef002048764051a71fb00f8f978e9ec32b780dc850bdb059af362cc56494234b
+oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37194a6d48612e1a46a2d5d317ead97c70d9fc4569b0118fcd5f84c3dc9daa5a
+oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c179483659a784aa1ace2427daff48c556a6bcc3c330e6f3274e4dc95e4b49
+oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b56857c9b117629f35af2c3d64f522d33a9d8aa94faa81ec6956380a895118c4
+oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75979a52de119ed6c9cd6063ded2b7d6cb6c4870db4a05a6cc6a997d1fa9b653
+oid sha256:ff02966f5e9ff1944d3a06acaf420bd5bf127f85a91f6b1b2b1649b32fc1fd58
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.030908752232789993,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_scale4/lora/sft/checkpoint-300",
-  "epoch": 0.1545197012619109,
+  "epoch": 0.1802729848055627,
   "eval_steps": 50,
-  "global_step": 300,
+  "global_step": 350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -541,11 +541,100 @@
       "eval_steps_per_second": 0.755,
       "num_input_tokens_seen": 2995200,
       "step": 300
+    },
+    {
+      "epoch": 0.15709502961627608,
+      "grad_norm": 0.2314383428097956,
+      "learning_rate": 9.956959413129585e-05,
+      "loss": 0.0245,
+      "num_input_tokens_seen": 3045120,
+      "step": 305
+    },
+    {
+      "epoch": 0.15967035797064125,
+      "grad_norm": 0.2035043692790888,
+      "learning_rate": 9.953717199596598e-05,
+      "loss": 0.0121,
+      "num_input_tokens_seen": 3095040,
+      "step": 310
+    },
+    {
+      "epoch": 0.16224568632500644,
+      "grad_norm": 0.014152925203425593,
+      "learning_rate": 9.95035782992122e-05,
+      "loss": 0.054,
+      "num_input_tokens_seen": 3144960,
+      "step": 315
+    },
+    {
+      "epoch": 0.16482101467937163,
+      "grad_norm": 1.0162363271291994,
+      "learning_rate": 9.94688138355304e-05,
+      "loss": 0.0849,
+      "num_input_tokens_seen": 3194880,
+      "step": 320
+    },
+    {
+      "epoch": 0.1673963430337368,
+      "grad_norm": 3.92714662464831,
+      "learning_rate": 9.943287942710527e-05,
+      "loss": 0.0503,
+      "num_input_tokens_seen": 3244800,
+      "step": 325
+    },
+    {
+      "epoch": 0.16997167138810199,
+      "grad_norm": 0.47450325213422423,
+      "learning_rate": 9.939577592379088e-05,
+      "loss": 0.0302,
+      "num_input_tokens_seen": 3294720,
+      "step": 330
+    },
+    {
+      "epoch": 0.17254699974246718,
+      "grad_norm": 1.6706834486788253,
+      "learning_rate": 9.935750420309055e-05,
+      "loss": 0.082,
+      "num_input_tokens_seen": 3344640,
+      "step": 335
+    },
+    {
+      "epoch": 0.17512232809683234,
+      "grad_norm": 0.16856461555999938,
+      "learning_rate": 9.931806517013612e-05,
+      "loss": 0.0411,
+      "num_input_tokens_seen": 3394560,
+      "step": 340
+    },
+    {
+      "epoch": 0.17769765645119753,
+      "grad_norm": 8.475859938133665,
+      "learning_rate": 9.927745975766654e-05,
+      "loss": 0.1158,
+      "num_input_tokens_seen": 3444480,
+      "step": 345
+    },
+    {
+      "epoch": 0.1802729848055627,
+      "grad_norm": 0.7891476176494399,
+      "learning_rate": 9.923568892600578e-05,
+      "loss": 0.1761,
+      "num_input_tokens_seen": 3494400,
+      "step": 350
+    },
+    {
+      "epoch": 0.1802729848055627,
+      "eval_loss": 0.043948542326688766,
+      "eval_runtime": 19.3589,
+      "eval_samples_per_second": 3.099,
+      "eval_steps_per_second": 0.775,
+      "num_input_tokens_seen": 3494400,
+      "step": 350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 2995200,
+  "num_input_tokens_seen": 3494400,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -560,7 +649,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 197584705814528.0,
+  "total_flos": 230524685385728.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null