Training in progress, step 1350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b44254d9a1623586ea1f5a75bfc6d736cfd5e37677b54824320d6e82460a2035
+oid sha256:c5539aab9f718348f04ec41613d234e7bd892b381194398a0eadf2adcfde0ddb
 size 29034840

last-checkpoint/global_step1350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e24af6e1a43be5add6cbc89509eb769890f71ee6fa6b54fe0923600620399dbd
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdc11fd4b339cc5d601d29b4a63f03eb3b4425c4b7691bcb26a63957620b86b5
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:649a3c6c2e41790aba280e8054ec3e4fd58e892c2b27b75d5453836d0575bf49
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c7b9e52dd1f46868c6b578707680d4a3b5d2f23924ee30a2c75ef30b65e6c4b
+size 43429616

last-checkpoint/global_step1350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf6a04ed575d4bbe95b19e28c451beac968995fe7f81b545d3f82185be8aab33
+size 637299

last-checkpoint/global_step1350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6797229513b11978627019edbf1729dfa52f86c1151aac9777d9693a2be4536b
+size 637171

last-checkpoint/global_step1350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3dff6f8ada3bacd5905b06ee331028a57bd1b141fb55f651aca5cbe974a3a95
+size 637171

last-checkpoint/global_step1350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0cfc18b2fc2a54e8761219cee72f1cb08510e09bfc522456e8dbdebcd2dde79
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1300
+global_step1350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c364a094b8b4b8d6b015687012206e88b2233dd7d6a4f6f395d7aef77752ea67
+oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bd2e297a13ac051fc5f3cce9c34767e51a5cb4574835aa8bd1309d8cdc48053
+oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:028047dd1753d92e11bf971ee14a5c981a9a3ea6631f228e38475027eb5ae430
+oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000eb4cb4096582f15856f380d5f2a9a00eaecdbb95f2289a7a81a0a624fdf72
+oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:522af9421667c7c03e6690f39891bfba59de53aa879f1f8a4c0690d7908aa17c
+oid sha256:5473741dbc1be4510ceec76e5e3cd10aedf1c7667da998c551f8978b6c6c33d3
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2966395914554596,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1200",
-  "epoch": 0.38404726735598227,
+  "epoch": 0.3988183161004431,
   "eval_steps": 50,
-  "global_step": 1300,
+  "global_step": 1350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2321,11 +2321,100 @@
       "eval_steps_per_second": 0.786,
       "num_input_tokens_seen": 13493264,
       "step": 1300
+    },
+    {
+      "epoch": 0.38552437223042835,
+      "grad_norm": 15.50071615139337,
+      "learning_rate": 9.453534651910765e-05,
+      "loss": 0.402,
+      "num_input_tokens_seen": 13545256,
+      "step": 1305
+    },
+    {
+      "epoch": 0.3870014771048744,
+      "grad_norm": 23.183495844663526,
+      "learning_rate": 9.447969751791577e-05,
+      "loss": 0.3075,
+      "num_input_tokens_seen": 13597792,
+      "step": 1310
+    },
+    {
+      "epoch": 0.38847858197932056,
+      "grad_norm": 9.67544956653079,
+      "learning_rate": 9.442378315129455e-05,
+      "loss": 0.3702,
+      "num_input_tokens_seen": 13649848,
+      "step": 1315
+    },
+    {
+      "epoch": 0.38995568685376664,
+      "grad_norm": 2.9059361985914416,
+      "learning_rate": 9.436760375282859e-05,
+      "loss": 0.3603,
+      "num_input_tokens_seen": 13701592,
+      "step": 1320
+    },
+    {
+      "epoch": 0.3914327917282127,
+      "grad_norm": 10.431238621222658,
+      "learning_rate": 9.431115965768358e-05,
+      "loss": 0.4072,
+      "num_input_tokens_seen": 13753064,
+      "step": 1325
+    },
+    {
+      "epoch": 0.3929098966026588,
+      "grad_norm": 11.216612661805582,
+      "learning_rate": 9.425445120260445e-05,
+      "loss": 0.3279,
+      "num_input_tokens_seen": 13805528,
+      "step": 1330
+    },
+    {
+      "epoch": 0.39438700147710487,
+      "grad_norm": 32.22838128750362,
+      "learning_rate": 9.419747872591325e-05,
+      "loss": 0.3754,
+      "num_input_tokens_seen": 13858192,
+      "step": 1335
+    },
+    {
+      "epoch": 0.39586410635155095,
+      "grad_norm": 1.8703742105152936,
+      "learning_rate": 9.414024256750723e-05,
+      "loss": 0.3754,
+      "num_input_tokens_seen": 13910128,
+      "step": 1340
+    },
+    {
+      "epoch": 0.397341211225997,
+      "grad_norm": 5.011302513950015,
+      "learning_rate": 9.408274306885674e-05,
+      "loss": 0.3235,
+      "num_input_tokens_seen": 13962536,
+      "step": 1345
+    },
+    {
+      "epoch": 0.3988183161004431,
+      "grad_norm": 15.197987760428996,
+      "learning_rate": 9.402498057300317e-05,
+      "loss": 0.3731,
+      "num_input_tokens_seen": 14014736,
+      "step": 1350
+    },
+    {
+      "epoch": 0.3988183161004431,
+      "eval_loss": 0.5565826892852783,
+      "eval_runtime": 19.3029,
+      "eval_samples_per_second": 3.108,
+      "eval_steps_per_second": 0.777,
+      "num_input_tokens_seen": 14014736,
+      "step": 1350
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 13493264,
+  "num_input_tokens_seen": 14014736,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2340,7 +2429,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 890156538658816.0,
+  "total_flos": 924522107633664.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null