Training in progress, step 1350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69b791d45f5e8c5ec48a79f58002a2b81c7d9dc83eccb3702983e59a8069f0f5
+oid sha256:10b824529fcf1b1cc25b9074d2e972d61eae3cfa9d99a178c08c460258fd9cfb
 size 18516456

last-checkpoint/global_step1350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1ff7758d634fd2817a944d2d4933749ce7e6d0682f36eaba1b11c022468402d
+size 27700976

last-checkpoint/global_step1350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:526d81af1a35b367eb6676a5cf443c017fa62dc7df664d3cb8fabbd164eefff7
+size 27700976

last-checkpoint/global_step1350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ed9346eb435cc9bef44166cf49d154244148637180ab0a5ac560e2cddfcc98b
+size 27700976

last-checkpoint/global_step1350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3649e71adc03e4a984ad48d9f9cc4d6f5cbf17ebf99ca7e2f85da1cc5aa0aee0
+size 27700976

last-checkpoint/global_step1350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba820a9b8f03145e1eef8bd5c6101f62091bf3ef869529103a40cefdc42a2d54
+size 411571

last-checkpoint/global_step1350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e32230362605afd218805da11ef95236d0ddb2e306753c377e8fabf7e581afb8
+size 411507

last-checkpoint/global_step1350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8bedad9cec5c02d9c769524a8aaba6c7bc518663b06904b7110383d2933a2cc
+size 411507

last-checkpoint/global_step1350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c79e6a630a4d21da27b81765db87aa09442e7d1a1cdfe71098b67e6d64a5ae
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1300
+global_step1350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c364a094b8b4b8d6b015687012206e88b2233dd7d6a4f6f395d7aef77752ea67
+oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bd2e297a13ac051fc5f3cce9c34767e51a5cb4574835aa8bd1309d8cdc48053
+oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:028047dd1753d92e11bf971ee14a5c981a9a3ea6631f228e38475027eb5ae430
+oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000eb4cb4096582f15856f380d5f2a9a00eaecdbb95f2289a7a81a0a624fdf72
+oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:480bbb635a5188e39c207a9dac4c52a057993fae728ac539c81e273941d4879d
+oid sha256:ca5b091f7e9fe97f38f540a5a26edd08efe79a114c6bc8ff4d4c0d9acb5b20f2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7189856171607971,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1300",
-  "epoch": 0.6695853721349472,
+  "best_metric": 0.712917685508728,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1350",
+  "epoch": 0.695338655678599,
   "eval_steps": 50,
-  "global_step": 1300,
+  "global_step": 1350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2321,11 +2321,100 @@
       "eval_steps_per_second": 0.936,
       "num_input_tokens_seen": 15205064,
       "step": 1300
+    },
+    {
+      "epoch": 0.6721607004893124,
+      "grad_norm": 2.6154468701895066,
+      "learning_rate": 7.250431889878455e-05,
+      "loss": 0.7524,
+      "num_input_tokens_seen": 15263560,
+      "step": 1305
+    },
+    {
+      "epoch": 0.6747360288436776,
+      "grad_norm": 1.9549500311782502,
+      "learning_rate": 7.228691778882693e-05,
+      "loss": 0.6748,
+      "num_input_tokens_seen": 15322016,
+      "step": 1310
+    },
+    {
+      "epoch": 0.6773113571980427,
+      "grad_norm": 2.991178206089954,
+      "learning_rate": 7.20689895899765e-05,
+      "loss": 0.7571,
+      "num_input_tokens_seen": 15380504,
+      "step": 1315
+    },
+    {
+      "epoch": 0.6798866855524079,
+      "grad_norm": 1.7022848080804835,
+      "learning_rate": 7.185053945626733e-05,
+      "loss": 0.6615,
+      "num_input_tokens_seen": 15438944,
+      "step": 1320
+    },
+    {
+      "epoch": 0.6824620139067731,
+      "grad_norm": 1.739259284519112,
+      "learning_rate": 7.163157255407732e-05,
+      "loss": 0.7421,
+      "num_input_tokens_seen": 15497384,
+      "step": 1325
+    },
+    {
+      "epoch": 0.6850373422611383,
+      "grad_norm": 1.9142982939434143,
+      "learning_rate": 7.141209406200599e-05,
+      "loss": 0.7886,
+      "num_input_tokens_seen": 15555856,
+      "step": 1330
+    },
+    {
+      "epoch": 0.6876126706155035,
+      "grad_norm": 1.7562659805497576,
+      "learning_rate": 7.1192109170752e-05,
+      "loss": 0.7484,
+      "num_input_tokens_seen": 15614368,
+      "step": 1335
+    },
+    {
+      "epoch": 0.6901879989698687,
+      "grad_norm": 1.7590122465257017,
+      "learning_rate": 7.097162308299054e-05,
+      "loss": 0.7086,
+      "num_input_tokens_seen": 15672864,
+      "step": 1340
+    },
+    {
+      "epoch": 0.6927633273242338,
+      "grad_norm": 2.1211445265818845,
+      "learning_rate": 7.07506410132501e-05,
+      "loss": 0.7494,
+      "num_input_tokens_seen": 15731376,
+      "step": 1345
+    },
+    {
+      "epoch": 0.695338655678599,
+      "grad_norm": 2.683073565523052,
+      "learning_rate": 7.052916818778918e-05,
+      "loss": 0.7278,
+      "num_input_tokens_seen": 15789848,
+      "step": 1350
+    },
+    {
+      "epoch": 0.695338655678599,
+      "eval_loss": 0.712917685508728,
+      "eval_runtime": 16.0726,
+      "eval_samples_per_second": 3.733,
+      "eval_steps_per_second": 0.933,
+      "num_input_tokens_seen": 15789848,
+      "step": 1350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 15205064,
+  "num_input_tokens_seen": 15789848,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2340,7 +2429,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 853788671344640.0,
+  "total_flos": 886628555423744.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null