Training in progress, step 2900, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1404e60fabb5103dbe434a7e9d7d0543eaa50c6cb487e2d805559319b22760ea
+oid sha256:f49768fb6efa912a563fdf9c78311249a1d28fb66c672197a06fe3282fb5f3cb
 size 18516456

last-checkpoint/global_step2899/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f460b267da24112c290816f70f6e7137581679540f12079396e6fa596f227d10
+size 27700976

last-checkpoint/global_step2899/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d69693a67b4ea41309a61a3ffb732cadb9a0af9665e90775ba2140ca9c9ef4d
+size 27700976

last-checkpoint/global_step2899/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd04bb9b2b31222f2a1031a263103260b1efd98e8a6328299f3354c8bd75e208
+size 27700976

last-checkpoint/global_step2899/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b8c495c7d5697816e1ca24073f934211e838d5ce8d558e296da94a623d57ce
+size 27700976

last-checkpoint/global_step2899/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f38235c2e23e938a4dd5ea4feae6b61cd97138c092570aeedf2bde303d6784
+size 411571

last-checkpoint/global_step2899/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1e479890dcb45e1d295d4f36f50fd9c05ff83ef783297bca83a8a59ee446f67
+size 411507

last-checkpoint/global_step2899/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4feeeb48e41148c692fda7743fe9c0e24bb7bab99843a4be41bdd24cdbf4b8fc
+size 411507

last-checkpoint/global_step2899/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c68da5f7c173d41b9d207e860c424c6418eb30b60731c2bdf8bb7524c2b86d27
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2849
+global_step2899

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce92cea831a04716b4b472f1dad1cc986b2021dee9aac057217f5d455b27ec42
+oid sha256:bfc5d4e344535f1dd0ff5275633ec3d55bb6249e432442365ff24445d82ec35c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cddb73bbdf0f6f6a2c3182d70f7ad5d587353b164c08dd4f383b940d6b61e4e
+oid sha256:8a898928042c09dc123c1025557279997043b7f607bc91ee2ff2d4b4d2b9f1ba
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b24b508e466beb446d37377d2a04757d3bc2b4230de3ac56b25a65d7753a74c1
+oid sha256:6923d07d979aa78d66765208f598662fd5092b5227cd87920feedfb923fa375f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c6a18a7de8b25b21673ba2ff7efbaaae00ec8c453c7975b467c1df87b87022
+oid sha256:e9ba35b9b3c512fbe857d909557329ba47dbefe5f521014123c05901c32edb6d
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28cb8f1d536b1fc196e353ec37fd07d574fab0a464ddd8b31c73c59dcab3c03b
+oid sha256:8dda654474af0b5740253998af53f5373a7c152f4ee919c2beac18e6e6497944
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.467679629152717,
+  "epoch": 1.4934329126963688,
   "eval_steps": 50,
-  "global_step": 2850,
+  "global_step": 2900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5080,11 +5080,100 @@
       "eval_steps_per_second": 0.938,
       "num_input_tokens_seen": 33327720,
       "step": 2850
+    },
+    {
+      "epoch": 1.4702549575070822,
+      "grad_norm": 6.856320486947826,
+      "learning_rate": 6.861743852128233e-06,
+      "loss": 0.2811,
+      "num_input_tokens_seen": 33386160,
+      "step": 2855
+    },
+    {
+      "epoch": 1.4728302858614473,
+      "grad_norm": 8.133776634702407,
+      "learning_rate": 6.7393129075627335e-06,
+      "loss": 0.2394,
+      "num_input_tokens_seen": 33444648,
+      "step": 2860
+    },
+    {
+      "epoch": 1.4754056142158125,
+      "grad_norm": 5.884612144672532,
+      "learning_rate": 6.6179050846274515e-06,
+      "loss": 0.243,
+      "num_input_tokens_seen": 33503144,
+      "step": 2865
+    },
+    {
+      "epoch": 1.4779809425701778,
+      "grad_norm": 7.133095118516192,
+      "learning_rate": 6.497523254635296e-06,
+      "loss": 0.242,
+      "num_input_tokens_seen": 33561600,
+      "step": 2870
+    },
+    {
+      "epoch": 1.4805562709245428,
+      "grad_norm": 3.725193081900286,
+      "learning_rate": 6.37817026463432e-06,
+      "loss": 0.1864,
+      "num_input_tokens_seen": 33620056,
+      "step": 2875
+    },
+    {
+      "epoch": 1.483131599278908,
+      "grad_norm": 5.26408055314188,
+      "learning_rate": 6.25984893734034e-06,
+      "loss": 0.2406,
+      "num_input_tokens_seen": 33678512,
+      "step": 2880
+    },
+    {
+      "epoch": 1.4857069276332733,
+      "grad_norm": 5.139938399894378,
+      "learning_rate": 6.142562071070179e-06,
+      "loss": 0.2287,
+      "num_input_tokens_seen": 33736960,
+      "step": 2885
+    },
+    {
+      "epoch": 1.4882822559876385,
+      "grad_norm": 5.551633292498772,
+      "learning_rate": 6.026312439675552e-06,
+      "loss": 0.2643,
+      "num_input_tokens_seen": 33795416,
+      "step": 2890
+    },
+    {
+      "epoch": 1.4908575843420036,
+      "grad_norm": 5.974549504189433,
+      "learning_rate": 5.911102792477357e-06,
+      "loss": 0.2956,
+      "num_input_tokens_seen": 33853936,
+      "step": 2895
+    },
+    {
+      "epoch": 1.4934329126963688,
+      "grad_norm": 5.786971041370645,
+      "learning_rate": 5.796935854200763e-06,
+      "loss": 0.29,
+      "num_input_tokens_seen": 33912440,
+      "step": 2900
+    },
+    {
+      "epoch": 1.4934329126963688,
+      "eval_loss": 0.8113046884536743,
+      "eval_runtime": 16.0025,
+      "eval_samples_per_second": 3.749,
+      "eval_steps_per_second": 0.937,
+      "num_input_tokens_seen": 33912440,
+      "step": 2900
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 33327720,
+  "num_input_tokens_seen": 33912440,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5099,7 +5188,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1871508641415168.0,
+  "total_flos": 1904344826904576.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null