Training in progress, step 2050, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b27b450bf69f26bb0a80206beeb4dd904291117c93705869b5a8ff84aa1e3f84
+oid sha256:e58273020b4ec80c27bd67a1dcf93114d440f82a76094d3bc50faf05938ff6dc
 size 18516456

last-checkpoint/global_step2049/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4659cb8935558c3f29594b6336fdf139feda949ae604a14b0ca0daf1dd0e39e8
+size 27700976

last-checkpoint/global_step2049/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe5cda623b4b40a90b10a4ffc064db2f9eeafa3c79959bf407d352bf536bd4d
+size 27700976

last-checkpoint/global_step2049/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd333e9899a1e7c19b8c120109c9af616a96c14990e85766838bac9c5c801b66
+size 27700976

last-checkpoint/global_step2049/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a2a05a345b53894b99bcca51f2d7b0bf5ddfe872c78fee6af0890fb76c759a5
+size 27700976

last-checkpoint/global_step2049/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9086ce0cdbb623c2ce84fad441a76e576e34a80aec2724cc0323f604ead25f05
+size 411571

last-checkpoint/global_step2049/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f9680b16c527b3532e5f34fcddccd5d261758fbf64c1765a5583d2877490d6
+size 411507

last-checkpoint/global_step2049/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2bb017dbec9d0f2a9dbc3e3548d74d100e3603cb3606daa0edf8ddf36497c1b
+size 411507

last-checkpoint/global_step2049/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55416eff78220c5c9e94910c7dad6f64419db9d9c1fbfadc4d6f66ca6ca5b918
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1999
+global_step2049

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4a695de1db3382235d3f8ae213672491aa2fdc3ba3be96403a089077ad3c2bf
+oid sha256:5d1ddd7e7b4dc44903837b0414e4659f8383cd8f16b41dd396d4eaf5b9829f79
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2025b170fa1d4693537c2d73f89a6495c58940d033678742a74810c0154a6a7
+oid sha256:eeddfb77fe4d3b495c4e08307767e08df90e96ef241c3eb80d5f75adec393e80
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0865c4d7d921b23a22c91c2f3b2c6cca03dae0eb27c43dee575c9602605c94d6
+oid sha256:9e6260196fabb00061b1f1c8de6288382570dc14d02d2aa308050ca858880a97
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f4ec19d9df4417359523e8cc4d27875614c1021ebcc6391b27632aa7897b7ea
+oid sha256:ea023bc5b1def54e0c49389175c0fae812f5f764c502525ce775d993d5ab2c03
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f900ab4b5112c328cc663d738cb777b204b3c8bdcbb2ceb91ed099a09d207ee1
+oid sha256:fdc2918e410c169f1230e99c777b80638166c2769acb78c23195e4c48ca5b249
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.029873808910636,
+  "epoch": 1.0556270924542879,
   "eval_steps": 50,
-  "global_step": 2000,
+  "global_step": 2050,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3567,11 +3567,100 @@
       "eval_steps_per_second": 0.932,
       "num_input_tokens_seen": 23386232,
       "step": 2000
+    },
+    {
+      "epoch": 1.0324491372650013,
+      "grad_norm": 5.8532055715340245,
+      "learning_rate": 3.938253580098613e-05,
+      "loss": 0.362,
+      "num_input_tokens_seen": 23444712,
+      "step": 2005
+    },
+    {
+      "epoch": 1.0350244656193666,
+      "grad_norm": 7.087739461357715,
+      "learning_rate": 3.914505045103845e-05,
+      "loss": 0.3903,
+      "num_input_tokens_seen": 23503192,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0375997939737316,
+      "grad_norm": 6.061997147134047,
+      "learning_rate": 3.8907821822248605e-05,
+      "loss": 0.3341,
+      "num_input_tokens_seen": 23561688,
+      "step": 2015
+    },
+    {
+      "epoch": 1.0401751223280968,
+      "grad_norm": 6.783069419644998,
+      "learning_rate": 3.867085552510864e-05,
+      "loss": 0.4794,
+      "num_input_tokens_seen": 23620160,
+      "step": 2020
+    },
+    {
+      "epoch": 1.042750450682462,
+      "grad_norm": 4.11088291372727,
+      "learning_rate": 3.843415716390644e-05,
+      "loss": 0.4104,
+      "num_input_tokens_seen": 23678624,
+      "step": 2025
+    },
+    {
+      "epoch": 1.045325779036827,
+      "grad_norm": 5.727855298190317,
+      "learning_rate": 3.819773233659314e-05,
+      "loss": 0.3639,
+      "num_input_tokens_seen": 23737064,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0479011073911924,
+      "grad_norm": 6.936114108935384,
+      "learning_rate": 3.7961586634650767e-05,
+      "loss": 0.4294,
+      "num_input_tokens_seen": 23795568,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0504764357455576,
+      "grad_norm": 5.577801320854008,
+      "learning_rate": 3.772572564296005e-05,
+      "loss": 0.4713,
+      "num_input_tokens_seen": 23854040,
+      "step": 2040
+    },
+    {
+      "epoch": 1.0530517640999228,
+      "grad_norm": 7.466883391944433,
+      "learning_rate": 3.749015493966817e-05,
+      "loss": 0.3864,
+      "num_input_tokens_seen": 23912520,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0556270924542879,
+      "grad_norm": 4.120909561971508,
+      "learning_rate": 3.7254880096057073e-05,
+      "loss": 0.3602,
+      "num_input_tokens_seen": 23971048,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0556270924542879,
+      "eval_loss": 0.7190810441970825,
+      "eval_runtime": 16.0858,
+      "eval_samples_per_second": 3.73,
+      "eval_steps_per_second": 0.932,
+      "num_input_tokens_seen": 23971048,
+      "step": 2050
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 23386232,
+  "num_input_tokens_seen": 23971048,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3586,7 +3675,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1313241973129216.0,
+  "total_flos": 1346083234512896.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null