Training in progress, step 1550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c22365f9f1922257bce2b624ba131f902a872d3af6a64b3fc1e949516f00f5ac
+oid sha256:c433a8aa8f1650b5f131001712aa61d13d2d36f41df4d65d21c83dee89eb91ab
 size 18516456

last-checkpoint/global_step1550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16d7ebe27708083904b852fe115e64b12162b20902da7f2c1e2fe91f23111d09
+size 27700976

last-checkpoint/global_step1550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449164cc8d966cef473ae8dc64f85ae163e445e03aedefec776b0319085375c8
+size 27700976

last-checkpoint/global_step1550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e7b5f20438ddfdc7004ca2513ac5b8f1ceea90a1d8d8671f94704c13491a889
+size 27700976

last-checkpoint/global_step1550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:108bcd1a0645dbb2138024332876d5815aafed8c65411a604f8264c49ee86b07
+size 27700976

last-checkpoint/global_step1550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be709632e5d1fcdaf9cec70f99c018e4356be3683a1cb28eed1e4adab5f5fe93
+size 411571

last-checkpoint/global_step1550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eddc0ec4f9016093abb6753d09cb365044055ceab60982d7e1460257cdcf0db0
+size 411507

last-checkpoint/global_step1550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8e218616a923b3c2449e19337a97f40cf4b60e78fa26c554369524a0b898da
+size 411507

last-checkpoint/global_step1550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e02b898ca30292290d26463d8070037ac164ebcd3bc9b49bf8c96bab757faa52
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1450
+global_step1550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd4f3298d54e9509917373bcd359e11c92a8e0aa77b2cc0825602efd186ad77e
+oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fdab02a791039ff46df83a272c972ab0f366fcea91338790dc016dbbbf6de80
+oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2808762a5f80587f05c1e2e8de26b5aef1010331cea366649319d593dbb0e66
+oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c052112cbe3f325fd4543a02558091b80c9c8abcd8db8822be9a8306eab6f2b7
+oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22e337e693b8d4ecad5d49a11b7f0649adec1748625a9390ba4ddabc0e901287
+oid sha256:8eb069683e7f84aa36296476346fc663361d9b05ad7b09b71f22f44afdb0ea48
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.6643603444099426,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1400",
-  "epoch": 0.7468452227659027,
+  "best_metric": 0.632923424243927,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1500",
+  "epoch": 0.7983517898532063,
   "eval_steps": 50,
-  "global_step": 1450,
+  "global_step": 1550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2588,11 +2588,189 @@
       "eval_steps_per_second": 0.935,
       "num_input_tokens_seen": 16959632,
       "step": 1450
+    },
+    {
+      "epoch": 0.7494205511202678,
+      "grad_norm": 1.9571252974715032,
+      "learning_rate": 6.57742480148798e-05,
+      "loss": 0.6533,
+      "num_input_tokens_seen": 17018072,
+      "step": 1455
+    },
+    {
+      "epoch": 0.751995879474633,
+      "grad_norm": 3.2075825448529542,
+      "learning_rate": 6.554332297097031e-05,
+      "loss": 0.7114,
+      "num_input_tokens_seen": 17076560,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7545712078289982,
+      "grad_norm": 2.0030816579741266,
+      "learning_rate": 6.53120303251801e-05,
+      "loss": 0.6568,
+      "num_input_tokens_seen": 17135016,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7571465361833634,
+      "grad_norm": 2.65056436638165,
+      "learning_rate": 6.508037554761432e-05,
+      "loss": 0.7016,
+      "num_input_tokens_seen": 17193496,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7597218645377286,
+      "grad_norm": 1.9541651871708403,
+      "learning_rate": 6.484836411694267e-05,
+      "loss": 0.6612,
+      "num_input_tokens_seen": 17251944,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7622971928920937,
+      "grad_norm": 3.0540242692558577,
+      "learning_rate": 6.461600152026965e-05,
+      "loss": 0.6115,
+      "num_input_tokens_seen": 17310456,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7648725212464589,
+      "grad_norm": 2.796196437541352,
+      "learning_rate": 6.438329325300499e-05,
+      "loss": 0.6458,
+      "num_input_tokens_seen": 17368968,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7674478496008241,
+      "grad_norm": 3.1979427976381207,
+      "learning_rate": 6.415024481873352e-05,
+      "loss": 0.6434,
+      "num_input_tokens_seen": 17427424,
+      "step": 1490
+    },
+    {
+      "epoch": 0.7700231779551893,
+      "grad_norm": 3.8375601078700203,
+      "learning_rate": 6.391686172908506e-05,
+      "loss": 0.5973,
+      "num_input_tokens_seen": 17485936,
+      "step": 1495
+    },
+    {
+      "epoch": 0.7725985063095545,
+      "grad_norm": 2.405705749864128,
+      "learning_rate": 6.368314950360415e-05,
+      "loss": 0.6021,
+      "num_input_tokens_seen": 17544440,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7725985063095545,
+      "eval_loss": 0.632923424243927,
+      "eval_runtime": 16.1038,
+      "eval_samples_per_second": 3.726,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 17544440,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7751738346639196,
+      "grad_norm": 2.8519087211521734,
+      "learning_rate": 6.344911366961934e-05,
+      "loss": 0.5779,
+      "num_input_tokens_seen": 17602952,
+      "step": 1505
+    },
+    {
+      "epoch": 0.7777491630182848,
+      "grad_norm": 2.861290579940173,
+      "learning_rate": 6.321475976211266e-05,
+      "loss": 0.6707,
+      "num_input_tokens_seen": 17661440,
+      "step": 1510
+    },
+    {
+      "epoch": 0.78032449137265,
+      "grad_norm": 3.541365161144121,
+      "learning_rate": 6.298009332358856e-05,
+      "loss": 0.6326,
+      "num_input_tokens_seen": 17719928,
+      "step": 1515
+    },
+    {
+      "epoch": 0.7828998197270152,
+      "grad_norm": 2.969962641272996,
+      "learning_rate": 6.274511990394294e-05,
+      "loss": 0.6472,
+      "num_input_tokens_seen": 17778424,
+      "step": 1520
+    },
+    {
+      "epoch": 0.7854751480813804,
+      "grad_norm": 2.762063548864621,
+      "learning_rate": 6.250984506033183e-05,
+      "loss": 0.6215,
+      "num_input_tokens_seen": 17836936,
+      "step": 1525
+    },
+    {
+      "epoch": 0.7880504764357456,
+      "grad_norm": 3.2198855545004097,
+      "learning_rate": 6.227427435703997e-05,
+      "loss": 0.6102,
+      "num_input_tokens_seen": 17895392,
+      "step": 1530
+    },
+    {
+      "epoch": 0.7906258047901107,
+      "grad_norm": 3.846544371420393,
+      "learning_rate": 6.203841336534924e-05,
+      "loss": 0.6161,
+      "num_input_tokens_seen": 17953872,
+      "step": 1535
+    },
+    {
+      "epoch": 0.7932011331444759,
+      "grad_norm": 3.811248686105134,
+      "learning_rate": 6.180226766340688e-05,
+      "loss": 0.6103,
+      "num_input_tokens_seen": 18012320,
+      "step": 1540
+    },
+    {
+      "epoch": 0.7957764614988411,
+      "grad_norm": 2.9539705466919703,
+      "learning_rate": 6.156584283609359e-05,
+      "loss": 0.5791,
+      "num_input_tokens_seen": 18070792,
+      "step": 1545
+    },
+    {
+      "epoch": 0.7983517898532063,
+      "grad_norm": 3.0546686267383283,
+      "learning_rate": 6.132914447489137e-05,
+      "loss": 0.667,
+      "num_input_tokens_seen": 18129304,
+      "step": 1550
+    },
+    {
+      "epoch": 0.7983517898532063,
+      "eval_loss": 0.6617516279220581,
+      "eval_runtime": 16.0333,
+      "eval_samples_per_second": 3.742,
+      "eval_steps_per_second": 0.936,
+      "num_input_tokens_seen": 18129304,
+      "step": 1550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 16959632,
+  "num_input_tokens_seen": 18129304,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2607,7 +2785,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 952313688883200.0,
+  "total_flos": 1017997776650240.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null