Training in progress, step 400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77c3ac7da08d446d1e9f20eb9182580b5c74675d4321465f97ce8b96cb262f6a
+oid sha256:519addb7c2ccddb2bb04acac1bf357ce6be904b2c9ab029829133da4e8c89699
 size 18516456

last-checkpoint/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a06ea146d517743841dedafd4921891ebd0740a15d0c2bfd8823ec58f5abe76
+size 27700976

last-checkpoint/global_step400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa7b44d0b1d7d864ec298564a04853692e19cb9fdd33312abe9e58126501ff2d
+size 27700976

last-checkpoint/global_step400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71482d39b44d5b522b2e453b2636dbcb5ccf903fb0f056fb8a89d703ab50ac8f
+size 27700976

last-checkpoint/global_step400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a70da10ffd43dc6207b8212d2562506e68344107084de3485651fd9a6a60c425
+size 27700976

last-checkpoint/global_step400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:764c4e135e2ee68387fcfee70aaba78925dd84a6e3b986f3045f817de0787977
+size 411571

last-checkpoint/global_step400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ed1fc180fa186296f1bbe342bd8456ef30c21967763599fea69478a2d84630b
+size 411507

last-checkpoint/global_step400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6694f2a330fc6060aedde96fdf469b233e5ea372b59f1dcffeda8b5e87949454
+size 411507

last-checkpoint/global_step400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93754d9303b7886359f141fd3968d58e1e1e73315cacdc8dfd4682f09349f1ae
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step350
+global_step400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
+oid sha256:543ef05f530d40ee20b8d626b07a69b86597aca643e48897571062f973efe84f
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
+oid sha256:7a23f732e43838ce0398d2636885ac16badbb9bcbc04d1406069ba3027bc5ae0
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
+oid sha256:e10cce960e7068b051c05e35ed6160656be9091c63f13796ac2ed7e9c84e5a72
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
+oid sha256:b6f6049e212b1df5cefc5d834afcd8cc052c73f1457449e9fe8a38d514f54078
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff02966f5e9ff1944d3a06acaf420bd5bf127f85a91f6b1b2b1649b32fc1fd58
+oid sha256:32762cec872a9c6d771fb0f76b3f72991fda55aee1494a130c6c6b449c48a001
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.8908902406692505,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-300",
-  "epoch": 0.1802729848055627,
+  "epoch": 0.20602626834921453,
   "eval_steps": 50,
-  "global_step": 350,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -630,11 +630,100 @@
       "eval_steps_per_second": 0.905,
       "num_input_tokens_seen": 4093688,
       "step": 350
+    },
+    {
+      "epoch": 0.1828483131599279,
+      "grad_norm": 0.32233149132200706,
+      "learning_rate": 9.91927536630402e-05,
+      "loss": 0.8812,
+      "num_input_tokens_seen": 4152160,
+      "step": 355
+    },
+    {
+      "epoch": 0.18542364151429308,
+      "grad_norm": 0.5071871697326992,
+      "learning_rate": 9.91486549841951e-05,
+      "loss": 0.9109,
+      "num_input_tokens_seen": 4210648,
+      "step": 360
+    },
+    {
+      "epoch": 0.18799896986865824,
+      "grad_norm": 0.4532792519849944,
+      "learning_rate": 9.91033939324107e-05,
+      "loss": 0.9176,
+      "num_input_tokens_seen": 4269136,
+      "step": 365
+    },
+    {
+      "epoch": 0.19057429822302344,
+      "grad_norm": 0.5409761562534501,
+      "learning_rate": 9.905697157811761e-05,
+      "loss": 0.9077,
+      "num_input_tokens_seen": 4327664,
+      "step": 370
+    },
+    {
+      "epoch": 0.19314962657738863,
+      "grad_norm": 0.3432361562809093,
+      "learning_rate": 9.900938901921131e-05,
+      "loss": 0.893,
+      "num_input_tokens_seen": 4386120,
+      "step": 375
+    },
+    {
+      "epoch": 0.1957249549317538,
+      "grad_norm": 0.4756530294720616,
+      "learning_rate": 9.896064738102635e-05,
+      "loss": 0.9094,
+      "num_input_tokens_seen": 4444560,
+      "step": 380
+    },
+    {
+      "epoch": 0.19830028328611898,
+      "grad_norm": 0.424836974193983,
+      "learning_rate": 9.891074781630966e-05,
+      "loss": 0.9091,
+      "num_input_tokens_seen": 4503016,
+      "step": 385
+    },
+    {
+      "epoch": 0.20087561164048418,
+      "grad_norm": 0.31316926977469683,
+      "learning_rate": 9.885969150519331e-05,
+      "loss": 0.9033,
+      "num_input_tokens_seen": 4561496,
+      "step": 390
+    },
+    {
+      "epoch": 0.20345093999484934,
+      "grad_norm": 0.6108378682480797,
+      "learning_rate": 9.88074796551666e-05,
+      "loss": 0.8851,
+      "num_input_tokens_seen": 4619944,
+      "step": 395
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "grad_norm": 0.38294566619219206,
+      "learning_rate": 9.875411350104744e-05,
+      "loss": 0.9004,
+      "num_input_tokens_seen": 4678384,
+      "step": 400
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "eval_loss": 0.9086406826972961,
+      "eval_runtime": 16.7827,
+      "eval_samples_per_second": 3.575,
+      "eval_steps_per_second": 0.894,
+      "num_input_tokens_seen": 4678384,
+      "step": 400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 4093688,
+  "num_input_tokens_seen": 4678384,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -649,7 +738,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 229835067359232.0,
+  "total_flos": 262667545346048.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null