Training in progress, step 8000, checkpoint

checkpoint-8000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2df9f8a10b115d8fc1bf1d7e1e523d814cec28794c892bfcf77f4c83cb6489a
+oid sha256:82bb8589145efcd6f1cad2d530b6a2f46b6ad943614a4c34a89f9fb2dcb8e09f
 size 1711848436

checkpoint-8000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1657829b84318da781c750c87a0b2504d9139c5ef45fe15a2aff589552668273
+oid sha256:c8a2db9df52f995d7f0ef39a741d95b54bd56066eb4c90279a5caea968e8ea32
 size 3424043887

checkpoint-8000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41054f6a7f29c2148ed6fea73ad66d1725aa683821303aeecc507f856769e9c4
+oid sha256:4b2247dee284601e519b2a7e136f215fcb4b92488ebba0a76124c0bc4db33f7a
 size 14503

checkpoint-8000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:853cafe738981ecb1cabe8a3013a21d8f81574a2d84d254ad9e7e212804eb56b
+oid sha256:9b501706ac003ce6bbb834c567e73f630ded94c35d4679963029f62784bf955e
 size 623

checkpoint-8000/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 0.9886976480484009,
+  "best_metric": 1.0260424613952637,
   "best_model_checkpoint": "./output/clip-finetuned-csu-p14-336-e3l58-l/checkpoint-8000",
-  "epoch": 1.4738393515106853,
+  "epoch": 0.8532423208191127,
   "eval_steps": 500,
   "global_step": 8000,
   "is_hyper_param_search": false,
@@ -9,248 +9,248 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.09211495946941783,
-      "grad_norm": 19.808820724487305,
-      "learning_rate": 4.84647506755097e-08,
-      "loss": 0.3758,
+      "epoch": 0.05332764505119454,
+      "grad_norm": 217.40109252929688,
+      "learning_rate": 4.911120591581342e-08,
+      "loss": 0.4667,
       "step": 500
     },
     {
-      "epoch": 0.09211495946941783,
-      "eval_loss": 1.418845295906067,
-      "eval_runtime": 75.2377,
-      "eval_samples_per_second": 16.042,
-      "eval_steps_per_second": 2.007,
+      "epoch": 0.05332764505119454,
+      "eval_loss": 1.4427576065063477,
+      "eval_runtime": 60.328,
+      "eval_samples_per_second": 16.361,
+      "eval_steps_per_second": 2.055,
       "step": 500
     },
     {
-      "epoch": 0.18422991893883567,
-      "grad_norm": 76.37991333007812,
-      "learning_rate": 4.69295013510194e-08,
-      "loss": 0.4108,
+      "epoch": 0.10665529010238908,
+      "grad_norm": 70.38941955566406,
+      "learning_rate": 4.822241183162685e-08,
+      "loss": 0.4534,
       "step": 1000
     },
     {
-      "epoch": 0.18422991893883567,
-      "eval_loss": 1.3513683080673218,
-      "eval_runtime": 74.9686,
-      "eval_samples_per_second": 16.1,
-      "eval_steps_per_second": 2.014,
+      "epoch": 0.10665529010238908,
+      "eval_loss": 1.3823094367980957,
+      "eval_runtime": 61.4884,
+      "eval_samples_per_second": 16.052,
+      "eval_steps_per_second": 2.017,
       "step": 1000
     },
     {
-      "epoch": 0.2763448784082535,
-      "grad_norm": 0.5795987844467163,
-      "learning_rate": 4.539425202652911e-08,
-      "loss": 0.4335,
+      "epoch": 0.1599829351535836,
+      "grad_norm": 389.9967956542969,
+      "learning_rate": 4.733361774744027e-08,
+      "loss": 0.3753,
       "step": 1500
     },
     {
-      "epoch": 0.2763448784082535,
-      "eval_loss": 1.2895005941390991,
-      "eval_runtime": 76.713,
-      "eval_samples_per_second": 15.734,
-      "eval_steps_per_second": 1.968,
+      "epoch": 0.1599829351535836,
+      "eval_loss": 1.3328827619552612,
+      "eval_runtime": 63.0378,
+      "eval_samples_per_second": 15.657,
+      "eval_steps_per_second": 1.967,
       "step": 1500
     },
     {
-      "epoch": 0.36845983787767134,
-      "grad_norm": 0.1463615745306015,
-      "learning_rate": 4.385900270203881e-08,
-      "loss": 0.3436,
+      "epoch": 0.21331058020477817,
+      "grad_norm": 13.427445411682129,
+      "learning_rate": 4.6444823663253694e-08,
+      "loss": 0.3361,
       "step": 2000
     },
     {
-      "epoch": 0.36845983787767134,
-      "eval_loss": 1.2414860725402832,
-      "eval_runtime": 76.9749,
-      "eval_samples_per_second": 15.68,
+      "epoch": 0.21331058020477817,
+      "eval_loss": 1.2922736406326294,
+      "eval_runtime": 63.1958,
+      "eval_samples_per_second": 15.618,
       "eval_steps_per_second": 1.962,
       "step": 2000
     },
     {
-      "epoch": 0.46057479734708917,
-      "grad_norm": 323.8576354980469,
-      "learning_rate": 4.232375337754851e-08,
-      "loss": 0.366,
+      "epoch": 0.2666382252559727,
+      "grad_norm": 320.7197265625,
+      "learning_rate": 4.555602957906712e-08,
+      "loss": 0.3597,
       "step": 2500
     },
     {
-      "epoch": 0.46057479734708917,
-      "eval_loss": 1.1949334144592285,
-      "eval_runtime": 77.0381,
-      "eval_samples_per_second": 15.668,
-      "eval_steps_per_second": 1.96,
+      "epoch": 0.2666382252559727,
+      "eval_loss": 1.257211685180664,
+      "eval_runtime": 63.3241,
+      "eval_samples_per_second": 15.586,
+      "eval_steps_per_second": 1.958,
       "step": 2500
     },
     {
-      "epoch": 0.552689756816507,
-      "grad_norm": 339.28533935546875,
-      "learning_rate": 4.078850405305821e-08,
-      "loss": 0.3191,
+      "epoch": 0.3199658703071672,
+      "grad_norm": 0.0019184721168130636,
+      "learning_rate": 4.4667235494880546e-08,
+      "loss": 0.3042,
       "step": 3000
     },
     {
-      "epoch": 0.552689756816507,
-      "eval_loss": 1.1560280323028564,
-      "eval_runtime": 77.0026,
-      "eval_samples_per_second": 15.675,
-      "eval_steps_per_second": 1.961,
+      "epoch": 0.3199658703071672,
+      "eval_loss": 1.2240594625473022,
+      "eval_runtime": 63.5118,
+      "eval_samples_per_second": 15.54,
+      "eval_steps_per_second": 1.952,
       "step": 3000
     },
     {
-      "epoch": 0.6448047162859248,
-      "grad_norm": 60.27859878540039,
-      "learning_rate": 3.925325472856792e-08,
-      "loss": 0.2779,
+      "epoch": 0.37329351535836175,
+      "grad_norm": 11.738512992858887,
+      "learning_rate": 4.3778441410693965e-08,
+      "loss": 0.3252,
       "step": 3500
     },
     {
-      "epoch": 0.6448047162859248,
-      "eval_loss": 1.1233855485916138,
-      "eval_runtime": 76.9006,
-      "eval_samples_per_second": 15.696,
-      "eval_steps_per_second": 1.964,
+      "epoch": 0.37329351535836175,
+      "eval_loss": 1.1988952159881592,
+      "eval_runtime": 63.5384,
+      "eval_samples_per_second": 15.534,
+      "eval_steps_per_second": 1.952,
       "step": 3500
     },
     {
-      "epoch": 0.7369196757553427,
-      "grad_norm": 18.268455505371094,
-      "learning_rate": 3.7718005404077616e-08,
-      "loss": 0.2942,
+      "epoch": 0.42662116040955633,
+      "grad_norm": 5.839956283569336,
+      "learning_rate": 4.28896473265074e-08,
+      "loss": 0.2439,
       "step": 4000
     },
     {
-      "epoch": 0.7369196757553427,
-      "eval_loss": 1.0993207693099976,
-      "eval_runtime": 77.2834,
-      "eval_samples_per_second": 15.618,
-      "eval_steps_per_second": 1.954,
+      "epoch": 0.42662116040955633,
+      "eval_loss": 1.1730674505233765,
+      "eval_runtime": 62.3497,
+      "eval_samples_per_second": 15.83,
+      "eval_steps_per_second": 1.989,
       "step": 4000
     },
     {
-      "epoch": 0.8290346352247605,
-      "grad_norm": 505.22723388671875,
-      "learning_rate": 3.6182756079587324e-08,
-      "loss": 0.2615,
+      "epoch": 0.47994880546075086,
+      "grad_norm": 154.76502990722656,
+      "learning_rate": 4.2000853242320816e-08,
+      "loss": 0.3004,
       "step": 4500
     },
     {
-      "epoch": 0.8290346352247605,
-      "eval_loss": 1.080470085144043,
-      "eval_runtime": 77.1214,
-      "eval_samples_per_second": 15.651,
-      "eval_steps_per_second": 1.958,
+      "epoch": 0.47994880546075086,
+      "eval_loss": 1.1497682332992554,
+      "eval_runtime": 63.5491,
+      "eval_samples_per_second": 15.531,
+      "eval_steps_per_second": 1.951,
       "step": 4500
     },
     {
-      "epoch": 0.9211495946941783,
-      "grad_norm": 1.2971173524856567,
-      "learning_rate": 3.4647506755097025e-08,
-      "loss": 0.2715,
+      "epoch": 0.5332764505119454,
+      "grad_norm": 0.13150489330291748,
+      "learning_rate": 4.111205915813424e-08,
+      "loss": 0.3205,
       "step": 5000
     },
     {
-      "epoch": 0.9211495946941783,
-      "eval_loss": 1.0614728927612305,
-      "eval_runtime": 77.2982,
-      "eval_samples_per_second": 15.615,
-      "eval_steps_per_second": 1.953,
+      "epoch": 0.5332764505119454,
+      "eval_loss": 1.1258739233016968,
+      "eval_runtime": 62.1884,
+      "eval_samples_per_second": 15.871,
+      "eval_steps_per_second": 1.994,
       "step": 5000
     },
     {
-      "epoch": 1.013264554163596,
-      "grad_norm": 0.17611829936504364,
-      "learning_rate": 3.3112257430606727e-08,
-      "loss": 0.2509,
+      "epoch": 0.58660409556314,
+      "grad_norm": 0.3719326853752136,
+      "learning_rate": 4.022326507394767e-08,
+      "loss": 0.3091,
       "step": 5500
     },
     {
-      "epoch": 1.013264554163596,
-      "eval_loss": 1.0461839437484741,
-      "eval_runtime": 75.3494,
-      "eval_samples_per_second": 16.019,
-      "eval_steps_per_second": 2.004,
+      "epoch": 0.58660409556314,
+      "eval_loss": 1.1053005456924438,
+      "eval_runtime": 63.437,
+      "eval_samples_per_second": 15.559,
+      "eval_steps_per_second": 1.955,
       "step": 5500
     },
     {
-      "epoch": 1.105379513633014,
-      "grad_norm": 228.2562713623047,
-      "learning_rate": 3.1577008106116435e-08,
-      "loss": 0.2653,
+      "epoch": 0.6399317406143344,
+      "grad_norm": 170.89846801757812,
+      "learning_rate": 3.9334470989761093e-08,
+      "loss": 0.2657,
       "step": 6000
     },
     {
-      "epoch": 1.105379513633014,
-      "eval_loss": 1.0305675268173218,
-      "eval_runtime": 77.0542,
-      "eval_samples_per_second": 15.664,
-      "eval_steps_per_second": 1.96,
+      "epoch": 0.6399317406143344,
+      "eval_loss": 1.0857468843460083,
+      "eval_runtime": 62.4055,
+      "eval_samples_per_second": 15.816,
+      "eval_steps_per_second": 1.987,
       "step": 6000
     },
     {
-      "epoch": 1.1974944731024317,
-      "grad_norm": 432.4784240722656,
-      "learning_rate": 3.0041758781626136e-08,
-      "loss": 0.2199,
+      "epoch": 0.693259385665529,
+      "grad_norm": 323.0997619628906,
+      "learning_rate": 3.844567690557451e-08,
+      "loss": 0.2251,
       "step": 6500
     },
     {
-      "epoch": 1.1974944731024317,
-      "eval_loss": 1.0182703733444214,
-      "eval_runtime": 76.296,
-      "eval_samples_per_second": 15.82,
-      "eval_steps_per_second": 1.979,
+      "epoch": 0.693259385665529,
+      "eval_loss": 1.0695208311080933,
+      "eval_runtime": 63.1085,
+      "eval_samples_per_second": 15.64,
+      "eval_steps_per_second": 1.965,
       "step": 6500
     },
     {
-      "epoch": 1.2896094325718497,
-      "grad_norm": 0.6385967135429382,
-      "learning_rate": 2.8506509457135834e-08,
-      "loss": 0.1844,
+      "epoch": 0.7465870307167235,
+      "grad_norm": 0.13254186511039734,
+      "learning_rate": 3.7556882821387945e-08,
+      "loss": 0.248,
       "step": 7000
     },
     {
-      "epoch": 1.2896094325718497,
-      "eval_loss": 1.0078601837158203,
-      "eval_runtime": 76.5441,
-      "eval_samples_per_second": 15.769,
-      "eval_steps_per_second": 1.973,
+      "epoch": 0.7465870307167235,
+      "eval_loss": 1.0535719394683838,
+      "eval_runtime": 62.3504,
+      "eval_samples_per_second": 15.83,
+      "eval_steps_per_second": 1.989,
       "step": 7000
     },
     {
-      "epoch": 1.3817243920412676,
-      "grad_norm": 57.47743606567383,
-      "learning_rate": 2.697126013264554e-08,
-      "loss": 0.2143,
+      "epoch": 0.7999146757679181,
+      "grad_norm": 63.14700698852539,
+      "learning_rate": 3.6668088737201364e-08,
+      "loss": 0.2745,
       "step": 7500
     },
     {
-      "epoch": 1.3817243920412676,
-      "eval_loss": 0.9969784617424011,
-      "eval_runtime": 75.6186,
-      "eval_samples_per_second": 15.962,
-      "eval_steps_per_second": 1.997,
+      "epoch": 0.7999146757679181,
+      "eval_loss": 1.0388950109481812,
+      "eval_runtime": 62.2311,
+      "eval_samples_per_second": 15.86,
+      "eval_steps_per_second": 1.993,
       "step": 7500
     },
     {
-      "epoch": 1.4738393515106853,
-      "grad_norm": 0.11704438179731369,
-      "learning_rate": 2.5436010808155244e-08,
-      "loss": 0.2011,
+      "epoch": 0.8532423208191127,
+      "grad_norm": 623.6299438476562,
+      "learning_rate": 3.577929465301478e-08,
+      "loss": 0.2857,
       "step": 8000
     },
     {
-      "epoch": 1.4738393515106853,
-      "eval_loss": 0.9886976480484009,
-      "eval_runtime": 75.238,
-      "eval_samples_per_second": 16.042,
-      "eval_steps_per_second": 2.007,
+      "epoch": 0.8532423208191127,
+      "eval_loss": 1.0260424613952637,
+      "eval_runtime": 62.2792,
+      "eval_samples_per_second": 15.848,
+      "eval_steps_per_second": 1.991,
       "step": 8000
     }
   ],
   "logging_steps": 500,
-  "max_steps": 16284,
+  "max_steps": 28128,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,

checkpoint-8000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbcece3d46fd39b0ba80eb3e495fdf46c9f891af5b4bddce9de77ad5a467b625
+oid sha256:590a3eed3f4b2ba9fcb0b831d5e4c3f2410a46be8f57155ebf9fdd97c6c96524
 size 4847