Training in progress, step 2000, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a2971e3fa441c6195173934959b060bb37d1918f45fcb772cd683b88b14b691
-size 2751039840
+oid sha256:2e803bd49f3930468bb25dec6f3b145be3563fa0473cf89fe849fd156ba829d6
+size 2751040864

last-checkpoint/pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e997daa058d1d742d0043f65fc86671ea2099fdf09fb77765feeb444f4f447b
+oid sha256:b938fee26d72af327f488b7a6d3dd3965f17be62270e5fa901cc06715df86f83
 size 5000078781

last-checkpoint/pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0acbc24840ee77a560ac012b96b3ffc8f3e7d2bebdb9ceee0b3779a62612f5d
+oid sha256:207e1eb9852e0014a1e5967f14829387ae512bf72200a7bd8dbe6cd2b9744fa1
 size 482838574

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3300103406756fad9a1f1f8a21d36f1372b8c21da5142037d90f074362090e43
+oid sha256:cfe664d6ceb782278e808b1191d468184a14f4c1487f92773562a2a538616d86
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e18c3ed3aeaa20b55f954231cfa9b3d71b919b964de3ed9a5d9be2d054fa1c6
+oid sha256:0483b2fb5f8063a86ad62f9b7bae0bbe612492a917afee7ab754e22d70616737
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4362771511077881,
-  "best_model_checkpoint": "../KevinKibe/nllb-200-distilled-1.3B-finetuned-finetuned-finetuned-finetuned/checkpoint-200",
-  "epoch": 200.0,
+  "best_metric": 0.3573495149612427,
+  "best_model_checkpoint": "../KevinKibe/nllb-200-distilled-1.3B-finetuned-finetuned-finetuned-finetuned/checkpoint-400",
+  "epoch": 2000.0,
   "eval_steps": 200,
-  "global_step": 200,
+  "global_step": 2000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -24,6 +24,159 @@
       "eval_samples_per_second": 1.065,
       "eval_steps_per_second": 0.533,
       "step": 200
+    },
+    {
+      "epoch": 400.0,
+      "grad_norm": 0.0009787885937839746,
+      "learning_rate": 4.524848217064997e-05,
+      "loss": 0.0003,
+      "step": 400
+    },
+    {
+      "epoch": 400.0,
+      "eval_gen_len": 22.5,
+      "eval_loss": 0.3573495149612427,
+      "eval_rouge": 0.2713,
+      "eval_runtime": 1.0507,
+      "eval_samples_per_second": 1.903,
+      "eval_steps_per_second": 0.952,
+      "step": 400
+    },
+    {
+      "epoch": 600.0,
+      "grad_norm": 0.00037960291956551373,
+      "learning_rate": 3.972638318855291e-05,
+      "loss": 0.0,
+      "step": 600
+    },
+    {
+      "epoch": 600.0,
+      "eval_gen_len": 19.5,
+      "eval_loss": 0.37564775347709656,
+      "eval_rouge": 0.2762,
+      "eval_runtime": 0.9,
+      "eval_samples_per_second": 2.222,
+      "eval_steps_per_second": 1.111,
+      "step": 600
+    },
+    {
+      "epoch": 800.0,
+      "grad_norm": 0.00027565439813770354,
+      "learning_rate": 3.2762763215215306e-05,
+      "loss": 0.0,
+      "step": 800
+    },
+    {
+      "epoch": 800.0,
+      "eval_gen_len": 19.5,
+      "eval_loss": 0.3804549276828766,
+      "eval_rouge": 0.2762,
+      "eval_runtime": 0.8354,
+      "eval_samples_per_second": 2.394,
+      "eval_steps_per_second": 1.197,
+      "step": 800
+    },
+    {
+      "epoch": 1000.0,
+      "grad_norm": 0.00021920226572547108,
+      "learning_rate": 2.5039269892020772e-05,
+      "loss": 0.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1000.0,
+      "eval_gen_len": 19.5,
+      "eval_loss": 0.38089361786842346,
+      "eval_rouge": 0.2762,
+      "eval_runtime": 0.9583,
+      "eval_samples_per_second": 2.087,
+      "eval_steps_per_second": 1.044,
+      "step": 1000
+    },
+    {
+      "epoch": 1200.0,
+      "grad_norm": 0.00018822183483280241,
+      "learning_rate": 1.731193255818582e-05,
+      "loss": 0.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1200.0,
+      "eval_gen_len": 19.5,
+      "eval_loss": 0.38475799560546875,
+      "eval_rouge": 0.2762,
+      "eval_runtime": 0.8748,
+      "eval_samples_per_second": 2.286,
+      "eval_steps_per_second": 1.143,
+      "step": 1200
+    },
+    {
+      "epoch": 1400.0,
+      "grad_norm": 0.00018423172878101468,
+      "learning_rate": 1.0337156831471246e-05,
+      "loss": 0.0,
+      "step": 1400
+    },
+    {
+      "epoch": 1400.0,
+      "eval_gen_len": 22.5,
+      "eval_loss": 0.3964395225048065,
+      "eval_rouge": 0.2713,
+      "eval_runtime": 0.8684,
+      "eval_samples_per_second": 2.303,
+      "eval_steps_per_second": 1.152,
+      "step": 1400
+    },
+    {
+      "epoch": 1600.0,
+      "grad_norm": 0.00017020950326696038,
+      "learning_rate": 4.7976823561278865e-06,
+      "loss": 0.0,
+      "step": 1600
+    },
+    {
+      "epoch": 1600.0,
+      "eval_gen_len": 22.5,
+      "eval_loss": 0.39614760875701904,
+      "eval_rouge": 0.2713,
+      "eval_runtime": 1.0804,
+      "eval_samples_per_second": 1.851,
+      "eval_steps_per_second": 0.926,
+      "step": 1600
+    },
+    {
+      "epoch": 1800.0,
+      "grad_norm": 0.00016995037731248885,
+      "learning_rate": 1.2357514895905003e-06,
+      "loss": 0.0,
+      "step": 1800
+    },
+    {
+      "epoch": 1800.0,
+      "eval_gen_len": 22.5,
+      "eval_loss": 0.3964083194732666,
+      "eval_rouge": 0.2713,
+      "eval_runtime": 0.9109,
+      "eval_samples_per_second": 2.196,
+      "eval_steps_per_second": 1.098,
+      "step": 1800
+    },
+    {
+      "epoch": 2000.0,
+      "grad_norm": 0.00016411281831096858,
+      "learning_rate": 3.0842507411921185e-11,
+      "loss": 0.0,
+      "step": 2000
+    },
+    {
+      "epoch": 2000.0,
+      "eval_gen_len": 22.5,
+      "eval_loss": 0.3965863287448883,
+      "eval_rouge": 0.2713,
+      "eval_runtime": 0.8715,
+      "eval_samples_per_second": 2.295,
+      "eval_steps_per_second": 1.147,
+      "step": 2000
     }
   ],
   "logging_steps": 200,
@@ -31,7 +184,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2000,
   "save_steps": 200,
-  "total_flos": 500057466470400.0,
+  "total_flos": 5000574664704000.0,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null