Model save

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.8595
+- Loss: 1.7616
 
 ## Model description
 
@@ -48,22 +48,22 @@ The following hyperparameters were used during training:
 - gradient_accumulation_steps: 2
 - total_train_batch_size: 256
 - total_eval_batch_size: 128
-- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 1.9457        | 0.9927 | 68   | 1.8595          |
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.3703        | 1.0   | 137  | 1.7616          |
 
 
 ### Framework versions
 
 - PEFT 0.13.1.dev0
-- Transformers 4.46.2
-- Pytorch 2.5.1+cu124
+- Transformers 4.46.3
+- Pytorch 2.3.1+cu121
 - Datasets 3.1.0
 - Tokenizers 0.20.3

all_results.json

@@ -1,14 +1,9 @@
 {
-    "epoch": 0.9927007299270073,
-    "eval_loss": 1.859512209892273,
-    "eval_runtime": 1.6224,
-    "eval_samples": 518,
-    "eval_samples_per_second": 107.251,
-    "eval_steps_per_second": 1.233,
-    "total_flos": 4.015265797185208e+17,
-    "train_loss": 2.0918032141292797,
-    "train_runtime": 338.6733,
-    "train_samples": 51241,
-    "train_samples_per_second": 51.761,
-    "train_steps_per_second": 0.201
+    "epoch": 1.0,
+    "total_flos": 8.089579620799611e+17,
+    "train_loss": 1.4743173070197557,
+    "train_runtime": 700.8271,
+    "train_samples": 116368,
+    "train_samples_per_second": 49.884,
+    "train_steps_per_second": 0.195
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9927007299270073,
-    "total_flos": 4.015265797185208e+17,
-    "train_loss": 2.0918032141292797,
-    "train_runtime": 338.6733,
-    "train_samples": 51241,
-    "train_samples_per_second": 51.761,
-    "train_steps_per_second": 0.201
+    "epoch": 1.0,
+    "total_flos": 8.089579620799611e+17,
+    "train_loss": 1.4743173070197557,
+    "train_runtime": 700.8271,
+    "train_samples": 116368,
+    "train_samples_per_second": 49.884,
+    "train_steps_per_second": 0.195
 }

trainer_state.json

@@ -1,131 +1,229 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9927007299270073,
+  "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 68,
+  "global_step": 137,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.014598540145985401,
-      "grad_norm": 2.3731939792633057,
-      "learning_rate": 2.857142857142857e-05,
-      "loss": 2.5137,
+      "epoch": 0.0072992700729927005,
+      "grad_norm": 0.6737354397773743,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 1.9885,
       "step": 1
     },
+    {
+      "epoch": 0.0364963503649635,
+      "grad_norm": 0.5888584852218628,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 1.96,
+      "step": 5
+    },
     {
       "epoch": 0.072992700729927,
-      "grad_norm": 2.052441358566284,
+      "grad_norm": 0.44593068957328796,
       "learning_rate": 0.00014285714285714287,
-      "loss": 2.5009,
-      "step": 5
+      "loss": 1.927,
+      "step": 10
+    },
+    {
+      "epoch": 0.10948905109489052,
+      "grad_norm": 0.5411117076873779,
+      "learning_rate": 0.00019996738360808565,
+      "loss": 1.8403,
+      "step": 15
     },
     {
       "epoch": 0.145985401459854,
-      "grad_norm": 0.4984005093574524,
-      "learning_rate": 0.00019880878960910772,
-      "loss": 2.3982,
-      "step": 10
+      "grad_norm": 0.4955069422721863,
+      "learning_rate": 0.00019882804237803488,
+      "loss": 1.6916,
+      "step": 20
+    },
+    {
+      "epoch": 0.18248175182481752,
+      "grad_norm": 0.51289963722229,
+      "learning_rate": 0.00019607909582962477,
+      "loss": 1.5926,
+      "step": 25
     },
     {
       "epoch": 0.21897810218978103,
-      "grad_norm": 0.5986955165863037,
-      "learning_rate": 0.0001916316904487005,
-      "loss": 2.2688,
-      "step": 15
+      "grad_norm": 0.2931584417819977,
+      "learning_rate": 0.0001917653158603628,
+      "loss": 1.5275,
+      "step": 30
+    },
+    {
+      "epoch": 0.25547445255474455,
+      "grad_norm": 0.20454026758670807,
+      "learning_rate": 0.00018595696069872013,
+      "loss": 1.4805,
+      "step": 35
     },
     {
       "epoch": 0.291970802919708,
-      "grad_norm": 0.4892929494380951,
-      "learning_rate": 0.00017841198065767107,
-      "loss": 2.1764,
-      "step": 20
+      "grad_norm": 0.15512123703956604,
+      "learning_rate": 0.00017874863061334657,
+      "loss": 1.4545,
+      "step": 40
+    },
+    {
+      "epoch": 0.3284671532846715,
+      "grad_norm": 0.14049112796783447,
+      "learning_rate": 0.00017025772716520323,
+      "loss": 1.4489,
+      "step": 45
     },
     {
       "epoch": 0.36496350364963503,
-      "grad_norm": 0.46845513582229614,
-      "learning_rate": 0.00016002142805483685,
-      "loss": 2.0874,
-      "step": 25
+      "grad_norm": 0.12365967035293579,
+      "learning_rate": 0.0001606225410966638,
+      "loss": 1.4337,
+      "step": 50
+    },
+    {
+      "epoch": 0.40145985401459855,
+      "grad_norm": 0.12197204679250717,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 1.4202,
+      "step": 55
     },
     {
       "epoch": 0.43795620437956206,
-      "grad_norm": 0.34990543127059937,
-      "learning_rate": 0.00013767278936351854,
-      "loss": 2.0344,
-      "step": 30
+      "grad_norm": 0.1118067055940628,
+      "learning_rate": 0.0001385631124488136,
+      "loss": 1.4099,
+      "step": 60
+    },
+    {
+      "epoch": 0.4744525547445255,
+      "grad_norm": 0.10998713970184326,
+      "learning_rate": 0.0001264981502196662,
+      "loss": 1.3993,
+      "step": 65
     },
     {
       "epoch": 0.5109489051094891,
-      "grad_norm": 0.26925715804100037,
-      "learning_rate": 0.00011283983551465511,
-      "loss": 1.9991,
-      "step": 35
+      "grad_norm": 0.11097019165754318,
+      "learning_rate": 0.00011400161449686293,
+      "loss": 1.3987,
+      "step": 70
+    },
+    {
+      "epoch": 0.5474452554744526,
+      "grad_norm": 0.1111425831913948,
+      "learning_rate": 0.00010127703547159739,
+      "loss": 1.3845,
+      "step": 75
     },
     {
       "epoch": 0.583941605839416,
-      "grad_norm": 0.24737241864204407,
-      "learning_rate": 8.71601644853449e-05,
-      "loss": 1.9808,
-      "step": 40
+      "grad_norm": 0.11766236275434494,
+      "learning_rate": 8.853165746015997e-05,
+      "loss": 1.381,
+      "step": 80
+    },
+    {
+      "epoch": 0.6204379562043796,
+      "grad_norm": 0.11702164262533188,
+      "learning_rate": 7.597306353045393e-05,
+      "loss": 1.3715,
+      "step": 85
     },
     {
       "epoch": 0.656934306569343,
-      "grad_norm": 0.24646887183189392,
-      "learning_rate": 6.232721063648148e-05,
-      "loss": 1.9615,
-      "step": 45
+      "grad_norm": 0.1169838011264801,
+      "learning_rate": 6.380579461128819e-05,
+      "loss": 1.3859,
+      "step": 90
+    },
+    {
+      "epoch": 0.6934306569343066,
+      "grad_norm": 0.10516153275966644,
+      "learning_rate": 5.222801814877369e-05,
+      "loss": 1.368,
+      "step": 95
     },
     {
       "epoch": 0.7299270072992701,
-      "grad_norm": 0.2257496565580368,
-      "learning_rate": 3.997857194516319e-05,
-      "loss": 1.9739,
-      "step": 50
+      "grad_norm": 0.11746218055486679,
+      "learning_rate": 4.142830056718052e-05,
+      "loss": 1.3735,
+      "step": 100
+    },
+    {
+      "epoch": 0.7664233576642335,
+      "grad_norm": 0.11317908763885498,
+      "learning_rate": 3.158253610095697e-05,
+      "loss": 1.3675,
+      "step": 105
     },
     {
       "epoch": 0.8029197080291971,
-      "grad_norm": 0.20391573011875153,
-      "learning_rate": 2.1588019342328968e-05,
-      "loss": 1.9468,
-      "step": 55
+      "grad_norm": 0.1135973334312439,
+      "learning_rate": 2.2851082017805703e-05,
+      "loss": 1.3633,
+      "step": 110
+    },
+    {
+      "epoch": 0.8394160583941606,
+      "grad_norm": 0.11384975910186768,
+      "learning_rate": 1.5376146891235598e-05,
+      "loss": 1.3684,
+      "step": 115
     },
     {
       "epoch": 0.8759124087591241,
-      "grad_norm": 0.19893679022789001,
-      "learning_rate": 8.368309551299536e-06,
-      "loss": 1.9727,
-      "step": 60
+      "grad_norm": 0.10755354166030884,
+      "learning_rate": 9.279474459608805e-06,
+      "loss": 1.3735,
+      "step": 120
+    },
+    {
+      "epoch": 0.9124087591240876,
+      "grad_norm": 0.11003435403108597,
+      "learning_rate": 4.660360794506946e-06,
+      "loss": 1.3648,
+      "step": 125
     },
     {
       "epoch": 0.948905109489051,
-      "grad_norm": 0.19601836800575256,
-      "learning_rate": 1.1912103908922945e-06,
-      "loss": 1.9457,
-      "step": 65
+      "grad_norm": 0.10320563614368439,
+      "learning_rate": 1.5940370726542863e-06,
+      "loss": 1.3714,
+      "step": 130
+    },
+    {
+      "epoch": 0.9854014598540146,
+      "grad_norm": 0.10423589497804642,
+      "learning_rate": 1.3044429107700318e-07,
+      "loss": 1.3703,
+      "step": 135
     },
     {
-      "epoch": 0.9927007299270073,
-      "eval_loss": 1.859512209892273,
-      "eval_runtime": 1.6081,
-      "eval_samples_per_second": 108.202,
-      "eval_steps_per_second": 1.244,
-      "step": 68
+      "epoch": 1.0,
+      "eval_loss": 1.7615731954574585,
+      "eval_runtime": 0.8323,
+      "eval_samples_per_second": 10.813,
+      "eval_steps_per_second": 1.201,
+      "step": 137
     },
     {
-      "epoch": 0.9927007299270073,
-      "step": 68,
-      "total_flos": 4.015265797185208e+17,
-      "train_loss": 2.0918032141292797,
-      "train_runtime": 338.6733,
-      "train_samples_per_second": 51.761,
-      "train_steps_per_second": 0.201
+      "epoch": 1.0,
+      "step": 137,
+      "total_flos": 8.089579620799611e+17,
+      "train_loss": 1.4743173070197557,
+      "train_runtime": 700.8271,
+      "train_samples_per_second": 49.884,
+      "train_steps_per_second": 0.195
     }
   ],
   "logging_steps": 5,
-  "max_steps": 68,
+  "max_steps": 137,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -141,7 +239,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.015265797185208e+17,
+  "total_flos": 8.089579620799611e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null