End of training

README.md

@@ -1,17 +1,21 @@
 ---
 base_model: meta-llama/Llama-3.2-1B
+datasets: hanseungwook/wiki-sports-knowledge-qa
 library_name: transformers
-model_name: Llama-3.2-1B-sportsqa-V1
+model_name: meta-llama/Llama-3.2-1B
 tags:
 - generated_from_trainer
+- question-answering
+- QA
+- text-generation
 - trl
 - sft
 licence: license
 ---
 
-# Model Card for Llama-3.2-1B-sportsqa-V1
+# Model Card for meta-llama/Llama-3.2-1B
 
-This model is a fine-tuned version of [meta-llama/Llama-3.2-1B](https://huggingface.co/meta-llama/Llama-3.2-1B).
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B](https://huggingface.co/meta-llama/Llama-3.2-1B) on the [hanseungwook/wiki-sports-knowledge-qa](https://huggingface.co/datasets/hanseungwook/wiki-sports-knowledge-qa) dataset.
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,8 +1,12 @@
 {
-    "epoch": 6.889589905362776,
-    "total_flos": 1.6473245221453824e+16,
-    "train_loss": 0.8712233146031697,
-    "train_runtime": 1146.6252,
-    "train_samples_per_second": 30.921,
-    "train_steps_per_second": 0.159
+    "epoch": 4.971608832807571,
+    "eval_loss": 0.8520277738571167,
+    "eval_runtime": 6.5794,
+    "eval_samples_per_second": 85.57,
+    "eval_steps_per_second": 10.791,
+    "total_flos": 1.1913588129595392e+16,
+    "train_loss": 0.9871828225942758,
+    "train_runtime": 692.3499,
+    "train_samples_per_second": 36.578,
+    "train_steps_per_second": 0.188
 }

eval_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 4.971608832807571,
+    "eval_loss": 0.8520277738571167,
+    "eval_runtime": 6.5794,
+    "eval_samples_per_second": 85.57,
+    "eval_steps_per_second": 10.791
+}

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 6.889589905362776,
-    "total_flos": 1.6473245221453824e+16,
-    "train_loss": 0.8712233146031697,
-    "train_runtime": 1146.6252,
-    "train_samples_per_second": 30.921,
-    "train_steps_per_second": 0.159
+    "epoch": 4.971608832807571,
+    "total_flos": 1.1913588129595392e+16,
+    "train_loss": 0.9871828225942758,
+    "train_runtime": 692.3499,
+    "train_samples_per_second": 36.578,
+    "train_steps_per_second": 0.188
 }

trainer_state.json

@@ -1,282 +1,222 @@
 {
-  "best_metric": 0.8432615399360657,
-  "best_model_checkpoint": "Llama-3.2-1B-sportsqa-V1/checkpoint-150",
-  "epoch": 6.511041009463723,
+  "best_metric": 0.8520275354385376,
+  "best_model_checkpoint": "Llama-3.2-1B-sportsqa-V1/checkpoint-130",
+  "epoch": 4.971608832807571,
   "eval_steps": 10,
-  "global_step": 170,
+  "global_step": 130,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.3785488958990536,
-      "grad_norm": 0.9724205136299133,
+      "grad_norm": 0.5394421219825745,
       "learning_rate": 0.0002,
-      "loss": 1.4368,
+      "loss": 1.8051,
       "step": 10
     },
     {
       "epoch": 0.3785488958990536,
-      "eval_loss": 1.348753809928894,
-      "eval_runtime": 6.5589,
-      "eval_samples_per_second": 85.838,
-      "eval_steps_per_second": 10.825,
+      "eval_loss": 1.5353755950927734,
+      "eval_runtime": 6.6028,
+      "eval_samples_per_second": 85.267,
+      "eval_steps_per_second": 10.753,
       "step": 10
     },
     {
       "epoch": 0.7570977917981072,
-      "grad_norm": 0.5118973255157471,
+      "grad_norm": 0.6756414771080017,
       "learning_rate": 0.0002,
-      "loss": 1.2118,
+      "loss": 1.4193,
       "step": 20
     },
     {
       "epoch": 0.7570977917981072,
-      "eval_loss": 1.1572917699813843,
-      "eval_runtime": 6.43,
-      "eval_samples_per_second": 87.558,
-      "eval_steps_per_second": 11.042,
+      "eval_loss": 1.3329508304595947,
+      "eval_runtime": 6.4761,
+      "eval_samples_per_second": 86.935,
+      "eval_steps_per_second": 10.963,
       "step": 20
     },
     {
       "epoch": 1.1482649842271293,
-      "grad_norm": 0.41862204670906067,
+      "grad_norm": 0.4857617914676666,
       "learning_rate": 0.0002,
-      "loss": 1.1684,
+      "loss": 1.2895,
       "step": 30
     },
     {
       "epoch": 1.1482649842271293,
-      "eval_loss": 1.0873514413833618,
-      "eval_runtime": 6.4834,
-      "eval_samples_per_second": 86.838,
-      "eval_steps_per_second": 10.951,
+      "eval_loss": 1.1635785102844238,
+      "eval_runtime": 6.4995,
+      "eval_samples_per_second": 86.622,
+      "eval_steps_per_second": 10.924,
       "step": 30
     },
     {
       "epoch": 1.526813880126183,
-      "grad_norm": 0.42144763469696045,
+      "grad_norm": 0.43100810050964355,
       "learning_rate": 0.0002,
-      "loss": 0.9907,
+      "loss": 1.0738,
       "step": 40
     },
     {
       "epoch": 1.526813880126183,
-      "eval_loss": 1.0390795469284058,
-      "eval_runtime": 6.4784,
-      "eval_samples_per_second": 86.904,
-      "eval_steps_per_second": 10.959,
+      "eval_loss": 1.0900534391403198,
+      "eval_runtime": 6.4609,
+      "eval_samples_per_second": 87.14,
+      "eval_steps_per_second": 10.989,
       "step": 40
     },
     {
       "epoch": 1.9053627760252367,
-      "grad_norm": 0.6625965237617493,
+      "grad_norm": 0.661459743976593,
       "learning_rate": 0.0002,
-      "loss": 0.8936,
+      "loss": 0.9551,
       "step": 50
     },
     {
       "epoch": 1.9053627760252367,
-      "eval_loss": 0.9832194447517395,
-      "eval_runtime": 6.4575,
-      "eval_samples_per_second": 87.185,
-      "eval_steps_per_second": 10.995,
+      "eval_loss": 1.032881498336792,
+      "eval_runtime": 6.3928,
+      "eval_samples_per_second": 88.067,
+      "eval_steps_per_second": 11.106,
       "step": 50
     },
     {
       "epoch": 2.2965299684542586,
-      "grad_norm": 0.5491335391998291,
+      "grad_norm": 0.41446322202682495,
       "learning_rate": 0.0002,
-      "loss": 0.9537,
+      "loss": 1.0296,
       "step": 60
     },
     {
       "epoch": 2.2965299684542586,
-      "eval_loss": 0.9610708355903625,
-      "eval_runtime": 6.4935,
-      "eval_samples_per_second": 86.702,
-      "eval_steps_per_second": 10.934,
+      "eval_loss": 0.9979809522628784,
+      "eval_runtime": 6.4989,
+      "eval_samples_per_second": 86.63,
+      "eval_steps_per_second": 10.925,
       "step": 60
     },
     {
       "epoch": 2.6750788643533125,
-      "grad_norm": 0.6222676038742065,
+      "grad_norm": 0.6213859915733337,
       "learning_rate": 0.0002,
-      "loss": 0.7753,
+      "loss": 0.8314,
       "step": 70
     },
     {
       "epoch": 2.6750788643533125,
-      "eval_loss": 0.9286745190620422,
-      "eval_runtime": 6.5271,
-      "eval_samples_per_second": 86.256,
-      "eval_steps_per_second": 10.878,
+      "eval_loss": 0.9495619535446167,
+      "eval_runtime": 6.4778,
+      "eval_samples_per_second": 86.912,
+      "eval_steps_per_second": 10.961,
       "step": 70
     },
     {
       "epoch": 3.0662460567823344,
-      "grad_norm": 0.47406917810440063,
+      "grad_norm": 0.5072933435440063,
       "learning_rate": 0.0002,
-      "loss": 0.8796,
+      "loss": 0.9114,
       "step": 80
     },
     {
       "epoch": 3.0662460567823344,
-      "eval_loss": 0.9002482891082764,
-      "eval_runtime": 6.5139,
-      "eval_samples_per_second": 86.43,
-      "eval_steps_per_second": 10.9,
+      "eval_loss": 0.9263410568237305,
+      "eval_runtime": 6.4355,
+      "eval_samples_per_second": 87.484,
+      "eval_steps_per_second": 11.033,
       "step": 80
     },
     {
       "epoch": 3.444794952681388,
-      "grad_norm": 0.54665207862854,
+      "grad_norm": 0.5831320881843567,
       "learning_rate": 0.0002,
-      "loss": 0.6697,
+      "loss": 0.7374,
       "step": 90
     },
     {
       "epoch": 3.444794952681388,
-      "eval_loss": 0.8929345607757568,
-      "eval_runtime": 6.349,
-      "eval_samples_per_second": 88.676,
-      "eval_steps_per_second": 11.183,
+      "eval_loss": 0.9075612425804138,
+      "eval_runtime": 6.4648,
+      "eval_samples_per_second": 87.087,
+      "eval_steps_per_second": 10.983,
       "step": 90
     },
     {
       "epoch": 3.823343848580442,
-      "grad_norm": 0.47858959436416626,
+      "grad_norm": 0.5210297107696533,
       "learning_rate": 0.0002,
-      "loss": 0.7382,
+      "loss": 0.7606,
       "step": 100
     },
     {
       "epoch": 3.823343848580442,
-      "eval_loss": 0.8687711358070374,
-      "eval_runtime": 6.4617,
-      "eval_samples_per_second": 87.129,
-      "eval_steps_per_second": 10.988,
+      "eval_loss": 0.8877434730529785,
+      "eval_runtime": 6.4394,
+      "eval_samples_per_second": 87.43,
+      "eval_steps_per_second": 11.026,
       "step": 100
     },
     {
       "epoch": 4.214511041009464,
-      "grad_norm": 0.6165306568145752,
+      "grad_norm": 0.7101346254348755,
       "learning_rate": 0.0002,
-      "loss": 0.6795,
+      "loss": 0.7158,
       "step": 110
     },
     {
       "epoch": 4.214511041009464,
-      "eval_loss": 0.8820136189460754,
-      "eval_runtime": 6.4456,
-      "eval_samples_per_second": 87.346,
-      "eval_steps_per_second": 11.015,
+      "eval_loss": 0.8949338793754578,
+      "eval_runtime": 6.4802,
+      "eval_samples_per_second": 86.88,
+      "eval_steps_per_second": 10.956,
       "step": 110
     },
     {
       "epoch": 4.593059936908517,
-      "grad_norm": 0.5470077395439148,
+      "grad_norm": 0.4959416091442108,
       "learning_rate": 0.0002,
-      "loss": 0.646,
+      "loss": 0.6792,
       "step": 120
     },
     {
       "epoch": 4.593059936908517,
-      "eval_loss": 0.8543145060539246,
-      "eval_runtime": 6.4186,
-      "eval_samples_per_second": 87.713,
-      "eval_steps_per_second": 11.062,
+      "eval_loss": 0.8711220622062683,
+      "eval_runtime": 6.5288,
+      "eval_samples_per_second": 86.233,
+      "eval_steps_per_second": 10.875,
       "step": 120
     },
     {
       "epoch": 4.971608832807571,
-      "grad_norm": 0.5192540287971497,
+      "grad_norm": 0.4901735484600067,
       "learning_rate": 0.0002,
-      "loss": 0.5972,
+      "loss": 0.6251,
       "step": 130
     },
     {
       "epoch": 4.971608832807571,
-      "eval_loss": 0.8454616069793701,
-      "eval_runtime": 6.3674,
-      "eval_samples_per_second": 88.419,
-      "eval_steps_per_second": 11.151,
+      "eval_loss": 0.8520275354385376,
+      "eval_runtime": 6.5294,
+      "eval_samples_per_second": 86.226,
+      "eval_steps_per_second": 10.874,
       "step": 130
     },
     {
-      "epoch": 5.3627760252365935,
-      "grad_norm": 0.5664217472076416,
-      "learning_rate": 0.0002,
-      "loss": 0.6179,
-      "step": 140
-    },
-    {
-      "epoch": 5.3627760252365935,
-      "eval_loss": 0.845314085483551,
-      "eval_runtime": 6.4608,
-      "eval_samples_per_second": 87.141,
-      "eval_steps_per_second": 10.989,
-      "step": 140
-    },
-    {
-      "epoch": 5.7413249211356465,
-      "grad_norm": 0.5141603350639343,
-      "learning_rate": 0.0002,
-      "loss": 0.5488,
-      "step": 150
-    },
-    {
-      "epoch": 5.7413249211356465,
-      "eval_loss": 0.8432615399360657,
-      "eval_runtime": 6.4351,
-      "eval_samples_per_second": 87.489,
-      "eval_steps_per_second": 11.033,
-      "step": 150
-    },
-    {
-      "epoch": 6.132492113564669,
-      "grad_norm": 0.4900747835636139,
-      "learning_rate": 0.0002,
-      "loss": 0.5574,
-      "step": 160
-    },
-    {
-      "epoch": 6.132492113564669,
-      "eval_loss": 0.8658251166343689,
-      "eval_runtime": 6.4211,
-      "eval_samples_per_second": 87.679,
-      "eval_steps_per_second": 11.057,
-      "step": 160
-    },
-    {
-      "epoch": 6.511041009463723,
-      "grad_norm": 0.5367661118507385,
-      "learning_rate": 0.0002,
-      "loss": 0.4778,
-      "step": 170
-    },
-    {
-      "epoch": 6.511041009463723,
-      "eval_loss": 0.8633659482002258,
-      "eval_runtime": 6.4456,
-      "eval_samples_per_second": 87.346,
-      "eval_steps_per_second": 11.015,
-      "step": 170
-    },
-    {
-      "epoch": 6.511041009463723,
-      "step": 170,
-      "total_flos": 1.5618225878728704e+16,
-      "train_loss": 0.8142486740561092,
-      "train_runtime": 890.4577,
-      "train_samples_per_second": 39.817,
-      "train_steps_per_second": 0.204
+      "epoch": 4.971608832807571,
+      "step": 130,
+      "total_flos": 1.1913588129595392e+16,
+      "train_loss": 0.9871828225942758,
+      "train_runtime": 692.3499,
+      "train_samples_per_second": 36.578,
+      "train_steps_per_second": 0.188
     }
   ],
   "logging_steps": 10,
-  "max_steps": 182,
+  "max_steps": 130,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 5,
   "save_steps": 10,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -285,7 +225,7 @@
         "early_stopping_threshold": 0.01
       },
       "attributes": {
-        "early_stopping_patience_counter": 5
+        "early_stopping_patience_counter": 0
       }
     },
     "TrainerControl": {
@@ -299,7 +239,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.5618225878728704e+16,
+  "total_flos": 1.1913588129595392e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null