update model checkpoint

README.md

@@ -1,14 +1,11 @@
 ---
-license: gemma
 library_name: peft
 tags:
 - alignment-handbook
-- trl
-- sft
 - generated_from_trainer
-base_model: google/gemma-7b
 datasets:
 - llama-duo/synth_summarize_dataset_dedup
+base_model: google/gemma-7b
 model-index:
 - name: gemma7b-summarize-gpt4o-4k
   results: []
@@ -21,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the llama-duo/synth_summarize_dataset_dedup dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.0658
+- Loss: 6.0322
 
 ## Model description
 
@@ -42,13 +39,13 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 4
-- eval_batch_size: 4
+- eval_batch_size: 2
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 2
+- num_devices: 8
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 16
-- total_eval_batch_size: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -56,24 +53,24 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 16.725        | 0.9804 | 25   | 6.5520          |
-| 1.5122        | 2.0    | 51   | 2.6626          |
-| 1.1154        | 2.9804 | 76   | 2.5917          |
-| 0.9204        | 4.0    | 102  | 2.6570          |
-| 0.779         | 4.9804 | 127  | 2.7498          |
-| 0.6207        | 6.0    | 153  | 2.9976          |
-| 0.4762        | 6.9804 | 178  | 3.4668          |
-| 0.3908        | 8.0    | 204  | 3.8246          |
-| 0.3418        | 8.9804 | 229  | 4.0561          |
-| 0.3252        | 9.8039 | 250  | 4.0658          |
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 44.3098       | 1.0   | 7    | 13.7204         |
+| 25.7366       | 2.0   | 14   | 8.6916          |
+| 19.0375       | 3.0   | 21   | 7.6308          |
+| 18.2973       | 4.0   | 28   | 7.1198          |
+| 14.8387       | 5.0   | 35   | 6.8470          |
+| 12.5684       | 6.0   | 42   | 6.8495          |
+| 9.6308        | 7.0   | 49   | 6.6799          |
+| 5.7187        | 8.0   | 56   | 6.0818          |
+| 4.8487        | 9.0   | 63   | 6.0318          |
+| 4.4303        | 10.0  | 70   | 6.0322          |
 
 
 ### Framework versions
 
 - PEFT 0.10.0
 - Transformers 4.40.0
-- Pytorch 2.2.1+cu121
+- Pytorch 2.1.2+cu121
 - Datasets 2.18.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -21,12 +21,7 @@
   "revision": null,
   "target_modules": [
     "v_proj",
-    "up_proj",
-    "down_proj",
-    "q_proj",
-    "k_proj",
-    "gate_proj",
-    "o_proj"
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d0324a90521c27231d849c8c00f3dca466edadcf64bb738c9a3abae4b6ef709
-size 50056096
+oid sha256:e8bd0611a140bb49692da9a4e4e6c94e205e6baa337c23ddd71a8b04006987ac
+size 6437384

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 9.803921568627452,
-    "eval_loss": 4.065803050994873,
-    "eval_runtime": 2.0617,
+    "epoch": 10.0,
+    "eval_loss": 6.032225608825684,
+    "eval_runtime": 0.2331,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.85,
-    "eval_steps_per_second": 0.97,
-    "total_flos": 1.9110914639160934e+17,
-    "train_loss": 3.4213723726272582,
-    "train_runtime": 1646.3335,
-    "train_samples": 3749,
-    "train_samples_per_second": 2.448,
-    "train_steps_per_second": 0.152
+    "eval_samples_per_second": 42.9,
+    "eval_steps_per_second": 4.29,
+    "total_flos": 2.1344245100262195e+17,
+    "train_loss": 14.937398610796247,
+    "train_runtime": 171.2768,
+    "train_samples": 4038,
+    "train_samples_per_second": 25.573,
+    "train_steps_per_second": 0.409
 }

config.json

@@ -23,9 +23,9 @@
     "_load_in_4bit": true,
     "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
+    "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
     "llm_int8_has_fp16_weight": false,
     "llm_int8_skip_modules": null,

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 9.803921568627452,
-    "eval_loss": 4.065803050994873,
-    "eval_runtime": 2.0617,
+    "epoch": 10.0,
+    "eval_loss": 6.032225608825684,
+    "eval_runtime": 0.2331,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.85,
-    "eval_steps_per_second": 0.97
+    "eval_samples_per_second": 42.9,
+    "eval_steps_per_second": 4.29
 }

runs/Jun13_05-47-39_gpu1-2/events.out.tfevents.1718228907.gpu1-2.1119368.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9206e38c31c7f482395e1debcf45d6543f300b9fb75dc5ea12a9754fd215b1
+size 11560

runs/Jun13_05-47-39_gpu1-2/events.out.tfevents.1718229079.gpu1-2.1119368.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83e53ba28fdee41c2c8aca1ef0d2294fdcb16f2132303d6d9ea828e75435eca8
+size 354

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 9.803921568627452,
-    "total_flos": 1.9110914639160934e+17,
-    "train_loss": 3.4213723726272582,
-    "train_runtime": 1646.3335,
-    "train_samples": 3749,
-    "train_samples_per_second": 2.448,
-    "train_steps_per_second": 0.152
+    "epoch": 10.0,
+    "total_flos": 2.1344245100262195e+17,
+    "train_loss": 14.937398610796247,
+    "train_runtime": 171.2768,
+    "train_samples": 4038,
+    "train_samples_per_second": 25.573,
+    "train_steps_per_second": 0.409
 }

trainer_state.json

@@ -1,466 +1,214 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 9.803921568627452,
+  "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 250,
+  "global_step": 70,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0392156862745098,
-      "grad_norm": 536.0,
-      "learning_rate": 8.000000000000001e-06,
-      "loss": 51.7984,
+      "epoch": 0.14285714285714285,
+      "grad_norm": 183.0,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 47.2142,
       "step": 1
     },
     {
-      "epoch": 0.19607843137254902,
-      "grad_norm": 314.0,
-      "learning_rate": 4e-05,
-      "loss": 43.035,
+      "epoch": 0.7142857142857143,
+      "grad_norm": 97.5,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 44.3098,
       "step": 5
     },
     {
-      "epoch": 0.39215686274509803,
-      "grad_norm": 45.25,
-      "learning_rate": 8e-05,
-      "loss": 28.3835,
-      "step": 10
-    },
-    {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 11.5,
-      "learning_rate": 0.00012,
-      "loss": 21.6194,
-      "step": 15
-    },
-    {
-      "epoch": 0.7843137254901961,
-      "grad_norm": 10.0625,
-      "learning_rate": 0.00016,
-      "loss": 19.3838,
-      "step": 20
-    },
-    {
-      "epoch": 0.9803921568627451,
-      "grad_norm": 37.75,
-      "learning_rate": 0.0002,
-      "loss": 16.725,
-      "step": 25
-    },
-    {
-      "epoch": 0.9803921568627451,
-      "eval_loss": 6.552042484283447,
-      "eval_runtime": 2.0523,
-      "eval_samples_per_second": 4.873,
-      "eval_steps_per_second": 0.975,
-      "step": 25
+      "epoch": 1.0,
+      "eval_loss": 13.720385551452637,
+      "eval_runtime": 0.2481,
+      "eval_samples_per_second": 40.305,
+      "eval_steps_per_second": 4.031,
+      "step": 7
     },
     {
-      "epoch": 1.1764705882352942,
-      "grad_norm": 10.875,
-      "learning_rate": 0.00019975640502598244,
-      "loss": 7.143,
-      "step": 30
-    },
-    {
-      "epoch": 1.3725490196078431,
-      "grad_norm": 7.75,
-      "learning_rate": 0.00019902680687415705,
-      "loss": 2.2222,
-      "step": 35
-    },
-    {
-      "epoch": 1.5686274509803921,
-      "grad_norm": 2.453125,
-      "learning_rate": 0.00019781476007338058,
-      "loss": 1.7799,
-      "step": 40
-    },
-    {
-      "epoch": 1.7647058823529411,
-      "grad_norm": 4.34375,
-      "learning_rate": 0.0001961261695938319,
-      "loss": 1.5965,
-      "step": 45
-    },
-    {
-      "epoch": 1.9607843137254903,
-      "grad_norm": 2.4375,
-      "learning_rate": 0.00019396926207859084,
-      "loss": 1.5122,
-      "step": 50
+      "epoch": 1.4285714285714286,
+      "grad_norm": 18.375,
+      "learning_rate": 0.00019888308262251285,
+      "loss": 25.7366,
+      "step": 10
     },
     {
       "epoch": 2.0,
-      "eval_loss": 2.6625783443450928,
-      "eval_runtime": 2.0517,
-      "eval_samples_per_second": 4.874,
-      "eval_steps_per_second": 0.975,
-      "step": 51
-    },
-    {
-      "epoch": 2.156862745098039,
-      "grad_norm": 1.7421875,
-      "learning_rate": 0.0001913545457642601,
-      "loss": 1.3478,
-      "step": 55
-    },
-    {
-      "epoch": 2.3529411764705883,
-      "grad_norm": 2.046875,
-      "learning_rate": 0.00018829475928589271,
-      "loss": 1.26,
-      "step": 60
+      "eval_loss": 8.691579818725586,
+      "eval_runtime": 0.2437,
+      "eval_samples_per_second": 41.04,
+      "eval_steps_per_second": 4.104,
+      "step": 14
     },
     {
-      "epoch": 2.549019607843137,
-      "grad_norm": 3.8125,
-      "learning_rate": 0.0001848048096156426,
-      "loss": 1.2315,
-      "step": 65
+      "epoch": 2.142857142857143,
+      "grad_norm": 4.53125,
+      "learning_rate": 0.00019214762118704076,
+      "loss": 20.6774,
+      "step": 15
     },
     {
-      "epoch": 2.7450980392156863,
+      "epoch": 2.857142857142857,
       "grad_norm": 4.46875,
-      "learning_rate": 0.00018090169943749476,
-      "loss": 1.1548,
-      "step": 70
-    },
-    {
-      "epoch": 2.9411764705882355,
-      "grad_norm": 1.5,
-      "learning_rate": 0.0001766044443118978,
-      "loss": 1.1154,
-      "step": 75
-    },
-    {
-      "epoch": 2.980392156862745,
-      "eval_loss": 2.591742753982544,
-      "eval_runtime": 2.0491,
-      "eval_samples_per_second": 4.88,
-      "eval_steps_per_second": 0.976,
-      "step": 76
-    },
-    {
-      "epoch": 3.1372549019607843,
-      "grad_norm": 1.9375,
-      "learning_rate": 0.0001719339800338651,
-      "loss": 1.0096,
-      "step": 80
-    },
-    {
-      "epoch": 3.3333333333333335,
-      "grad_norm": 3.40625,
-      "learning_rate": 0.00016691306063588583,
-      "loss": 0.9379,
-      "step": 85
-    },
-    {
-      "epoch": 3.5294117647058822,
-      "grad_norm": 1.65625,
-      "learning_rate": 0.0001615661475325658,
-      "loss": 0.9333,
-      "step": 90
+      "learning_rate": 0.00017971325072229226,
+      "loss": 19.0375,
+      "step": 20
     },
     {
-      "epoch": 3.7254901960784315,
-      "grad_norm": 3.109375,
-      "learning_rate": 0.0001559192903470747,
-      "loss": 0.9114,
-      "step": 95
+      "epoch": 3.0,
+      "eval_loss": 7.630770683288574,
+      "eval_runtime": 0.2338,
+      "eval_samples_per_second": 42.776,
+      "eval_steps_per_second": 4.278,
+      "step": 21
     },
     {
-      "epoch": 3.9215686274509802,
-      "grad_norm": 11.0,
-      "learning_rate": 0.00015000000000000001,
-      "loss": 0.9204,
-      "step": 100
+      "epoch": 3.571428571428571,
+      "grad_norm": 8.375,
+      "learning_rate": 0.00016234898018587337,
+      "loss": 18.2973,
+      "step": 25
     },
     {
       "epoch": 4.0,
-      "eval_loss": 2.6570026874542236,
-      "eval_runtime": 2.0401,
-      "eval_samples_per_second": 4.902,
-      "eval_steps_per_second": 0.98,
-      "step": 102
+      "eval_loss": 7.119815826416016,
+      "eval_runtime": 0.2413,
+      "eval_samples_per_second": 41.435,
+      "eval_steps_per_second": 4.143,
+      "step": 28
     },
     {
-      "epoch": 4.117647058823529,
-      "grad_norm": 1.4296875,
-      "learning_rate": 0.00014383711467890774,
-      "loss": 0.8556,
-      "step": 105
-    },
-    {
-      "epoch": 4.313725490196078,
-      "grad_norm": 1.3984375,
-      "learning_rate": 0.00013746065934159123,
-      "loss": 0.786,
-      "step": 110
-    },
-    {
-      "epoch": 4.509803921568627,
-      "grad_norm": 1.0703125,
-      "learning_rate": 0.00013090169943749476,
-      "loss": 0.797,
-      "step": 115
-    },
-    {
-      "epoch": 4.705882352941177,
-      "grad_norm": 1.9609375,
-      "learning_rate": 0.00012419218955996676,
-      "loss": 0.7689,
-      "step": 120
+      "epoch": 4.285714285714286,
+      "grad_norm": 12.3125,
+      "learning_rate": 0.00014112871031306119,
+      "loss": 16.6166,
+      "step": 30
     },
     {
-      "epoch": 4.901960784313726,
-      "grad_norm": 1.4296875,
+      "epoch": 5.0,
+      "grad_norm": 16.625,
       "learning_rate": 0.00011736481776669306,
-      "loss": 0.779,
-      "step": 125
-    },
-    {
-      "epoch": 4.980392156862745,
-      "eval_loss": 2.749800205230713,
-      "eval_runtime": 2.0472,
-      "eval_samples_per_second": 4.885,
-      "eval_steps_per_second": 0.977,
-      "step": 127
-    },
-    {
-      "epoch": 5.098039215686274,
-      "grad_norm": 15.25,
-      "learning_rate": 0.00011045284632676536,
-      "loss": 0.6889,
-      "step": 130
-    },
-    {
-      "epoch": 5.294117647058823,
-      "grad_norm": 1.28125,
-      "learning_rate": 0.00010348994967025012,
-      "loss": 0.6232,
-      "step": 135
-    },
-    {
-      "epoch": 5.490196078431373,
-      "grad_norm": 1.359375,
-      "learning_rate": 9.651005032974994e-05,
-      "loss": 0.6134,
-      "step": 140
+      "loss": 14.8387,
+      "step": 35
     },
     {
-      "epoch": 5.686274509803922,
-      "grad_norm": 1.25,
-      "learning_rate": 8.954715367323468e-05,
-      "loss": 0.612,
-      "step": 145
+      "epoch": 5.0,
+      "eval_loss": 6.846951484680176,
+      "eval_runtime": 0.2339,
+      "eval_samples_per_second": 42.753,
+      "eval_steps_per_second": 4.275,
+      "step": 35
     },
     {
-      "epoch": 5.882352941176471,
-      "grad_norm": 1.0390625,
-      "learning_rate": 8.263518223330697e-05,
-      "loss": 0.6207,
-      "step": 150
+      "epoch": 5.714285714285714,
+      "grad_norm": 23.5,
+      "learning_rate": 9.252699064135758e-05,
+      "loss": 12.5684,
+      "step": 40
     },
     {
       "epoch": 6.0,
-      "eval_loss": 2.9976024627685547,
-      "eval_runtime": 2.0445,
-      "eval_samples_per_second": 4.891,
-      "eval_steps_per_second": 0.978,
-      "step": 153
-    },
-    {
-      "epoch": 6.078431372549019,
-      "grad_norm": 1.1328125,
-      "learning_rate": 7.580781044003324e-05,
-      "loss": 0.5836,
-      "step": 155
-    },
-    {
-      "epoch": 6.2745098039215685,
-      "grad_norm": 1.203125,
-      "learning_rate": 6.909830056250527e-05,
-      "loss": 0.5047,
-      "step": 160
-    },
-    {
-      "epoch": 6.470588235294118,
-      "grad_norm": 1.4765625,
-      "learning_rate": 6.25393406584088e-05,
-      "loss": 0.4862,
-      "step": 165
-    },
-    {
-      "epoch": 6.666666666666667,
-      "grad_norm": 1.21875,
-      "learning_rate": 5.616288532109225e-05,
-      "loss": 0.4953,
-      "step": 170
-    },
-    {
-      "epoch": 6.862745098039216,
-      "grad_norm": 1.1484375,
-      "learning_rate": 5.000000000000002e-05,
-      "loss": 0.4762,
-      "step": 175
+      "eval_loss": 6.8495354652404785,
+      "eval_runtime": 0.2342,
+      "eval_samples_per_second": 42.698,
+      "eval_steps_per_second": 4.27,
+      "step": 42
     },
     {
-      "epoch": 6.980392156862745,
-      "eval_loss": 3.466813325881958,
-      "eval_runtime": 2.0565,
-      "eval_samples_per_second": 4.863,
-      "eval_steps_per_second": 0.973,
-      "step": 178
-    },
-    {
-      "epoch": 7.0588235294117645,
-      "grad_norm": 1.203125,
-      "learning_rate": 4.4080709652925336e-05,
-      "loss": 0.4669,
-      "step": 180
-    },
-    {
-      "epoch": 7.254901960784314,
-      "grad_norm": 1.4140625,
-      "learning_rate": 3.843385246743417e-05,
-      "loss": 0.3974,
-      "step": 185
+      "epoch": 6.428571428571429,
+      "grad_norm": 26.0,
+      "learning_rate": 6.815133497483157e-05,
+      "loss": 9.6308,
+      "step": 45
     },
     {
-      "epoch": 7.450980392156863,
-      "grad_norm": 1.0625,
-      "learning_rate": 3.308693936411421e-05,
-      "loss": 0.3956,
-      "step": 190
+      "epoch": 7.0,
+      "eval_loss": 6.679884910583496,
+      "eval_runtime": 0.2349,
+      "eval_samples_per_second": 42.574,
+      "eval_steps_per_second": 4.257,
+      "step": 49
     },
     {
-      "epoch": 7.647058823529412,
-      "grad_norm": 1.0703125,
-      "learning_rate": 2.8066019966134904e-05,
-      "loss": 0.386,
-      "step": 195
+      "epoch": 7.142857142857143,
+      "grad_norm": 26.25,
+      "learning_rate": 4.574537361342407e-05,
+      "loss": 7.3589,
+      "step": 50
     },
     {
-      "epoch": 7.8431372549019605,
-      "grad_norm": 1.1328125,
-      "learning_rate": 2.339555568810221e-05,
-      "loss": 0.3908,
-      "step": 200
+      "epoch": 7.857142857142857,
+      "grad_norm": 23.125,
+      "learning_rate": 2.669481281701739e-05,
+      "loss": 5.7187,
+      "step": 55
     },
     {
       "epoch": 8.0,
-      "eval_loss": 3.824570417404175,
-      "eval_runtime": 2.0426,
-      "eval_samples_per_second": 4.896,
-      "eval_steps_per_second": 0.979,
-      "step": 204
-    },
-    {
-      "epoch": 8.03921568627451,
-      "grad_norm": 0.984375,
-      "learning_rate": 1.9098300562505266e-05,
-      "loss": 0.3721,
-      "step": 205
-    },
-    {
-      "epoch": 8.235294117647058,
-      "grad_norm": 0.96875,
-      "learning_rate": 1.5195190384357404e-05,
-      "loss": 0.3342,
-      "step": 210
-    },
-    {
-      "epoch": 8.431372549019608,
-      "grad_norm": 1.046875,
-      "learning_rate": 1.1705240714107302e-05,
-      "loss": 0.346,
-      "step": 215
+      "eval_loss": 6.081761360168457,
+      "eval_runtime": 0.2356,
+      "eval_samples_per_second": 42.45,
+      "eval_steps_per_second": 4.245,
+      "step": 56
     },
     {
-      "epoch": 8.627450980392156,
-      "grad_norm": 1.046875,
-      "learning_rate": 8.645454235739903e-06,
-      "loss": 0.3381,
-      "step": 220
-    },
-    {
-      "epoch": 8.823529411764707,
-      "grad_norm": 1.046875,
-      "learning_rate": 6.030737921409169e-06,
-      "loss": 0.3418,
-      "step": 225
-    },
-    {
-      "epoch": 8.980392156862745,
-      "eval_loss": 4.056135177612305,
-      "eval_runtime": 2.0561,
-      "eval_samples_per_second": 4.864,
-      "eval_steps_per_second": 0.973,
-      "step": 229
-    },
-    {
-      "epoch": 9.019607843137255,
-      "grad_norm": 0.88671875,
-      "learning_rate": 3.873830406168111e-06,
-      "loss": 0.3294,
-      "step": 230
-    },
-    {
-      "epoch": 9.215686274509803,
-      "grad_norm": 0.89453125,
-      "learning_rate": 2.1852399266194314e-06,
-      "loss": 0.3265,
-      "step": 235
+      "epoch": 8.571428571428571,
+      "grad_norm": 22.625,
+      "learning_rate": 1.2177842662977135e-05,
+      "loss": 4.8487,
+      "step": 60
     },
     {
-      "epoch": 9.411764705882353,
-      "grad_norm": 0.9296875,
-      "learning_rate": 9.731931258429638e-07,
-      "loss": 0.3268,
-      "step": 240
+      "epoch": 9.0,
+      "eval_loss": 6.031832695007324,
+      "eval_runtime": 0.2458,
+      "eval_samples_per_second": 40.691,
+      "eval_steps_per_second": 4.069,
+      "step": 63
     },
     {
-      "epoch": 9.607843137254902,
-      "grad_norm": 0.91796875,
-      "learning_rate": 2.4359497401758024e-07,
-      "loss": 0.3257,
-      "step": 245
+      "epoch": 9.285714285714286,
+      "grad_norm": 22.25,
+      "learning_rate": 3.092271377092215e-06,
+      "loss": 4.4729,
+      "step": 65
     },
     {
-      "epoch": 9.803921568627452,
-      "grad_norm": 0.9140625,
+      "epoch": 10.0,
+      "grad_norm": 21.75,
       "learning_rate": 0.0,
-      "loss": 0.3252,
-      "step": 250
+      "loss": 4.4303,
+      "step": 70
     },
     {
-      "epoch": 9.803921568627452,
-      "eval_loss": 4.065803050994873,
-      "eval_runtime": 2.0429,
-      "eval_samples_per_second": 4.895,
-      "eval_steps_per_second": 0.979,
-      "step": 250
+      "epoch": 10.0,
+      "eval_loss": 6.032225608825684,
+      "eval_runtime": 0.2341,
+      "eval_samples_per_second": 42.713,
+      "eval_steps_per_second": 4.271,
+      "step": 70
     },
     {
-      "epoch": 9.803921568627452,
-      "step": 250,
-      "total_flos": 1.9110914639160934e+17,
-      "train_loss": 3.4213723726272582,
-      "train_runtime": 1646.3335,
-      "train_samples_per_second": 2.448,
-      "train_steps_per_second": 0.152
+      "epoch": 10.0,
+      "step": 70,
+      "total_flos": 2.1344245100262195e+17,
+      "train_loss": 14.937398610796247,
+      "train_runtime": 171.2768,
+      "train_samples_per_second": 25.573,
+      "train_steps_per_second": 0.409
     }
   ],
   "logging_steps": 5,
-  "max_steps": 250,
+  "max_steps": 70,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 100,
-  "total_flos": 1.9110914639160934e+17,
+  "total_flos": 2.1344245100262195e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfce4a04a478c718b10def89fee944166ea94cfde8205dce51c434187249fb6b
+oid sha256:419732def1570977ca1dc458a816ea21cf64063785f04b3d10fe09e6b0bc2b83
 size 5176