Model save

README.md

@@ -3,6 +3,8 @@ license: apache-2.0
 base_model: mistralai/Mistral-7B-v0.1
 tags:
 - generated_from_trainer
+datasets:
+- generator
 model-index:
 - name: zephyr-7b-sft-lora
   results: []
@@ -13,9 +15,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-sft-lora
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0667
+- Loss: 1.0927
 
 ## Model description
 
@@ -45,13 +47,14 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.063         | 0.67  | 272  | 1.0667          |
+| 1.0931        | 1.0   | 272  | 1.0927          |
 
 
 ### Framework versions

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "q_proj",
+    "v_proj",
     "o_proj",
-    "v_proj"
+    "q_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48b34a428074e5b7cc32b3fd5343936b35b0a5c869b9ecd3c09f7000dcb50b11
+oid sha256:5eaa5158d98175f4ed59693761cec1820cae65c52813b72136dc84fe785a420b
 size 109086672

all_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 0.67,
-    "eval_loss": 1.0666700601577759,
-    "eval_runtime": 2384.3246,
+    "epoch": 1.0,
+    "eval_loss": 1.0926721096038818,
+    "eval_runtime": 2381.0415,
     "eval_samples": 23110,
-    "eval_samples_per_second": 9.692,
-    "eval_steps_per_second": 0.152,
-    "train_loss": 1.1599246359923308,
-    "train_runtime": 72626.7254,
+    "eval_samples_per_second": 6.481,
+    "eval_steps_per_second": 0.102,
+    "train_loss": 1.1944836974143982,
+    "train_runtime": 72472.9134,
     "train_samples": 207865,
-    "train_samples_per_second": 2.862,
-    "train_steps_per_second": 0.006
+    "train_samples_per_second": 1.924,
+    "train_steps_per_second": 0.004
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.67,
-    "eval_loss": 1.0666700601577759,
-    "eval_runtime": 2384.3246,
+    "epoch": 1.0,
+    "eval_loss": 1.0926721096038818,
+    "eval_runtime": 2381.0415,
     "eval_samples": 23110,
-    "eval_samples_per_second": 9.692,
-    "eval_steps_per_second": 0.152
+    "eval_samples_per_second": 6.481,
+    "eval_steps_per_second": 0.102
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.67,
-    "train_loss": 1.1599246359923308,
-    "train_runtime": 72626.7254,
+    "epoch": 1.0,
+    "train_loss": 1.1944836974143982,
+    "train_runtime": 72472.9134,
     "train_samples": 207865,
-    "train_samples_per_second": 2.862,
-    "train_steps_per_second": 0.006
+    "train_samples_per_second": 1.924,
+    "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6699507389162561,
+  "epoch": 0.9986232216613126,
   "eval_steps": 500,
   "global_step": 272,
   "is_hyper_param_search": false,
@@ -10,357 +10,357 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.9999700625010444e-05,
-      "loss": 1.6931,
+      "learning_rate": 7.142857142857143e-07,
+      "loss": 1.6842,
       "step": 1
     },
     {
-      "epoch": 0.01,
-      "learning_rate": 1.999251652147735e-05,
-      "loss": 1.6625,
+      "epoch": 0.02,
+      "learning_rate": 3.5714285714285718e-06,
+      "loss": 1.6847,
       "step": 5
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.997007728639956e-05,
-      "loss": 1.5879,
+      "epoch": 0.04,
+      "learning_rate": 7.1428571428571436e-06,
+      "loss": 1.6885,
       "step": 10
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 1.9932715879473385e-05,
-      "loss": 1.5023,
+      "epoch": 0.06,
+      "learning_rate": 1.0714285714285714e-05,
+      "loss": 1.6795,
       "step": 15
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 1.9880488219356086e-05,
-      "loss": 1.4524,
+      "epoch": 0.07,
+      "learning_rate": 1.4285714285714287e-05,
+      "loss": 1.6085,
       "step": 20
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.981347247496222e-05,
-      "loss": 1.3857,
+      "epoch": 0.09,
+      "learning_rate": 1.785714285714286e-05,
+      "loss": 1.5545,
       "step": 25
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 1.973176894846855e-05,
-      "loss": 1.348,
+      "epoch": 0.11,
+      "learning_rate": 1.9996684675143132e-05,
+      "loss": 1.465,
       "step": 30
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 1.963549992519223e-05,
-      "loss": 1.3286,
+      "epoch": 0.13,
+      "learning_rate": 1.995941251172356e-05,
+      "loss": 1.4256,
       "step": 35
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.9524809490566878e-05,
-      "loss": 1.2921,
+      "epoch": 0.15,
+      "learning_rate": 1.9880878960910772e-05,
+      "loss": 1.3637,
       "step": 40
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.939986331449053e-05,
-      "loss": 1.2735,
+      "epoch": 0.17,
+      "learning_rate": 1.9761409383589216e-05,
+      "loss": 1.3306,
       "step": 45
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 1.926084840336821e-05,
-      "loss": 1.2587,
+      "epoch": 0.18,
+      "learning_rate": 1.960149873671602e-05,
+      "loss": 1.307,
       "step": 50
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 1.910797282022027e-05,
-      "loss": 1.2334,
+      "epoch": 0.2,
+      "learning_rate": 1.94018095227371e-05,
+      "loss": 1.2769,
       "step": 55
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 1.894146537327533e-05,
-      "loss": 1.2086,
+      "epoch": 0.22,
+      "learning_rate": 1.916316904487005e-05,
+      "loss": 1.2581,
       "step": 60
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 1.8761575273514005e-05,
-      "loss": 1.189,
+      "epoch": 0.24,
+      "learning_rate": 1.888656597962499e-05,
+      "loss": 1.2415,
       "step": 65
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 1.8568571761675893e-05,
-      "loss": 1.1707,
+      "epoch": 0.26,
+      "learning_rate": 1.8573146280763327e-05,
+      "loss": 1.2178,
       "step": 70
     },
     {
-      "epoch": 0.18,
-      "learning_rate": 1.8362743705288127e-05,
-      "loss": 1.1654,
+      "epoch": 0.28,
+      "learning_rate": 1.8224208431664093e-05,
+      "loss": 1.2036,
       "step": 75
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 1.814439916631857e-05,
-      "loss": 1.1668,
+      "epoch": 0.29,
+      "learning_rate": 1.7841198065767107e-05,
+      "loss": 1.1768,
       "step": 80
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 1.791386494010081e-05,
-      "loss": 1.1569,
+      "epoch": 0.31,
+      "learning_rate": 1.742570197738019e-05,
+      "loss": 1.1738,
       "step": 85
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 1.7671486066220965e-05,
-      "loss": 1.1376,
+      "epoch": 0.33,
+      "learning_rate": 1.6979441547663434e-05,
+      "loss": 1.1708,
       "step": 90
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 1.7417625312098453e-05,
-      "loss": 1.1222,
+      "epoch": 0.35,
+      "learning_rate": 1.650426561302634e-05,
+      "loss": 1.1551,
       "step": 95
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 1.7152662630033506e-05,
-      "loss": 1.1304,
+      "epoch": 0.37,
+      "learning_rate": 1.6002142805483686e-05,
+      "loss": 1.1515,
       "step": 100
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 1.6876994588534234e-05,
-      "loss": 1.1425,
+      "epoch": 0.39,
+      "learning_rate": 1.5475153396703756e-05,
+      "loss": 1.1406,
       "step": 105
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 1.659103377877423e-05,
-      "loss": 1.1127,
+      "epoch": 0.4,
+      "learning_rate": 1.4925480679538646e-05,
+      "loss": 1.1402,
       "step": 110
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 1.629520819706912e-05,
-      "loss": 1.107,
+      "epoch": 0.42,
+      "learning_rate": 1.4355401922742478e-05,
+      "loss": 1.1297,
       "step": 115
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 1.598996060429634e-05,
-      "loss": 1.1078,
+      "epoch": 0.44,
+      "learning_rate": 1.3767278936351853e-05,
+      "loss": 1.1265,
       "step": 120
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 1.56757478632168e-05,
-      "loss": 1.103,
+      "epoch": 0.46,
+      "learning_rate": 1.316354828681562e-05,
+      "loss": 1.113,
       "step": 125
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 1.5353040254690396e-05,
-      "loss": 1.1107,
+      "epoch": 0.48,
+      "learning_rate": 1.2546711202412287e-05,
+      "loss": 1.1146,
       "step": 130
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 1.5022320773808612e-05,
-      "loss": 1.1062,
+      "epoch": 0.5,
+      "learning_rate": 1.1919323210776459e-05,
+      "loss": 1.1153,
       "step": 135
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 1.4684084406997903e-05,
-      "loss": 1.091,
+      "epoch": 0.51,
+      "learning_rate": 1.1283983551465512e-05,
+      "loss": 1.1102,
       "step": 140
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 1.4338837391175582e-05,
-      "loss": 1.0942,
+      "epoch": 0.53,
+      "learning_rate": 1.0643324407429818e-05,
+      "loss": 1.0941,
       "step": 145
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 1.3987096456067236e-05,
-      "loss": 1.0887,
+      "epoch": 0.55,
+      "learning_rate": 1e-05,
+      "loss": 1.1037,
       "step": 150
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 1.3629388050819547e-05,
-      "loss": 1.0813,
+      "epoch": 0.57,
+      "learning_rate": 9.356675592570185e-06,
+      "loss": 1.099,
       "step": 155
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 1.3266247556066122e-05,
-      "loss": 1.0699,
+      "epoch": 0.59,
+      "learning_rate": 8.71601644853449e-06,
+      "loss": 1.0926,
       "step": 160
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 1.2898218482625606e-05,
-      "loss": 1.0875,
+      "epoch": 0.61,
+      "learning_rate": 8.080676789223545e-06,
+      "loss": 1.1019,
       "step": 165
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 1.252585165803135e-05,
-      "loss": 1.0872,
+      "epoch": 0.62,
+      "learning_rate": 7.453288797587714e-06,
+      "loss": 1.0985,
       "step": 170
     },
     {
-      "epoch": 0.43,
-      "learning_rate": 1.2149704402110243e-05,
-      "loss": 1.0709,
+      "epoch": 0.64,
+      "learning_rate": 6.8364517131843865e-06,
+      "loss": 1.0993,
       "step": 175
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 1.1770339692844484e-05,
-      "loss": 1.0772,
+      "epoch": 0.66,
+      "learning_rate": 6.232721063648148e-06,
+      "loss": 1.0935,
       "step": 180
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 1.1388325323764889e-05,
-      "loss": 1.0847,
+      "epoch": 0.68,
+      "learning_rate": 5.644598077257524e-06,
+      "loss": 1.0921,
       "step": 185
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 1.1004233054136726e-05,
-      "loss": 1.0778,
+      "epoch": 0.7,
+      "learning_rate": 5.074519320461358e-06,
+      "loss": 1.0967,
       "step": 190
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 1.0618637753210086e-05,
-      "loss": 1.0798,
+      "epoch": 0.72,
+      "learning_rate": 4.524846603296245e-06,
+      "loss": 1.0958,
       "step": 195
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 1.0232116539815558e-05,
-      "loss": 1.0672,
+      "epoch": 0.73,
+      "learning_rate": 3.997857194516319e-06,
+      "loss": 1.0936,
       "step": 200
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 9.845247918592937e-06,
-      "loss": 1.0715,
+      "epoch": 0.75,
+      "learning_rate": 3.4957343869736604e-06,
+      "loss": 1.0927,
       "step": 205
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 9.458610914145826e-06,
-      "loss": 1.07,
+      "epoch": 0.77,
+      "learning_rate": 3.0205584523365626e-06,
+      "loss": 1.0931,
       "step": 210
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 9.072784204417995e-06,
-      "loss": 1.0796,
+      "epoch": 0.79,
+      "learning_rate": 2.5742980226198133e-06,
+      "loss": 1.0928,
       "step": 215
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 8.688345254588579e-06,
-      "loss": 1.0686,
+      "epoch": 0.81,
+      "learning_rate": 2.158801934232897e-06,
+      "loss": 1.09,
       "step": 220
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 8.305869452782446e-06,
-      "loss": 1.0716,
+      "epoch": 0.83,
+      "learning_rate": 1.7757915683359084e-06,
+      "loss": 1.0963,
       "step": 225
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 7.92592924888925e-06,
-      "loss": 1.0707,
+      "epoch": 0.84,
+      "learning_rate": 1.426853719236676e-06,
+      "loss": 1.0874,
       "step": 230
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 7.549093297780133e-06,
-      "loss": 1.067,
+      "epoch": 0.86,
+      "learning_rate": 1.113434020375015e-06,
+      "loss": 1.0848,
       "step": 235
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 7.175925608204428e-06,
-      "loss": 1.0732,
+      "epoch": 0.88,
+      "learning_rate": 8.368309551299536e-07,
+      "loss": 1.0941,
       "step": 240
     },
     {
-      "epoch": 0.6,
-      "learning_rate": 6.806984698640202e-06,
-      "loss": 1.0746,
+      "epoch": 0.9,
+      "learning_rate": 5.98190477262901e-07,
+      "loss": 1.0872,
       "step": 245
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 6.442822761362015e-06,
-      "loss": 1.0622,
+      "epoch": 0.92,
+      "learning_rate": 3.985012632839824e-07,
+      "loss": 1.09,
       "step": 250
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 6.083984835977154e-06,
-      "loss": 1.0552,
+      "epoch": 0.94,
+      "learning_rate": 2.385906164107854e-07,
+      "loss": 1.0896,
       "step": 255
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 5.731007993667155e-06,
-      "loss": 1.0697,
+      "epoch": 0.95,
+      "learning_rate": 1.1912103908922945e-07,
+      "loss": 1.0803,
       "step": 260
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 5.38442053335571e-06,
-      "loss": 1.0599,
+      "epoch": 0.97,
+      "learning_rate": 4.058748827643899e-08,
+      "loss": 1.0919,
       "step": 265
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 5.044741191005908e-06,
-      "loss": 1.063,
+      "epoch": 0.99,
+      "learning_rate": 3.315324856869584e-09,
+      "loss": 1.0931,
       "step": 270
     },
     {
-      "epoch": 0.67,
-      "eval_loss": 1.066724419593811,
-      "eval_runtime": 2385.329,
-      "eval_samples_per_second": 9.688,
-      "eval_steps_per_second": 0.152,
+      "epoch": 1.0,
+      "eval_loss": 1.0926721096038818,
+      "eval_runtime": 2381.3881,
+      "eval_samples_per_second": 6.48,
+      "eval_steps_per_second": 0.102,
       "step": 272
     },
     {
-      "epoch": 0.67,
+      "epoch": 1.0,
       "step": 272,
-      "total_flos": 4.932776606721638e+16,
-      "train_loss": 1.1599246359923308,
-      "train_runtime": 72626.7254,
-      "train_samples_per_second": 2.862,
-      "train_steps_per_second": 0.006
+      "total_flos": 7631468079611904.0,
+      "train_loss": 1.1944836974143982,
+      "train_runtime": 72472.9134,
+      "train_samples_per_second": 1.924,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 5,
-  "max_steps": 406,
+  "max_steps": 272,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 4.932776606721638e+16,
+  "total_flos": 7631468079611904.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a116181e8e43014f8554b0cc2a81c8131193600bb4d374a77cd396fcfdc6416
+oid sha256:db90f8ee5898ac57b8ff11416193b63fff88243010bb512f35030b2ccadbeff0
 size 5624