Model save

README.md

@@ -2,13 +2,11 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: alignment-handbook/zephyr-7b-sft-full
-datasets:
-- EllieS/timedial_dpo
 model-index:
 - name: zephyr-dpo-timedial-selfgen-mix2
   results: []
@@ -19,17 +17,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-dpo-timedial-selfgen-mix2
 
-This model is a fine-tuned version of [EllieS/zephyr-dpo-timedial-selfgen](https://huggingface.co/EllieS/zephyr-dpo-timedial-selfgen) on the EllieS/timedial_dpo dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
+- Logits/chosen: -2.8807
+- Logits/rejected: -2.8679
+- Logps/chosen: -6.8403
+- Logps/rejected: -132.2824
 - Loss: 0.2588
-- Rewards/chosen: 0.2087
-- Rewards/rejected: -1.0200
 - Rewards/accuracies: 1.0
+- Rewards/chosen: 0.2087
 - Rewards/margins: 1.2287
-- Logps/rejected: -132.2824
-- Logps/chosen: -6.8403
-- Logits/rejected: -2.8679
-- Logits/chosen: -2.8807
+- Rewards/rejected: -1.0200
 
 ## Model description
 
@@ -62,10 +60,10 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4825        | 0.35  | 100  | 0.4632          | 0.2303         | -0.3022          | 1.0                | 0.5325          | -60.5070       | -4.6766      | -2.9751         | -2.9715       |
-| 0.2786        | 0.69  | 200  | 0.2588          | 0.2087         | -1.0200          | 1.0                | 1.2287          | -132.2824      | -6.8403      | -2.8679         | -2.8807       |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.4825        | 0.35  | 100  | -2.9715       | -2.9751         | -4.6766      | -60.5070       | 0.4632          | 1.0                | 0.2303         | 0.5325          | -0.3022          |
+| 0.2786        | 0.69  | 200  | -2.8807       | -2.8679         | -6.8403      | -132.2824      | 0.2588          | 1.0                | 0.2087         | 1.2287          | -1.0200          |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9964ad62a8bf5a21fb9b027f8ce26c447d3fa36afcd73ed4cfcdd37f63636b91
+oid sha256:9eebfd4ad7987bc9fd47f79609fadacdd943a8b6b0f4182920b062aedc908acb
 size 83946192

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4018066387275511,
-    "train_runtime": 624.4981,
+    "train_loss": 0.06608598743755512,
+    "train_runtime": 154.0704,
     "train_samples": 1157,
-    "train_samples_per_second": 1.853,
-    "train_steps_per_second": 0.463
+    "train_samples_per_second": 7.51,
+    "train_steps_per_second": 1.876
 }

runs/Apr25_04-55-28_612e66badb5c/events.out.tfevents.1714021048.612e66badb5c.14611.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec4532d2759cba55b54f4f95aa379f154fc94f1839ac4aa7c022e8e9341e943b
+size 10676

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4018066387275511,
-    "train_runtime": 624.4981,
+    "train_loss": 0.06608598743755512,
+    "train_runtime": 154.0704,
     "train_samples": 1157,
-    "train_samples_per_second": 1.853,
-    "train_steps_per_second": 0.463
+    "train_samples_per_second": 7.51,
+    "train_steps_per_second": 1.876
 }

trainer_state.json

@@ -357,132 +357,132 @@
     },
     {
       "epoch": 0.73,
-      "grad_norm": 1.0390625,
+      "grad_norm": 1.046875,
       "learning_rate": 1.0550820234444627e-06,
-      "logits/chosen": -2.8844265937805176,
-      "logits/rejected": -2.8720266819000244,
-      "logps/chosen": -7.328047752380371,
-      "logps/rejected": -136.2236785888672,
-      "loss": 0.2536,
+      "logits/chosen": -2.884814500808716,
+      "logits/rejected": -2.8724067211151123,
+      "logps/chosen": -7.440223693847656,
+      "logps/rejected": -136.234375,
+      "loss": 0.2538,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2088628113269806,
-      "rewards/margins": 1.2624285221099854,
-      "rewards/rejected": -1.0535659790039062,
+      "rewards/chosen": 0.20774102210998535,
+      "rewards/margins": 1.261413812637329,
+      "rewards/rejected": -1.0536725521087646,
       "step": 210
     },
     {
       "epoch": 0.76,
-      "grad_norm": 1.5,
+      "grad_norm": 1.5078125,
       "learning_rate": 8.197041935593181e-07,
-      "logits/chosen": -2.865748643875122,
-      "logits/rejected": -2.8470582962036133,
-      "logps/chosen": -3.5205326080322266,
-      "logps/rejected": -143.3411865234375,
-      "loss": 0.2291,
+      "logits/chosen": -2.8656671047210693,
+      "logits/rejected": -2.8470237255096436,
+      "logps/chosen": -3.4960861206054688,
+      "logps/rejected": -143.31100463867188,
+      "loss": 0.229,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2387482225894928,
-      "rewards/margins": 1.365382432937622,
-      "rewards/rejected": -1.126634120941162,
+      "rewards/chosen": 0.23899272084236145,
+      "rewards/margins": 1.3653249740600586,
+      "rewards/rejected": -1.1263322830200195,
       "step": 220
     },
     {
       "epoch": 0.79,
-      "grad_norm": 0.97265625,
+      "grad_norm": 0.9453125,
       "learning_rate": 6.088288602287159e-07,
-      "logits/chosen": -2.8658416271209717,
-      "logits/rejected": -2.8466126918792725,
-      "logps/chosen": -1.298507809638977,
-      "logps/rejected": -147.50143432617188,
-      "loss": 0.2162,
+      "logits/chosen": -2.866046190261841,
+      "logits/rejected": -2.8468241691589355,
+      "logps/chosen": -1.2698005437850952,
+      "logps/rejected": -147.30690002441406,
+      "loss": 0.2167,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2748534083366394,
-      "rewards/margins": 1.4337130784988403,
-      "rewards/rejected": -1.1588596105575562,
+      "rewards/chosen": 0.2751404643058777,
+      "rewards/margins": 1.4320547580718994,
+      "rewards/rejected": -1.156914234161377,
       "step": 230
     },
     {
       "epoch": 0.83,
-      "grad_norm": 1.34375,
+      "grad_norm": 1.40625,
       "learning_rate": 4.255310606625124e-07,
-      "logits/chosen": -2.8704545497894287,
-      "logits/rejected": -2.8518197536468506,
-      "logps/chosen": -2.5422964096069336,
-      "logps/rejected": -151.9074249267578,
-      "loss": 0.2081,
+      "logits/chosen": -2.8706631660461426,
+      "logits/rejected": -2.852060317993164,
+      "logps/chosen": -2.512279510498047,
+      "logps/rejected": -151.74819946289062,
+      "loss": 0.2084,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.25674667954444885,
-      "rewards/margins": 1.4753159284591675,
-      "rewards/rejected": -1.2185693979263306,
+      "rewards/chosen": 0.2570468485355377,
+      "rewards/margins": 1.4740240573883057,
+      "rewards/rejected": -1.2169771194458008,
       "step": 240
     },
     {
       "epoch": 0.86,
-      "grad_norm": 0.9921875,
+      "grad_norm": 0.96875,
       "learning_rate": 2.7248368952908055e-07,
-      "logits/chosen": -2.861412525177002,
-      "logits/rejected": -2.838165044784546,
-      "logps/chosen": -2.5676398277282715,
-      "logps/rejected": -152.24969482421875,
-      "loss": 0.2105,
+      "logits/chosen": -2.86161470413208,
+      "logits/rejected": -2.838373899459839,
+      "logps/chosen": -2.557521104812622,
+      "logps/rejected": -152.18783569335938,
+      "loss": 0.2107,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2601901888847351,
-      "rewards/margins": 1.4616148471832275,
-      "rewards/rejected": -1.2014245986938477,
+      "rewards/chosen": 0.2602913975715637,
+      "rewards/margins": 1.4610974788665771,
+      "rewards/rejected": -1.2008062601089478,
       "step": 250
     },
     {
       "epoch": 0.9,
-      "grad_norm": 0.88671875,
+      "grad_norm": 0.87890625,
       "learning_rate": 1.5191852213221198e-07,
-      "logits/chosen": -2.8463876247406006,
-      "logits/rejected": -2.817927837371826,
-      "logps/chosen": -3.504025936126709,
-      "logps/rejected": -156.09738159179688,
-      "loss": 0.2054,
+      "logits/chosen": -2.846947431564331,
+      "logits/rejected": -2.818504810333252,
+      "logps/chosen": -3.4454503059387207,
+      "logps/rejected": -156.12364196777344,
+      "loss": 0.2052,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2497003823518753,
-      "rewards/margins": 1.4921144247055054,
-      "rewards/rejected": -1.2424139976501465,
+      "rewards/chosen": 0.25028613209724426,
+      "rewards/margins": 1.4929627180099487,
+      "rewards/rejected": -1.2426766157150269,
       "step": 260
     },
     {
       "epoch": 0.93,
-      "grad_norm": 0.921875,
+      "grad_norm": 0.9296875,
       "learning_rate": 6.559367010166629e-08,
-      "logits/chosen": -2.8501267433166504,
-      "logits/rejected": -2.8306031227111816,
-      "logps/chosen": -2.110583782196045,
-      "logps/rejected": -152.65463256835938,
-      "loss": 0.2113,
+      "logits/chosen": -2.8502001762390137,
+      "logits/rejected": -2.8307251930236816,
+      "logps/chosen": -2.046212673187256,
+      "logps/rejected": -152.60047912597656,
+      "loss": 0.2112,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.26196378469467163,
-      "rewards/margins": 1.4719316959381104,
-      "rewards/rejected": -1.209967851638794,
+      "rewards/chosen": 0.2626075744628906,
+      "rewards/margins": 1.4720337390899658,
+      "rewards/rejected": -1.2094261646270752,
       "step": 270
     },
     {
       "epoch": 0.97,
-      "grad_norm": 1.453125,
+      "grad_norm": 1.5,
       "learning_rate": 1.4767944166687032e-08,
-      "logits/chosen": -2.8537425994873047,
-      "logits/rejected": -2.827571392059326,
-      "logps/chosen": -2.493337869644165,
-      "logps/rejected": -154.96142578125,
+      "logits/chosen": -2.854003429412842,
+      "logits/rejected": -2.827923059463501,
+      "logps/chosen": -2.4084525108337402,
+      "logps/rejected": -154.87083435058594,
       "loss": 0.2011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2604244649410248,
-      "rewards/margins": 1.5092506408691406,
-      "rewards/rejected": -1.2488261461257935,
+      "rewards/chosen": 0.2612733244895935,
+      "rewards/margins": 1.5091934204101562,
+      "rewards/rejected": -1.247920274734497,
       "step": 280
     },
     {
       "epoch": 1.0,
       "step": 289,
       "total_flos": 0.0,
-      "train_loss": 0.4018066387275511,
-      "train_runtime": 624.4981,
-      "train_samples_per_second": 1.853,
-      "train_steps_per_second": 0.463
+      "train_loss": 0.06608598743755512,
+      "train_runtime": 154.0704,
+      "train_samples_per_second": 7.51,
+      "train_steps_per_second": 1.876
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee4ee2378955f36b9d54f6508ae4b9cbdfdb3ba6dfb1358a2260f46e52a0e1c2
+oid sha256:68d7e87f0cf192833a8bc67551a5749dfc4594ac8ce2b861912976bde3429e7b
 size 5112