Model save

README.md

@@ -2,15 +2,9 @@
 license: mit
 base_model: HuggingFaceH4/mistral-7b-sft-beta
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: DPO-Zephyr-7B
   results: []
@@ -21,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # DPO-Zephyr-7B
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9874476987447699,
     "total_flos": 0.0,
-    "train_loss": 0.6490178754774191,
-    "train_runtime": 1630.8514,
+    "train_loss": 0.6490362135030455,
+    "train_runtime": 1628.642,
     "train_samples": 15283,
-    "train_samples_per_second": 9.371,
+    "train_samples_per_second": 9.384,
     "train_steps_per_second": 0.036
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c102c7c24a4ffc9699e03c2bef20903e0d393c638be088dd4c10475225b2f91c
+oid sha256:1d655c22edf89a46ecc9b8ab8f1dd1f136951064119abcf1b7e2f3f2a47138f9
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eec78610bcb027f2d9a6a0dbf2b4b18aa303873090b7bbac8f6a4193cd50c5b
+oid sha256:5c2831fb26b09c22f51c2c3908253744b2fa7bd5252b25175e23dc337bb1dd70
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc676b5fe1e6987a2d21ed941fe4d6ec069fa2a0ee9df004a9da8eb1462a4229
+oid sha256:8e911e70c1914212070f5a2b59d5003a2daf2522f3a2346faffd311c91d5cbbb
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9874476987447699,
     "total_flos": 0.0,
-    "train_loss": 0.6490178754774191,
-    "train_runtime": 1630.8514,
+    "train_loss": 0.6490362135030455,
+    "train_runtime": 1628.642,
     "train_samples": 15283,
-    "train_samples_per_second": 9.371,
+    "train_samples_per_second": 9.384,
     "train_steps_per_second": 0.036
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.016736401673640166,
-      "grad_norm": 7.772482099050406,
+      "grad_norm": 7.775175087490474,
       "learning_rate": 8.333333333333333e-08,
       "logits/chosen": -2.807276487350464,
       "logits/rejected": -2.7759768962860107,
@@ -25,86 +25,86 @@
     },
     {
       "epoch": 0.16736401673640167,
-      "grad_norm": 7.4630481490893485,
+      "grad_norm": 7.448045779799194,
       "learning_rate": 4.930057285201027e-07,
-      "logits/chosen": -2.7555835247039795,
-      "logits/rejected": -2.746177911758423,
-      "logps/chosen": -271.9296569824219,
-      "logps/rejected": -260.67138671875,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.5347222089767456,
-      "rewards/chosen": 0.0054025910794734955,
-      "rewards/margins": 0.0035073086619377136,
-      "rewards/rejected": 0.0018952824175357819,
+      "logits/chosen": -2.7555572986602783,
+      "logits/rejected": -2.746171474456787,
+      "logps/chosen": -271.9293212890625,
+      "logps/rejected": -260.67010498046875,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.005406347569078207,
+      "rewards/margins": 0.0034980433993041515,
+      "rewards/rejected": 0.0019083041697740555,
       "step": 10
     },
     {
       "epoch": 0.33472803347280333,
-      "grad_norm": 6.708300371266294,
+      "grad_norm": 6.734236245718351,
       "learning_rate": 4.187457503795526e-07,
-      "logits/chosen": -2.787097930908203,
-      "logits/rejected": -2.7682840824127197,
-      "logps/chosen": -261.45843505859375,
-      "logps/rejected": -250.8958740234375,
+      "logits/chosen": -2.7869458198547363,
+      "logits/rejected": -2.768073320388794,
+      "logps/chosen": -261.4452209472656,
+      "logps/rejected": -250.89541625976562,
       "loss": 0.6745,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.03223257511854172,
-      "rewards/margins": 0.040766604244709015,
-      "rewards/rejected": -0.008534022606909275,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 0.03236500173807144,
+      "rewards/margins": 0.04089399054646492,
+      "rewards/rejected": -0.008528990671038628,
       "step": 20
     },
     {
       "epoch": 0.502092050209205,
-      "grad_norm": 7.409956341692702,
+      "grad_norm": 7.437539615124712,
       "learning_rate": 2.8691164100062034e-07,
-      "logits/chosen": -2.7880468368530273,
-      "logits/rejected": -2.7689902782440186,
-      "logps/chosen": -293.8247985839844,
-      "logps/rejected": -254.217529296875,
-      "loss": 0.6487,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.023687291890382767,
-      "rewards/margins": 0.12690749764442444,
-      "rewards/rejected": -0.1505947858095169,
+      "logits/chosen": -2.7878143787384033,
+      "logits/rejected": -2.7688212394714355,
+      "logps/chosen": -293.8128662109375,
+      "logps/rejected": -254.24874877929688,
+      "loss": 0.6485,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.02356739528477192,
+      "rewards/margins": 0.12733949720859528,
+      "rewards/rejected": -0.15090689063072205,
       "step": 30
     },
     {
       "epoch": 0.6694560669456067,
-      "grad_norm": 8.167332409118467,
+      "grad_norm": 8.137783776888698,
       "learning_rate": 1.4248369943086995e-07,
-      "logits/chosen": -2.760864734649658,
-      "logits/rejected": -2.7383790016174316,
-      "logps/chosen": -264.1250915527344,
-      "logps/rejected": -255.5350799560547,
-      "loss": 0.6354,
+      "logits/chosen": -2.760474681854248,
+      "logits/rejected": -2.7379024028778076,
+      "logps/chosen": -264.1443176269531,
+      "logps/rejected": -255.5771026611328,
+      "loss": 0.6353,
       "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": -0.11205615103244781,
-      "rewards/margins": 0.16018818318843842,
-      "rewards/rejected": -0.27224433422088623,
+      "rewards/chosen": -0.11224844306707382,
+      "rewards/margins": 0.16041621565818787,
+      "rewards/rejected": -0.2726646959781647,
       "step": 40
     },
     {
       "epoch": 0.8368200836820083,
-      "grad_norm": 8.545155858110496,
+      "grad_norm": 8.560922020463133,
       "learning_rate": 3.473909705816111e-08,
-      "logits/chosen": -2.770779848098755,
-      "logits/rejected": -2.750847339630127,
-      "logps/chosen": -279.1131286621094,
-      "logps/rejected": -287.27044677734375,
+      "logits/chosen": -2.7703192234039307,
+      "logits/rejected": -2.7502920627593994,
+      "logps/chosen": -279.1363525390625,
+      "logps/rejected": -287.27764892578125,
       "loss": 0.6202,
       "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": -0.22104480862617493,
-      "rewards/margins": 0.13653476536273956,
-      "rewards/rejected": -0.3575795590877533,
+      "rewards/chosen": -0.22127707302570343,
+      "rewards/margins": 0.13637453317642212,
+      "rewards/rejected": -0.35765162110328674,
       "step": 50
     },
     {
       "epoch": 0.9874476987447699,
       "step": 59,
       "total_flos": 0.0,
-      "train_loss": 0.6490178754774191,
-      "train_runtime": 1630.8514,
-      "train_samples_per_second": 9.371,
+      "train_loss": 0.6490362135030455,
+      "train_runtime": 1628.642,
+      "train_samples_per_second": 9.384,
       "train_steps_per_second": 0.036
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd4aa07115a1b0094354c437b369487eb676a555745cf463b02b3eaa8496dd8f
+oid sha256:a96c5176af1b3e2b03b4b13bd4c023071e4d91fb081150a429e0a08207ff8287
 size 6264