Model save

README.md

@@ -2,14 +2,10 @@
 license: mit
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
 base_model: microsoft/phi-2
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: phi-2-gpo-test-longest-iter-1
   results: []
@@ -20,7 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # phi-2-gpo-test-longest-iter-1
 
-This model is a fine-tuned version of [DUAL-GPO/phi-2-gpo-test-longest-iter-0](https://huggingface.co/DUAL-GPO/phi-2-gpo-test-longest-iter-0) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0108
+- Rewards/chosen: -0.0007
+- Rewards/rejected: -0.0005
+- Rewards/accuracies: 0.4995
+- Rewards/margins: -0.0002
+- Logps/rejected: -278.7133
+- Logps/chosen: -306.4168
+- Logits/rejected: 0.0867
+- Logits/chosen: -0.0107
 
 ## Model description
 
@@ -53,6 +59,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0103        | 1.6   | 100  | 0.0108          | -0.0015        | -0.0014          | 0.5010             | -0.0001         | -278.8064      | -306.4953    | 0.0771          | -0.0200       |
 
 
 ### Framework versions

adapter_config.json

@@ -19,9 +19,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "q_proj",
     "dense",
-    "k_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd19c7268edd7a4a10fe46ee5283c8ff487c8c05fa5d543372ff69f41e333f3a
+oid sha256:b8680c357a8e3e01ad9a1e0ad14ff6f46858eb570d2f8d9a60a7311cb045637c
 size 41977616

all_results.json

@@ -1,8 +1,21 @@
 {
-    "epoch": 1.33,
-    "train_loss": 0.0022073863074183464,
-    "train_runtime": 7.0303,
-    "train_samples": 30567,
-    "train_samples_per_second": 2.845,
-    "train_steps_per_second": 0.284
+    "epoch": 1.98,
+    "eval_logits/chosen": -0.010728351771831512,
+    "eval_logits/rejected": 0.08673479408025742,
+    "eval_logps/chosen": -306.4167785644531,
+    "eval_logps/rejected": -278.7133483886719,
+    "eval_loss": 0.01081769447773695,
+    "eval_rewards/accuracies": 0.49950000643730164,
+    "eval_rewards/chosen": -0.0007103218231350183,
+    "eval_rewards/margins": -0.00023564710863865912,
+    "eval_rewards/rejected": -0.00047467477270402014,
+    "eval_runtime": 470.0123,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 4.255,
+    "eval_steps_per_second": 1.064,
+    "train_loss": 0.010314414998696696,
+    "train_runtime": 1522.7247,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.313,
+    "train_steps_per_second": 0.081
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.98,
+    "eval_logits/chosen": -0.010728351771831512,
+    "eval_logits/rejected": 0.08673479408025742,
+    "eval_logps/chosen": -306.4167785644531,
+    "eval_logps/rejected": -278.7133483886719,
+    "eval_loss": 0.01081769447773695,
+    "eval_rewards/accuracies": 0.49950000643730164,
+    "eval_rewards/chosen": -0.0007103218231350183,
+    "eval_rewards/margins": -0.00023564710863865912,
+    "eval_rewards/rejected": -0.00047467477270402014,
+    "eval_runtime": 470.0123,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 4.255,
+    "eval_steps_per_second": 1.064
+}

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.33,
-    "train_loss": 0.0022073863074183464,
-    "train_runtime": 7.0303,
-    "train_samples": 30567,
-    "train_samples_per_second": 2.845,
-    "train_steps_per_second": 0.284
+    "epoch": 1.98,
+    "train_loss": 0.010314414998696696,
+    "train_runtime": 1522.7247,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.313,
+    "train_steps_per_second": 0.081
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.3333333333333333,
+  "epoch": 1.984,
   "eval_steps": 100,
-  "global_step": 2,
+  "global_step": 124,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "learning_rate": 5e-06,
-      "logits/chosen": 0.11644890159368515,
-      "logits/rejected": 0.08266881108283997,
-      "logps/chosen": -386.4171447753906,
-      "logps/rejected": -280.0220031738281,
-      "loss": 0.0033,
+      "epoch": 0.02,
+      "learning_rate": 3.846153846153847e-07,
+      "logits/chosen": 0.48606717586517334,
+      "logits/rejected": 0.39871031045913696,
+      "logps/chosen": -212.65087890625,
+      "logps/rejected": -202.6477508544922,
+      "loss": 0.0102,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,17 +23,201 @@
       "step": 1
     },
     {
-      "epoch": 1.33,
-      "step": 2,
+      "epoch": 0.16,
+      "learning_rate": 3.846153846153847e-06,
+      "logits/chosen": 0.15301527082920074,
+      "logits/rejected": 0.20366798341274261,
+      "logps/chosen": -161.48782348632812,
+      "logps/rejected": -135.7443389892578,
+      "loss": 0.0105,
+      "rewards/accuracies": 0.3194444477558136,
+      "rewards/chosen": -0.002633917611092329,
+      "rewards/margins": -0.0009920704178512096,
+      "rewards/rejected": -0.0016418471932411194,
+      "step": 10
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.951096619903317e-06,
+      "logits/chosen": 0.09449799358844757,
+      "logits/rejected": 0.0444490909576416,
+      "logps/chosen": -182.85354614257812,
+      "logps/rejected": -149.98532104492188,
+      "loss": 0.0104,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.0008511155610904098,
+      "rewards/margins": -0.0002212001709267497,
+      "rewards/rejected": -0.000629915448371321,
+      "step": 20
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.716164218065246e-06,
+      "logits/chosen": 0.19659022986888885,
+      "logits/rejected": 0.24700064957141876,
+      "logps/chosen": -175.32278442382812,
+      "logps/rejected": -143.53839111328125,
+      "loss": 0.0104,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.0011144612217321992,
+      "rewards/margins": -0.000345538865076378,
+      "rewards/rejected": -0.000768922152929008,
+      "step": 30
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.3048902348863116e-06,
+      "logits/chosen": 0.14261968433856964,
+      "logits/rejected": 0.19110862910747528,
+      "logps/chosen": -181.75755310058594,
+      "logps/rejected": -164.41835021972656,
+      "loss": 0.0103,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": -0.0007939288625493646,
+      "rewards/margins": 0.00022867057123221457,
+      "rewards/rejected": -0.0010225994046777487,
+      "step": 40
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 3.7500000000000005e-06,
+      "logits/chosen": 0.07993963360786438,
+      "logits/rejected": 0.1553264558315277,
+      "logps/chosen": -170.30532836914062,
+      "logps/rejected": -144.61050415039062,
+      "loss": 0.0101,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.00015746071585454047,
+      "rewards/margins": 0.0011130261700600386,
+      "rewards/rejected": -0.0009555654833093286,
+      "step": 50
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.0956464785579125e-06,
+      "logits/chosen": 0.18755468726158142,
+      "logits/rejected": 0.17403154075145721,
+      "logps/chosen": -173.29977416992188,
+      "logps/rejected": -144.7869873046875,
+      "loss": 0.0105,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0005446333670988679,
+      "rewards/margins": -0.000993421534076333,
+      "rewards/rejected": 0.00044878822518512607,
+      "step": 60
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 2.39389699200963e-06,
+      "logits/chosen": 0.2639772593975067,
+      "logits/rejected": 0.2394874542951584,
+      "logps/chosen": -188.4938201904297,
+      "logps/rejected": -160.391357421875,
+      "loss": 0.0102,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.00019804532348643988,
+      "rewards/margins": 0.000605274923145771,
+      "rewards/rejected": -0.0008033200865611434,
+      "step": 70
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 1.700590188571887e-06,
+      "logits/chosen": 0.1733393371105194,
+      "logits/rejected": 0.2405407875776291,
+      "logps/chosen": -166.1864471435547,
+      "logps/rejected": -136.4402618408203,
+      "loss": 0.0102,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.0004581899265758693,
+      "rewards/margins": 0.0007379798917099833,
+      "rewards/rejected": -0.0002797898487187922,
+      "step": 80
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 1.0708929268538034e-06,
+      "logits/chosen": 0.16160213947296143,
+      "logits/rejected": 0.13504673540592194,
+      "logps/chosen": -175.88116455078125,
+      "logps/rejected": -145.18658447265625,
+      "loss": 0.01,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": 0.0007930597057566047,
+      "rewards/margins": 0.001615246757864952,
+      "rewards/rejected": -0.0008221869356930256,
+      "step": 90
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 5.549106142039018e-07,
+      "logits/chosen": 0.18513503670692444,
+      "logits/rejected": 0.14850696921348572,
+      "logps/chosen": -177.84588623046875,
+      "logps/rejected": -152.69290161132812,
+      "loss": 0.0103,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": 0.0004619792161975056,
+      "rewards/margins": 0.00023423954553436488,
+      "rewards/rejected": 0.00022773972887080163,
+      "step": 100
+    },
+    {
+      "epoch": 1.6,
+      "eval_logits/chosen": -0.019983915612101555,
+      "eval_logits/rejected": 0.0771104097366333,
+      "eval_logps/chosen": -306.49530029296875,
+      "eval_logps/rejected": -278.806396484375,
+      "eval_loss": 0.010805144906044006,
+      "eval_rewards/accuracies": 0.5009999871253967,
+      "eval_rewards/chosen": -0.0014956285012885928,
+      "eval_rewards/margins": -9.011628571897745e-05,
+      "eval_rewards/rejected": -0.0014055122155696154,
+      "eval_runtime": 470.6682,
+      "eval_samples_per_second": 4.249,
+      "eval_steps_per_second": 1.062,
+      "step": 100
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 1.937002879188285e-07,
+      "logits/chosen": 0.18757307529449463,
+      "logits/rejected": 0.19283342361450195,
+      "logps/chosen": -186.30393981933594,
+      "logps/rejected": -159.83816528320312,
+      "loss": 0.0104,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.0009433077648282051,
+      "rewards/margins": -0.0004140757955610752,
+      "rewards/rejected": 0.0013573834439739585,
+      "step": 110
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 1.6003680950742728e-08,
+      "logits/chosen": 0.07835674285888672,
+      "logits/rejected": 0.13737662136554718,
+      "logps/chosen": -174.80299377441406,
+      "logps/rejected": -154.0167694091797,
+      "loss": 0.0104,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -5.471259282785468e-05,
+      "rewards/margins": -0.0002534462546464056,
+      "rewards/rejected": 0.0001987336145248264,
+      "step": 120
+    },
+    {
+      "epoch": 1.98,
+      "step": 124,
       "total_flos": 0.0,
-      "train_loss": 0.0022073863074183464,
-      "train_runtime": 7.0303,
-      "train_samples_per_second": 2.845,
-      "train_steps_per_second": 0.284
+      "train_loss": 0.010314414998696696,
+      "train_runtime": 1522.7247,
+      "train_samples_per_second": 1.313,
+      "train_steps_per_second": 0.081
     }
   ],
   "logging_steps": 10,
-  "max_steps": 2,
+  "max_steps": 124,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cc55d47bd1a8d39bdfdfe5cc4e911faaa059da740828d4bdc128659fffc85f1
+oid sha256:3eedcaca58d1acbf62f2e929c67a3b555a144c3870a46aa43ff58d4b0da18c70
 size 5880