End of training

README.md

@@ -3,9 +3,10 @@ base_model: meta-llama/Llama-3.1-8B-Instruct
 library_name: peft
 license: llama3.1
 tags:
+- llama-factory
+- lora
 - trl
 - dpo
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Llama-3.1-8B-Instruct-SAA-Half
@@ -17,7 +18,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Llama-3.1-8B-Instruct-SAA-Half
 
-This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on the bct_non_cot_dpo_500 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9151
+- Rewards/chosen: -0.0869
+- Rewards/rejected: -0.1296
+- Rewards/accuracies: 0.8400
+- Rewards/margins: 0.0426
+- Logps/rejected: -1.2958
+- Logps/chosen: -0.8695
+- Logits/rejected: -0.4842
+- Logits/chosen: -0.4099
+- Sft Loss: 0.1050
+- Odds Ratio Loss: 8.1010
 
 ## Model description
 

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.4098573923110962,
+    "eval_logits/rejected": -0.48421889543533325,
+    "eval_logps/chosen": -0.8694608807563782,
+    "eval_logps/rejected": -1.295823335647583,
+    "eval_loss": 0.9151002764701843,
+    "eval_odds_ratio_loss": 8.100966453552246,
+    "eval_rewards/accuracies": 0.8399999737739563,
+    "eval_rewards/chosen": -0.08694608509540558,
+    "eval_rewards/margins": 0.04263625293970108,
+    "eval_rewards/rejected": -0.12958233058452606,
+    "eval_runtime": 2.0333,
+    "eval_samples_per_second": 24.59,
+    "eval_sft_loss": 0.10500368475914001,
+    "eval_steps_per_second": 12.295,
+    "total_flos": 1.516498186272768e+16,
+    "train_loss": 1.23334006468455,
+    "train_runtime": 194.0073,
+    "train_samples_per_second": 6.959,
+    "train_steps_per_second": 0.433
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.4098573923110962,
+    "eval_logits/rejected": -0.48421889543533325,
+    "eval_logps/chosen": -0.8694608807563782,
+    "eval_logps/rejected": -1.295823335647583,
+    "eval_loss": 0.9151002764701843,
+    "eval_odds_ratio_loss": 8.100966453552246,
+    "eval_rewards/accuracies": 0.8399999737739563,
+    "eval_rewards/chosen": -0.08694608509540558,
+    "eval_rewards/margins": 0.04263625293970108,
+    "eval_rewards/rejected": -0.12958233058452606,
+    "eval_runtime": 2.0333,
+    "eval_samples_per_second": 24.59,
+    "eval_sft_loss": 0.10500368475914001,
+    "eval_steps_per_second": 12.295
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.986666666666667,
+    "total_flos": 1.516498186272768e+16,
+    "train_loss": 1.23334006468455,
+    "train_runtime": 194.0073,
+    "train_samples_per_second": 6.959,
+    "train_steps_per_second": 0.433
+}

trainer_state.json

@@ -0,0 +1,178 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.986666666666667,
+  "eval_steps": 500,
+  "global_step": 84,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 8.846159934997559,
+      "learning_rate": 4.997807075247147e-06,
+      "logits/chosen": -0.43419259786605835,
+      "logits/rejected": -0.5061447024345398,
+      "logps/chosen": -1.7686179876327515,
+      "logps/rejected": -2.1890695095062256,
+      "loss": 1.8236,
+      "odds_ratio_loss": 15.943506240844727,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17686179280281067,
+      "rewards/margins": 0.042045146226882935,
+      "rewards/rejected": -0.2189069539308548,
+      "sft_loss": 0.22924074530601501,
+      "step": 10
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 6.154786586761475,
+      "learning_rate": 4.7392794005985324e-06,
+      "logits/chosen": -0.36880916357040405,
+      "logits/rejected": -0.45593205094337463,
+      "logps/chosen": -1.6094945669174194,
+      "logps/rejected": -2.0511698722839355,
+      "loss": 1.6628,
+      "odds_ratio_loss": 14.580706596374512,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.1609494537115097,
+      "rewards/margins": 0.04416752606630325,
+      "rewards/rejected": -0.20511698722839355,
+      "sft_loss": 0.20470395684242249,
+      "step": 20
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 5.7998881340026855,
+      "learning_rate": 4.093559974371725e-06,
+      "logits/chosen": -0.41758084297180176,
+      "logits/rejected": -0.5090541839599609,
+      "logps/chosen": -1.4586153030395508,
+      "logps/rejected": -1.9464032649993896,
+      "loss": 1.5084,
+      "odds_ratio_loss": 13.208415031433105,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.14586155116558075,
+      "rewards/margins": 0.04877878352999687,
+      "rewards/rejected": -0.19464033842086792,
+      "sft_loss": 0.18757374584674835,
+      "step": 30
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "grad_norm": 6.195462703704834,
+      "learning_rate": 3.1722995515381644e-06,
+      "logits/chosen": -0.4006771445274353,
+      "logits/rejected": -0.47693929076194763,
+      "logps/chosen": -1.2410690784454346,
+      "logps/rejected": -1.7496509552001953,
+      "loss": 1.2886,
+      "odds_ratio_loss": 11.358288764953613,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.12410689890384674,
+      "rewards/margins": 0.05085818096995354,
+      "rewards/rejected": -0.17496509850025177,
+      "sft_loss": 0.1527349054813385,
+      "step": 40
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 5.72018575668335,
+      "learning_rate": 2.134792428593971e-06,
+      "logits/chosen": -0.40342792868614197,
+      "logits/rejected": -0.47864627838134766,
+      "logps/chosen": -1.026789903640747,
+      "logps/rejected": -1.4996291399002075,
+      "loss": 1.0765,
+      "odds_ratio_loss": 9.527546882629395,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10267897695302963,
+      "rewards/margins": 0.047283925116062164,
+      "rewards/rejected": -0.14996293187141418,
+      "sft_loss": 0.12372473627328873,
+      "step": 50
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 4.517279624938965,
+      "learning_rate": 1.160433012552508e-06,
+      "logits/chosen": -0.4058423936367035,
+      "logits/rejected": -0.49612194299697876,
+      "logps/chosen": -0.8940722346305847,
+      "logps/rejected": -1.3635480403900146,
+      "loss": 0.9456,
+      "odds_ratio_loss": 8.374347686767578,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.08940722793340683,
+      "rewards/margins": 0.046947579830884933,
+      "rewards/rejected": -0.13635481894016266,
+      "sft_loss": 0.10813729465007782,
+      "step": 60
+    },
+    {
+      "epoch": 2.488888888888889,
+      "grad_norm": 3.7044575214385986,
+      "learning_rate": 4.1769689822475147e-07,
+      "logits/chosen": -0.41305217146873474,
+      "logits/rejected": -0.4884258210659027,
+      "logps/chosen": -0.8132768869400024,
+      "logps/rejected": -1.3795119524002075,
+      "loss": 0.8557,
+      "odds_ratio_loss": 7.654118537902832,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.08132769167423248,
+      "rewards/margins": 0.05662349984049797,
+      "rewards/rejected": -0.13795119524002075,
+      "sft_loss": 0.09026216715574265,
+      "step": 70
+    },
+    {
+      "epoch": 2.8444444444444446,
+      "grad_norm": 3.704728603363037,
+      "learning_rate": 3.5009907323737826e-08,
+      "logits/chosen": -0.4267275929450989,
+      "logits/rejected": -0.5049682855606079,
+      "logps/chosen": -0.7886781096458435,
+      "logps/rejected": -1.2518656253814697,
+      "loss": 0.8364,
+      "odds_ratio_loss": 7.4754438400268555,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.07886780798435211,
+      "rewards/margins": 0.046318747103214264,
+      "rewards/rejected": -0.12518656253814697,
+      "sft_loss": 0.08889047056436539,
+      "step": 80
+    },
+    {
+      "epoch": 2.986666666666667,
+      "step": 84,
+      "total_flos": 1.516498186272768e+16,
+      "train_loss": 1.23334006468455,
+      "train_runtime": 194.0073,
+      "train_samples_per_second": 6.959,
+      "train_steps_per_second": 0.433
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 84,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.516498186272768e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}