Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: selective-pairrm-33045197-mt0
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# selective-pairrm-33045197-mt0
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6825
+- Rewards/chosen: -0.2329
+- Rewards/rejected: -0.2692
+- Rewards/accuracies: 0.6055
+- Rewards/margins: 0.0362
+- Logps/rejected: -417.6746
+- Logps/chosen: -401.4102
+- Logits/rejected: -3.1643
+- Logits/chosen: -3.1708
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6786        | 0.32  | 100  | 0.6868          | -0.0869        | -0.1015          | 0.5547             | 0.0146          | -400.9028      | -386.8027    | -2.8786         | -2.8855       |
+| 0.6615        | 0.64  | 200  | 0.6828          | -0.1851        | -0.2144          | 0.5938             | 0.0294          | -412.2021      | -396.6207    | -3.0607         | -3.0672       |
+| 0.6539        | 0.96  | 300  | 0.6821          | -0.2322        | -0.2693          | 0.6055             | 0.0371          | -417.6892      | -401.3395    | -3.1645         | -3.1709       |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.170811176300049,
+    "eval_logits/rejected": -3.164327621459961,
+    "eval_logps/chosen": -401.41021728515625,
+    "eval_logps/rejected": -417.67462158203125,
+    "eval_loss": 0.682475209236145,
+    "eval_rewards/accuracies": 0.60546875,
+    "eval_rewards/chosen": -0.23294878005981445,
+    "eval_rewards/margins": 0.03622151538729668,
+    "eval_rewards/rejected": -0.26917028427124023,
+    "eval_runtime": 135.9607,
+    "eval_samples": 994,
+    "eval_samples_per_second": 7.355,
+    "eval_steps_per_second": 0.235,
+    "train_loss": 0.6714850996549313,
+    "train_runtime": 5456.9379,
+    "train_samples": 19766,
+    "train_samples_per_second": 3.664,
+    "train_steps_per_second": 0.057
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.170811176300049,
+    "eval_logits/rejected": -3.164327621459961,
+    "eval_logps/chosen": -401.41021728515625,
+    "eval_logps/rejected": -417.67462158203125,
+    "eval_loss": 0.682475209236145,
+    "eval_rewards/accuracies": 0.60546875,
+    "eval_rewards/chosen": -0.23294878005981445,
+    "eval_rewards/margins": 0.03622151538729668,
+    "eval_rewards/rejected": -0.26917028427124023,
+    "eval_runtime": 135.9607,
+    "eval_samples": 994,
+    "eval_samples_per_second": 7.355,
+    "eval_steps_per_second": 0.235
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4b12a337686d300a1ea5af9dda9f8051e5e1d2184116262d309cbe7b94eabcc
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f2e415430a366fe97bc48583377f19dad991c40b94175ede9b06719dce11db7
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b5b81a1d2e3b8743644c27b29c5a694ba37b54a68f3e972bb367265651c039e
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6714850996549313,
+    "train_runtime": 5456.9379,
+    "train_samples": 19766,
+    "train_samples_per_second": 3.664,
+    "train_steps_per_second": 0.057
+}

trainer_state.json

@@ -0,0 +1,526 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984,
+  "eval_steps": 100,
+  "global_step": 312,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5625e-08,
+      "logits/chosen": -2.6282742023468018,
+      "logits/rejected": -2.6322691440582275,
+      "logps/chosen": -400.73358154296875,
+      "logps/rejected": -385.47198486328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.60478138923645,
+      "logits/rejected": -2.5875940322875977,
+      "logps/chosen": -383.8570556640625,
+      "logps/rejected": -403.23974609375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 0.0001632043713470921,
+      "rewards/margins": -6.321006367215887e-05,
+      "rewards/rejected": 0.0002264144568471238,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.618523359298706,
+      "logits/rejected": -2.6062684059143066,
+      "logps/chosen": -400.2876892089844,
+      "logps/rejected": -425.55078125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.000641118735074997,
+      "rewards/margins": -7.16630820534192e-05,
+      "rewards/rejected": 0.0007127817953005433,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.552297830581665,
+      "logits/rejected": -2.536328077316284,
+      "logps/chosen": -371.4658508300781,
+      "logps/rejected": -386.770751953125,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.004528197459876537,
+      "rewards/margins": 0.001459120074287057,
+      "rewards/rejected": 0.0030690771527588367,
+      "step": 30
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989935734988097e-07,
+      "logits/chosen": -2.591301441192627,
+      "logits/rejected": -2.6019530296325684,
+      "logps/chosen": -371.7442321777344,
+      "logps/rejected": -381.6260986328125,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0025075082667171955,
+      "rewards/margins": 0.0020256205461919308,
+      "rewards/rejected": 0.00048188763321377337,
+      "step": 40
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.949188496058089e-07,
+      "logits/chosen": -2.6773521900177,
+      "logits/rejected": -2.6721460819244385,
+      "logps/chosen": -387.83477783203125,
+      "logps/rejected": -402.6687927246094,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.011017683893442154,
+      "rewards/margins": 0.013887738808989525,
+      "rewards/rejected": -0.002870055614039302,
+      "step": 50
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.877641290737883e-07,
+      "logits/chosen": -2.7053120136260986,
+      "logits/rejected": -2.7033491134643555,
+      "logps/chosen": -391.2000427246094,
+      "logps/rejected": -417.63751220703125,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0010398384183645248,
+      "rewards/margins": 0.009658296592533588,
+      "rewards/rejected": -0.008618457242846489,
+      "step": 60
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.776193866647039e-07,
+      "logits/chosen": -2.7314000129699707,
+      "logits/rejected": -2.742297649383545,
+      "logps/chosen": -418.1868591308594,
+      "logps/rejected": -433.50653076171875,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.015087150037288666,
+      "rewards/margins": 0.02047603204846382,
+      "rewards/rejected": -0.03556318208575249,
+      "step": 70
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.646121984004665e-07,
+      "logits/chosen": -2.7759556770324707,
+      "logits/rejected": -2.7733349800109863,
+      "logps/chosen": -398.1673583984375,
+      "logps/rejected": -409.0989685058594,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03768505901098251,
+      "rewards/margins": 0.025433484464883804,
+      "rewards/rejected": -0.06311853975057602,
+      "step": 80
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.489061372204452e-07,
+      "logits/chosen": -2.828214645385742,
+      "logits/rejected": -2.831789255142212,
+      "logps/chosen": -423.423828125,
+      "logps/rejected": -433.41729736328125,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.053007822483778,
+      "rewards/margins": 0.03302832692861557,
+      "rewards/rejected": -0.08603614568710327,
+      "step": 90
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3069871595684787e-07,
+      "logits/chosen": -2.896029233932495,
+      "logits/rejected": -2.893444538116455,
+      "logps/chosen": -397.0558166503906,
+      "logps/rejected": -403.83636474609375,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04668121784925461,
+      "rewards/margins": 0.03920525312423706,
+      "rewards/rejected": -0.08588646352291107,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": -2.885509967803955,
+      "eval_logits/rejected": -2.878624677658081,
+      "eval_logps/chosen": -386.8027038574219,
+      "eval_logps/rejected": -400.90277099609375,
+      "eval_loss": 0.6868489980697632,
+      "eval_rewards/accuracies": 0.5546875,
+      "eval_rewards/chosen": -0.0868735983967781,
+      "eval_rewards/margins": 0.014578516595065594,
+      "eval_rewards/rejected": -0.10145211219787598,
+      "eval_runtime": 137.4322,
+      "eval_samples_per_second": 7.276,
+      "eval_steps_per_second": 0.233,
+      "step": 100
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.10218903496256e-07,
+      "logits/chosen": -2.863513469696045,
+      "logits/rejected": -2.8547675609588623,
+      "logps/chosen": -425.29986572265625,
+      "logps/rejected": -426.5914611816406,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09437432140111923,
+      "rewards/margins": 0.027984386309981346,
+      "rewards/rejected": -0.12235872447490692,
+      "step": 110
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.877242453630256e-07,
+      "logits/chosen": -2.824404001235962,
+      "logits/rejected": -2.8097667694091797,
+      "logps/chosen": -416.0079650878906,
+      "logps/rejected": -433.6048889160156,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.09664727002382278,
+      "rewards/margins": 0.040948256850242615,
+      "rewards/rejected": -0.137595534324646,
+      "step": 120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.634976249348867e-07,
+      "logits/chosen": -2.8867764472961426,
+      "logits/rejected": -2.891484498977661,
+      "logps/chosen": -395.841552734375,
+      "logps/rejected": -422.6394958496094,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.11936166137456894,
+      "rewards/margins": 0.03277095407247543,
+      "rewards/rejected": -0.15213260054588318,
+      "step": 130
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.378437060203357e-07,
+      "logits/chosen": -2.8144431114196777,
+      "logits/rejected": -2.7991340160369873,
+      "logps/chosen": -436.4762268066406,
+      "logps/rejected": -477.21624755859375,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.15145839750766754,
+      "rewards/margins": 0.03745008260011673,
+      "rewards/rejected": -0.18890848755836487,
+      "step": 140
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.110851015344735e-07,
+      "logits/chosen": -2.962568759918213,
+      "logits/rejected": -2.957019090652466,
+      "logps/chosen": -405.751708984375,
+      "logps/rejected": -443.404296875,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.12555965781211853,
+      "rewards/margins": 0.05261915922164917,
+      "rewards/rejected": -0.1781788021326065,
+      "step": 150
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8355831645441387e-07,
+      "logits/chosen": -2.9129347801208496,
+      "logits/rejected": -2.9059901237487793,
+      "logps/chosen": -415.2442321777344,
+      "logps/rejected": -437.210693359375,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.17596140503883362,
+      "rewards/margins": 0.05255354568362236,
+      "rewards/rejected": -0.22851495444774628,
+      "step": 160
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5560951607395126e-07,
+      "logits/chosen": -2.997249126434326,
+      "logits/rejected": -3.0022037029266357,
+      "logps/chosen": -408.9958801269531,
+      "logps/rejected": -433.72076416015625,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16367743909358978,
+      "rewards/margins": 0.07268272340297699,
+      "rewards/rejected": -0.23636016249656677,
+      "step": 170
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2759017277414164e-07,
+      "logits/chosen": -3.01900315284729,
+      "logits/rejected": -3.011901378631592,
+      "logps/chosen": -409.57940673828125,
+      "logps/rejected": -429.5187072753906,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11230169236660004,
+      "rewards/margins": 0.061531912535429,
+      "rewards/rejected": -0.17383362352848053,
+      "step": 180
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.998526460541818e-07,
+      "logits/chosen": -3.0185275077819824,
+      "logits/rejected": -2.9947714805603027,
+      "logps/chosen": -423.85589599609375,
+      "logps/rejected": -451.7948303222656,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.16302022337913513,
+      "rewards/margins": 0.05766066908836365,
+      "rewards/rejected": -0.22068090736865997,
+      "step": 190
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626315e-07,
+      "logits/chosen": -3.038890838623047,
+      "logits/rejected": -3.028480052947998,
+      "logps/chosen": -393.0187072753906,
+      "logps/rejected": -410.3246154785156,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.19790220260620117,
+      "rewards/margins": 0.04545364901423454,
+      "rewards/rejected": -0.24335582554340363,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": -3.067202568054199,
+      "eval_logits/rejected": -3.0606539249420166,
+      "eval_logps/chosen": -396.62066650390625,
+      "eval_logps/rejected": -412.20208740234375,
+      "eval_loss": 0.6828470826148987,
+      "eval_rewards/accuracies": 0.59375,
+      "eval_rewards/chosen": -0.1850530207157135,
+      "eval_rewards/margins": 0.029392333701252937,
+      "eval_rewards/rejected": -0.2144453525543213,
+      "eval_runtime": 136.2085,
+      "eval_samples_per_second": 7.342,
+      "eval_steps_per_second": 0.235,
+      "step": 200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4661037375836987e-07,
+      "logits/chosen": -3.0557608604431152,
+      "logits/rejected": -3.044783353805542,
+      "logps/chosen": -411.9776916503906,
+      "logps/rejected": -446.8324279785156,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1823655068874359,
+      "rewards/margins": 0.09753072261810303,
+      "rewards/rejected": -0.27989625930786133,
+      "step": 210
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2177518064852348e-07,
+      "logits/chosen": -3.064290761947632,
+      "logits/rejected": -3.0677173137664795,
+      "logps/chosen": -410.56231689453125,
+      "logps/rejected": -445.7730407714844,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.161437526345253,
+      "rewards/margins": 0.10264303535223007,
+      "rewards/rejected": -0.26408058404922485,
+      "step": 220
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.855248903979505e-08,
+      "logits/chosen": -3.098358392715454,
+      "logits/rejected": -3.089136838912964,
+      "logps/chosen": -410.40423583984375,
+      "logps/rejected": -428.3938903808594,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.13886065781116486,
+      "rewards/margins": 0.08097346127033234,
+      "rewards/rejected": -0.2198341190814972,
+      "step": 230
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.723433775328384e-08,
+      "logits/chosen": -3.098928928375244,
+      "logits/rejected": -3.0982117652893066,
+      "logps/chosen": -399.87091064453125,
+      "logps/rejected": -431.13653564453125,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.18533360958099365,
+      "rewards/margins": 0.10331207513809204,
+      "rewards/rejected": -0.2886456847190857,
+      "step": 240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.808881491049722e-08,
+      "logits/chosen": -3.0935721397399902,
+      "logits/rejected": -3.0877461433410645,
+      "logps/chosen": -394.9942321777344,
+      "logps/rejected": -434.58526611328125,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19627241790294647,
+      "rewards/margins": 0.14824633300304413,
+      "rewards/rejected": -0.3445187509059906,
+      "step": 250
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1356686569674335e-08,
+      "logits/chosen": -3.061063051223755,
+      "logits/rejected": -3.0640881061553955,
+      "logps/chosen": -450.4789123535156,
+      "logps/rejected": -484.6214904785156,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.2441820651292801,
+      "rewards/margins": 0.12522391974925995,
+      "rewards/rejected": -0.36940592527389526,
+      "step": 260
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.724836895290805e-08,
+      "logits/chosen": -3.136080265045166,
+      "logits/rejected": -3.135943651199341,
+      "logps/chosen": -392.79010009765625,
+      "logps/rejected": -411.1092224121094,
+      "loss": 0.6535,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17221376299858093,
+      "rewards/margins": 0.06714353710412979,
+      "rewards/rejected": -0.23935727775096893,
+      "step": 270
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5941282340065697e-08,
+      "logits/chosen": -3.0997986793518066,
+      "logits/rejected": -3.0961849689483643,
+      "logps/chosen": -427.6429138183594,
+      "logps/rejected": -467.4734802246094,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2281384915113449,
+      "rewards/margins": 0.09002666175365448,
+      "rewards/rejected": -0.318165123462677,
+      "step": 280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.577619905828281e-09,
+      "logits/chosen": -3.1325037479400635,
+      "logits/rejected": -3.138279438018799,
+      "logps/chosen": -394.94561767578125,
+      "logps/rejected": -414.731201171875,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2411809265613556,
+      "rewards/margins": 0.07669158279895782,
+      "rewards/rejected": -0.31787246465682983,
+      "step": 290
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2625595580163247e-09,
+      "logits/chosen": -3.1289730072021484,
+      "logits/rejected": -3.099738597869873,
+      "logps/chosen": -440.1055603027344,
+      "logps/rejected": -485.18463134765625,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1983877271413803,
+      "rewards/margins": 0.13716521859169006,
+      "rewards/rejected": -0.3355529308319092,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": -3.170891523361206,
+      "eval_logits/rejected": -3.164473295211792,
+      "eval_logps/chosen": -401.33953857421875,
+      "eval_logps/rejected": -417.6891784667969,
+      "eval_loss": 0.6821330785751343,
+      "eval_rewards/accuracies": 0.60546875,
+      "eval_rewards/chosen": -0.23224163055419922,
+      "eval_rewards/margins": 0.03707445412874222,
+      "eval_rewards/rejected": -0.2693161070346832,
+      "eval_runtime": 136.3294,
+      "eval_samples_per_second": 7.335,
+      "eval_steps_per_second": 0.235,
+      "step": 300
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.294126437336733e-11,
+      "logits/chosen": -3.1333985328674316,
+      "logits/rejected": -3.1103413105010986,
+      "logps/chosen": -421.52532958984375,
+      "logps/rejected": -455.3125,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2057611495256424,
+      "rewards/margins": 0.10267385095357895,
+      "rewards/rejected": -0.30843502283096313,
+      "step": 310
+    },
+    {
+      "epoch": 1.0,
+      "step": 312,
+      "total_flos": 0.0,
+      "train_loss": 0.6714850996549313,
+      "train_runtime": 5456.9379,
+      "train_samples_per_second": 3.664,
+      "train_steps_per_second": 0.057
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}