Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: kaist-sft-mistral-OHP-15k-Stratified-1-fix
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# kaist-sft-mistral-OHP-15k-Stratified-1-fix
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.38.1
+- Pytorch 2.1.2+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.7502862380468454,
+    "train_runtime": 21621.4849,
+    "train_samples": 14928,
+    "train_samples_per_second": 2.071,
+    "train_steps_per_second": 0.032
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd9e471c303d394508f3abebefb2e00a379b6d67eea82f443c8de6f8778cf930
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06afcf76c5965bc7ee0af05a0272979a53b84877257234234fe87d9b83be94e0
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4719c82ae5383020e9d86bda01ff023a30f870e7af83254730e391d896cfef
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.7502862380468454,
+    "train_runtime": 21621.4849,
+    "train_samples": 14928,
+    "train_samples_per_second": 2.071,
+    "train_steps_per_second": 0.032
+}

trainer_state.json

@@ -0,0 +1,1272 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.996784565916399,
+  "eval_steps": 500,
+  "global_step": 699,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 26.875,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.06212496757507324,
+      "log_odds_ratio": -0.7014996409416199,
+      "logits/chosen": -2.1857399940490723,
+      "logits/rejected": -2.1817708015441895,
+      "logps/chosen": -0.9498230814933777,
+      "logps/rejected": -0.9784062504768372,
+      "loss": 1.3118,
+      "nll_loss": 1.3238964080810547,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 25.375,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.16793885827064514,
+      "log_odds_ratio": -0.6694984436035156,
+      "logits/chosen": -2.263418674468994,
+      "logits/rejected": -2.186417818069458,
+      "logps/chosen": -0.8150558471679688,
+      "logps/rejected": -0.9238722920417786,
+      "loss": 1.205,
+      "nll_loss": 1.1971018314361572,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 7.125,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.00997834000736475,
+      "log_odds_ratio": -0.7417184114456177,
+      "logits/chosen": -2.3210864067077637,
+      "logits/rejected": -2.2824535369873047,
+      "logps/chosen": -0.7758495807647705,
+      "logps/rejected": -0.7887269258499146,
+      "loss": 1.0752,
+      "nll_loss": 1.1008635759353638,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 6.46875,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.12381462007761002,
+      "log_odds_ratio": -0.6990920305252075,
+      "logits/chosen": -2.387354612350464,
+      "logits/rejected": -2.278778553009033,
+      "logps/chosen": -0.6582767963409424,
+      "logps/rejected": -0.7136567831039429,
+      "loss": 0.9839,
+      "nll_loss": 0.9438871145248413,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 5.8125,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.10633399337530136,
+      "log_odds_ratio": -0.702051043510437,
+      "logits/chosen": -2.3278417587280273,
+      "logits/rejected": -2.264376163482666,
+      "logps/chosen": -0.6687366366386414,
+      "logps/rejected": -0.7335516214370728,
+      "loss": 0.9634,
+      "nll_loss": 0.9985629320144653,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 6.09375,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.03238314390182495,
+      "log_odds_ratio": -0.7480851411819458,
+      "logits/chosen": -2.3533706665039062,
+      "logits/rejected": -2.2710163593292236,
+      "logps/chosen": -0.7110857963562012,
+      "logps/rejected": -0.7359805107116699,
+      "loss": 0.9691,
+      "nll_loss": 0.9716413617134094,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 5.96875,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.03486456722021103,
+      "log_odds_ratio": -0.7344475984573364,
+      "logits/chosen": -2.3200392723083496,
+      "logits/rejected": -2.2653918266296387,
+      "logps/chosen": -0.6566824316978455,
+      "logps/rejected": -0.6836172938346863,
+      "loss": 0.9322,
+      "nll_loss": 0.9105528593063354,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 6.375,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.06068553403019905,
+      "log_odds_ratio": -0.7511457204818726,
+      "logits/chosen": -2.2898621559143066,
+      "logits/rejected": -2.282437324523926,
+      "logps/chosen": -0.6991879343986511,
+      "logps/rejected": -0.7374504208564758,
+      "loss": 0.9628,
+      "nll_loss": 0.9841943979263306,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 5.71875,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.18408647179603577,
+      "log_odds_ratio": -0.6827269792556763,
+      "logits/chosen": -2.3683393001556396,
+      "logits/rejected": -2.2830727100372314,
+      "logps/chosen": -0.6308820843696594,
+      "logps/rejected": -0.7424929141998291,
+      "loss": 0.8944,
+      "nll_loss": 0.9193886518478394,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 5.9375,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.035386841744184494,
+      "log_odds_ratio": -0.7439508438110352,
+      "logits/chosen": -2.35438871383667,
+      "logits/rejected": -2.2569315433502197,
+      "logps/chosen": -0.6244274973869324,
+      "logps/rejected": -0.6488394141197205,
+      "loss": 0.8782,
+      "nll_loss": 0.8835960626602173,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 5.875,
+      "learning_rate": 4.996562390352354e-06,
+      "log_odds_chosen": 0.13479042053222656,
+      "log_odds_ratio": -0.7032457590103149,
+      "logits/chosen": -2.338320255279541,
+      "logits/rejected": -2.259917974472046,
+      "logps/chosen": -0.6392095685005188,
+      "logps/rejected": -0.7337976098060608,
+      "loss": 0.904,
+      "nll_loss": 0.9141031503677368,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 5.125,
+      "learning_rate": 4.986259015137485e-06,
+      "log_odds_chosen": 0.1813308447599411,
+      "log_odds_ratio": -0.6845273971557617,
+      "logits/chosen": -2.3334765434265137,
+      "logits/rejected": -2.208571434020996,
+      "logps/chosen": -0.6667813062667847,
+      "logps/rejected": -0.7594255208969116,
+      "loss": 0.8882,
+      "nll_loss": 0.9459765553474426,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 5.15625,
+      "learning_rate": 4.96911820954103e-06,
+      "log_odds_chosen": 0.11342030763626099,
+      "log_odds_ratio": -0.7043333649635315,
+      "logits/chosen": -2.245288372039795,
+      "logits/rejected": -2.161513328552246,
+      "logps/chosen": -0.6394560933113098,
+      "logps/rejected": -0.7009039521217346,
+      "loss": 0.8905,
+      "nll_loss": 0.8281471133232117,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 5.75,
+      "learning_rate": 4.945187112281936e-06,
+      "log_odds_chosen": 0.0401572659611702,
+      "log_odds_ratio": -0.7479963302612305,
+      "logits/chosen": -2.3580965995788574,
+      "logits/rejected": -2.3082127571105957,
+      "logps/chosen": -0.655213475227356,
+      "logps/rejected": -0.6817822456359863,
+      "loss": 0.9124,
+      "nll_loss": 0.9263037443161011,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 5.59375,
+      "learning_rate": 4.9145315359768575e-06,
+      "log_odds_chosen": 0.08960182219743729,
+      "log_odds_ratio": -0.7190832495689392,
+      "logits/chosen": -2.3376193046569824,
+      "logits/rejected": -2.2543139457702637,
+      "logps/chosen": -0.6171378493309021,
+      "logps/rejected": -0.6685695648193359,
+      "loss": 0.8818,
+      "nll_loss": 0.8352767825126648,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 5.71875,
+      "learning_rate": 4.877235786149681e-06,
+      "log_odds_chosen": 0.001317462301813066,
+      "log_odds_ratio": -0.7520455121994019,
+      "logits/chosen": -2.294182300567627,
+      "logits/rejected": -2.249718189239502,
+      "logps/chosen": -0.5758674144744873,
+      "logps/rejected": -0.5995661020278931,
+      "loss": 0.8668,
+      "nll_loss": 0.8348628282546997,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 5.71875,
+      "learning_rate": 4.833402429383947e-06,
+      "log_odds_chosen": 0.007824589498341084,
+      "log_odds_ratio": -0.7456755638122559,
+      "logits/chosen": -2.394925832748413,
+      "logits/rejected": -2.3120040893554688,
+      "logps/chosen": -0.6642740368843079,
+      "logps/rejected": -0.6849616169929504,
+      "loss": 0.8936,
+      "nll_loss": 0.8833368420600891,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 5.4375,
+      "learning_rate": 4.783152011255739e-06,
+      "log_odds_chosen": 0.06460268795490265,
+      "log_odds_ratio": -0.7266359925270081,
+      "logits/chosen": -2.2752573490142822,
+      "logits/rejected": -2.2238731384277344,
+      "logps/chosen": -0.6234402656555176,
+      "logps/rejected": -0.6678077578544617,
+      "loss": 0.8939,
+      "nll_loss": 0.8869989514350891,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 5.46875,
+      "learning_rate": 4.726622724822781e-06,
+      "log_odds_chosen": 0.07030437141656876,
+      "log_odds_ratio": -0.7147783041000366,
+      "logits/chosen": -2.2865307331085205,
+      "logits/rejected": -2.2341747283935547,
+      "logps/chosen": -0.6607599854469299,
+      "logps/rejected": -0.7076841592788696,
+      "loss": 0.8779,
+      "nll_loss": 0.8933170437812805,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 5.125,
+      "learning_rate": 4.663970030581408e-06,
+      "log_odds_chosen": 0.11411430686712265,
+      "log_odds_ratio": -0.7219884395599365,
+      "logits/chosen": -2.235715389251709,
+      "logits/rejected": -2.195624828338623,
+      "logps/chosen": -0.5864480137825012,
+      "logps/rejected": -0.6477882266044617,
+      "loss": 0.8489,
+      "nll_loss": 0.8421682119369507,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 5.28125,
+      "learning_rate": 4.59536622893656e-06,
+      "log_odds_chosen": -0.019357014447450638,
+      "log_odds_ratio": -0.7726019024848938,
+      "logits/chosen": -2.271784782409668,
+      "logits/rejected": -2.188135862350464,
+      "logps/chosen": -0.6931017637252808,
+      "logps/rejected": -0.7073479890823364,
+      "loss": 0.8628,
+      "nll_loss": 0.8622463345527649,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 5.4375,
+      "learning_rate": 4.520999986360555e-06,
+      "log_odds_chosen": 0.07649385929107666,
+      "log_odds_ratio": -0.7158172726631165,
+      "logits/chosen": -2.250640392303467,
+      "logits/rejected": -2.2138304710388184,
+      "logps/chosen": -0.6532458662986755,
+      "logps/rejected": -0.6902228593826294,
+      "loss": 0.8572,
+      "nll_loss": 0.868952751159668,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 5.65625,
+      "learning_rate": 4.441075816543745e-06,
+      "log_odds_chosen": 0.030604243278503418,
+      "log_odds_ratio": -0.7345600128173828,
+      "logits/chosen": -2.3321053981781006,
+      "logits/rejected": -2.2295913696289062,
+      "logps/chosen": -0.6273137331008911,
+      "logps/rejected": -0.6507210731506348,
+      "loss": 0.8789,
+      "nll_loss": 0.8177094459533691,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 230
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 5.84375,
+      "learning_rate": 4.355813517963924e-06,
+      "log_odds_chosen": 0.40961089730262756,
+      "log_odds_ratio": -0.5910171270370483,
+      "logits/chosen": -2.3318493366241455,
+      "logits/rejected": -2.2643091678619385,
+      "logps/chosen": -0.4916546940803528,
+      "logps/rejected": -0.6644417643547058,
+      "loss": 0.758,
+      "nll_loss": 0.740388810634613,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 240
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 6.25,
+      "learning_rate": 4.265447569421234e-06,
+      "log_odds_chosen": 0.3989773392677307,
+      "log_odds_ratio": -0.590238630771637,
+      "logits/chosen": -2.3867685794830322,
+      "logits/rejected": -2.3056893348693848,
+      "logps/chosen": -0.5096999406814575,
+      "logps/rejected": -0.6727738976478577,
+      "loss": 0.6881,
+      "nll_loss": 0.7256556153297424,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 250
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 6.0625,
+      "learning_rate": 4.170226485200899e-06,
+      "log_odds_chosen": 0.48682594299316406,
+      "log_odds_ratio": -0.5600502490997314,
+      "logits/chosen": -2.3787410259246826,
+      "logits/rejected": -2.30033016204834,
+      "logps/chosen": -0.47443705797195435,
+      "logps/rejected": -0.6933677792549133,
+      "loss": 0.725,
+      "nll_loss": 0.6927866339683533,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 260
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 5.875,
+      "learning_rate": 4.070412131637139e-06,
+      "log_odds_chosen": 0.38443347811698914,
+      "log_odds_ratio": -0.6165519952774048,
+      "logits/chosen": -2.3166797161102295,
+      "logits/rejected": -2.23835825920105,
+      "logps/chosen": -0.4729565978050232,
+      "logps/rejected": -0.623404324054718,
+      "loss": 0.719,
+      "nll_loss": 0.689362645149231,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 270
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 6.09375,
+      "learning_rate": 3.966279006957781e-06,
+      "log_odds_chosen": 0.4762851297855377,
+      "log_odds_ratio": -0.5657437443733215,
+      "logits/chosen": -2.3141160011291504,
+      "logits/rejected": -2.2335238456726074,
+      "logps/chosen": -0.5018330812454224,
+      "logps/rejected": -0.7071572542190552,
+      "loss": 0.7292,
+      "nll_loss": 0.7584232687950134,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 280
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 5.625,
+      "learning_rate": 3.858113486390056e-06,
+      "log_odds_chosen": 0.461935818195343,
+      "log_odds_ratio": -0.5941855907440186,
+      "logits/chosen": -2.339001178741455,
+      "logits/rejected": -2.2689056396484375,
+      "logps/chosen": -0.48000845313072205,
+      "logps/rejected": -0.7079610228538513,
+      "loss": 0.7325,
+      "nll_loss": 0.7081630229949951,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 290
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 6.375,
+      "learning_rate": 3.7462130346036e-06,
+      "log_odds_chosen": 0.4929097592830658,
+      "log_odds_ratio": -0.563489556312561,
+      "logits/chosen": -2.3332343101501465,
+      "logits/rejected": -2.2668535709381104,
+      "logps/chosen": -0.4893871247768402,
+      "logps/rejected": -0.7047632336616516,
+      "loss": 0.6818,
+      "nll_loss": 0.7156692147254944,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 300
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 5.625,
+      "learning_rate": 3.6308853876565232e-06,
+      "log_odds_chosen": 0.4562360346317291,
+      "log_odds_ratio": -0.5689755082130432,
+      "logits/chosen": -2.3754940032958984,
+      "logits/rejected": -2.27487850189209,
+      "logps/chosen": -0.4727197289466858,
+      "logps/rejected": -0.6664949655532837,
+      "loss": 0.7119,
+      "nll_loss": 0.6855541467666626,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 310
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 5.375,
+      "learning_rate": 3.512447706694254e-06,
+      "log_odds_chosen": 0.3677065372467041,
+      "log_odds_ratio": -0.6210616230964661,
+      "logits/chosen": -2.3326685428619385,
+      "logits/rejected": -2.284008502960205,
+      "logps/chosen": -0.5006684064865112,
+      "logps/rejected": -0.6665970087051392,
+      "loss": 0.7148,
+      "nll_loss": 0.7629369497299194,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 320
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 5.34375,
+      "learning_rate": 3.3912257057285684e-06,
+      "log_odds_chosen": 0.5201979279518127,
+      "log_odds_ratio": -0.5558315515518188,
+      "logits/chosen": -2.35974383354187,
+      "logits/rejected": -2.2589011192321777,
+      "logps/chosen": -0.44807687401771545,
+      "logps/rejected": -0.6775830984115601,
+      "loss": 0.7127,
+      "nll_loss": 0.6746495962142944,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 330
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 5.46875,
+      "learning_rate": 3.2675527558954897e-06,
+      "log_odds_chosen": 0.4437088966369629,
+      "log_odds_ratio": -0.5744566321372986,
+      "logits/chosen": -2.3658008575439453,
+      "logits/rejected": -2.302891969680786,
+      "logps/chosen": -0.5020125508308411,
+      "logps/rejected": -0.703687310218811,
+      "loss": 0.7293,
+      "nll_loss": 0.7448792457580566,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 340
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 6.21875,
+      "learning_rate": 3.1417689686554144e-06,
+      "log_odds_chosen": 0.4869805872440338,
+      "log_odds_ratio": -0.5718838572502136,
+      "logits/chosen": -2.3735198974609375,
+      "logits/rejected": -2.3357536792755127,
+      "logps/chosen": -0.4905489385128021,
+      "logps/rejected": -0.7079612612724304,
+      "loss": 0.7228,
+      "nll_loss": 0.7263522148132324,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 350
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 5.25,
+      "learning_rate": 3.0142202604567724e-06,
+      "log_odds_chosen": 0.4622114300727844,
+      "log_odds_ratio": -0.580187201499939,
+      "logits/chosen": -2.4162421226501465,
+      "logits/rejected": -2.32672119140625,
+      "logps/chosen": -0.4755636155605316,
+      "logps/rejected": -0.6753752827644348,
+      "loss": 0.6993,
+      "nll_loss": 0.7006024122238159,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 360
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 5.90625,
+      "learning_rate": 2.8852574014354394e-06,
+      "log_odds_chosen": 0.4685123562812805,
+      "log_odds_ratio": -0.5639849901199341,
+      "logits/chosen": -2.3573315143585205,
+      "logits/rejected": -2.2810652256011963,
+      "logps/chosen": -0.45586276054382324,
+      "logps/rejected": -0.650667667388916,
+      "loss": 0.6937,
+      "nll_loss": 0.6792919039726257,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 370
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 5.46875,
+      "learning_rate": 2.7552350507661063e-06,
+      "log_odds_chosen": 0.32628968358039856,
+      "log_odds_ratio": -0.6266478300094604,
+      "logits/chosen": -2.3865950107574463,
+      "logits/rejected": -2.2999796867370605,
+      "logps/chosen": -0.5150719285011292,
+      "logps/rejected": -0.6419684290885925,
+      "loss": 0.7454,
+      "nll_loss": 0.7746638059616089,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 380
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 5.65625,
+      "learning_rate": 2.6245107813184286e-06,
+      "log_odds_chosen": 0.4124643802642822,
+      "log_odds_ratio": -0.6084017157554626,
+      "logits/chosen": -2.300006151199341,
+      "logits/rejected": -2.2486844062805176,
+      "logps/chosen": -0.500686526298523,
+      "logps/rejected": -0.7045504450798035,
+      "loss": 0.7104,
+      "nll_loss": 0.7621539235115051,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 390
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 6.3125,
+      "learning_rate": 2.493444096300273e-06,
+      "log_odds_chosen": 0.3271673321723938,
+      "log_odds_ratio": -0.6363809704780579,
+      "logits/chosen": -2.332576274871826,
+      "logits/rejected": -2.2507784366607666,
+      "logps/chosen": -0.5043616890907288,
+      "logps/rejected": -0.6402055621147156,
+      "loss": 0.7316,
+      "nll_loss": 0.7292400598526001,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 5.40625,
+      "learning_rate": 2.3623954405923636e-06,
+      "log_odds_chosen": 0.35860806703567505,
+      "log_odds_ratio": -0.6040534973144531,
+      "logits/chosen": -2.3241639137268066,
+      "logits/rejected": -2.2687346935272217,
+      "logps/chosen": -0.4667055010795593,
+      "logps/rejected": -0.6222271919250488,
+      "loss": 0.7055,
+      "nll_loss": 0.6745852828025818,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 410
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 5.4375,
+      "learning_rate": 2.2317252094932383e-06,
+      "log_odds_chosen": 0.4658966064453125,
+      "log_odds_ratio": -0.5793284177780151,
+      "logits/chosen": -2.3523142337799072,
+      "logits/rejected": -2.2792649269104004,
+      "logps/chosen": -0.4811071753501892,
+      "logps/rejected": -0.6847606897354126,
+      "loss": 0.6927,
+      "nll_loss": 0.7262173295021057,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 420
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 5.75,
+      "learning_rate": 2.1017927576005657e-06,
+      "log_odds_chosen": 0.4586619436740875,
+      "log_odds_ratio": -0.5749837756156921,
+      "logits/chosen": -2.309678554534912,
+      "logits/rejected": -2.2428524494171143,
+      "logps/chosen": -0.4688965380191803,
+      "logps/rejected": -0.6654346585273743,
+      "loss": 0.6911,
+      "nll_loss": 0.7090088129043579,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 430
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 5.875,
+      "learning_rate": 1.9729554105544816e-06,
+      "log_odds_chosen": 0.39126816391944885,
+      "log_odds_ratio": -0.5965334177017212,
+      "logits/chosen": -2.246513605117798,
+      "logits/rejected": -2.1855998039245605,
+      "logps/chosen": -0.5077515840530396,
+      "logps/rejected": -0.685882031917572,
+      "loss": 0.73,
+      "nll_loss": 0.696818470954895,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 440
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 5.78125,
+      "learning_rate": 1.8455674823607312e-06,
+      "log_odds_chosen": 0.2512452006340027,
+      "log_odds_ratio": -0.6640199422836304,
+      "logits/chosen": -2.342719078063965,
+      "logits/rejected": -2.274512529373169,
+      "logps/chosen": -0.521196722984314,
+      "logps/rejected": -0.6386003494262695,
+      "loss": 0.7091,
+      "nll_loss": 0.7769466638565063,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 450
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 5.46875,
+      "learning_rate": 1.7199793009960766e-06,
+      "log_odds_chosen": 0.4366021156311035,
+      "log_odds_ratio": -0.5971043705940247,
+      "logits/chosen": -2.2552707195281982,
+      "logits/rejected": -2.2170627117156982,
+      "logps/chosen": -0.5236691832542419,
+      "logps/rejected": -0.7121491432189941,
+      "loss": 0.7011,
+      "nll_loss": 0.7103704214096069,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 460
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 5.9375,
+      "learning_rate": 1.5965362449756317e-06,
+      "log_odds_chosen": 0.5843161344528198,
+      "log_odds_ratio": -0.5287169218063354,
+      "logits/chosen": -2.3433125019073486,
+      "logits/rejected": -2.2500529289245605,
+      "logps/chosen": -0.43613916635513306,
+      "logps/rejected": -0.6702120304107666,
+      "loss": 0.6759,
+      "nll_loss": 0.6880279183387756,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 470
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 6.5,
+      "learning_rate": 1.4755777935316412e-06,
+      "log_odds_chosen": 0.8127249479293823,
+      "log_odds_ratio": -0.46383658051490784,
+      "logits/chosen": -2.3992159366607666,
+      "logits/rejected": -2.2700352668762207,
+      "logps/chosen": -0.4094364047050476,
+      "logps/rejected": -0.7208075523376465,
+      "loss": 0.6001,
+      "nll_loss": 0.5964864492416382,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 480
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 6.03125,
+      "learning_rate": 1.3574365930158272e-06,
+      "log_odds_chosen": 0.7765734791755676,
+      "log_odds_ratio": -0.4649999141693115,
+      "logits/chosen": -2.294707775115967,
+      "logits/rejected": -2.244535446166992,
+      "logps/chosen": -0.3731532692909241,
+      "logps/rejected": -0.6663237810134888,
+      "loss": 0.5958,
+      "nll_loss": 0.5779340863227844,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 490
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 5.34375,
+      "learning_rate": 1.242437542092731e-06,
+      "log_odds_chosen": 0.8575220108032227,
+      "log_odds_ratio": -0.42505908012390137,
+      "logits/chosen": -2.2913150787353516,
+      "logits/rejected": -2.2416951656341553,
+      "logps/chosen": -0.37527984380722046,
+      "logps/rejected": -0.6905524730682373,
+      "loss": 0.6326,
+      "nll_loss": 0.623663604259491,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 500
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.1308968982398893e-06,
+      "log_odds_chosen": 0.912135899066925,
+      "log_odds_ratio": -0.41568368673324585,
+      "logits/chosen": -2.313683271408081,
+      "logits/rejected": -2.226602554321289,
+      "logps/chosen": -0.3589690625667572,
+      "logps/rejected": -0.7188132405281067,
+      "loss": 0.5746,
+      "nll_loss": 0.5657839179039001,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 510
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 6.4375,
+      "learning_rate": 1.0231214080120354e-06,
+      "log_odds_chosen": 0.8424990773200989,
+      "log_odds_ratio": -0.4574614465236664,
+      "logits/chosen": -2.2810280323028564,
+      "logits/rejected": -2.2405149936676025,
+      "logps/chosen": -0.39062148332595825,
+      "logps/rejected": -0.705338716506958,
+      "loss": 0.616,
+      "nll_loss": 0.6053126454353333,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 520
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 5.46875,
+      "learning_rate": 9.194074634611577e-07,
+      "log_odds_chosen": 0.8043780326843262,
+      "log_odds_ratio": -0.4463415741920471,
+      "logits/chosen": -2.3421316146850586,
+      "logits/rejected": -2.27047061920166,
+      "logps/chosen": -0.3879227340221405,
+      "logps/rejected": -0.681617259979248,
+      "loss": 0.5906,
+      "nll_loss": 0.6051921844482422,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 530
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 5.71875,
+      "learning_rate": 8.200402870323634e-07,
+      "log_odds_chosen": 0.8168398141860962,
+      "log_odds_ratio": -0.4651332497596741,
+      "logits/chosen": -2.2850687503814697,
+      "logits/rejected": -2.203339099884033,
+      "logps/chosen": -0.4002053141593933,
+      "logps/rejected": -0.7145139575004578,
+      "loss": 0.6095,
+      "nll_loss": 0.5440846681594849,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 540
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 5.53125,
+      "learning_rate": 7.252931471771322e-07,
+      "log_odds_chosen": 0.957280158996582,
+      "log_odds_ratio": -0.4218501150608063,
+      "logits/chosen": -2.3870935440063477,
+      "logits/rejected": -2.3037381172180176,
+      "logps/chosen": -0.3641536235809326,
+      "logps/rejected": -0.7135647535324097,
+      "loss": 0.5756,
+      "nll_loss": 0.5520345568656921,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 550
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 6.0,
+      "learning_rate": 6.354266068411078e-07,
+      "log_odds_chosen": 0.8184865713119507,
+      "log_odds_ratio": -0.4773890972137451,
+      "logits/chosen": -2.2534213066101074,
+      "logits/rejected": -2.22419810295105,
+      "logps/chosen": -0.41363048553466797,
+      "logps/rejected": -0.7267066240310669,
+      "loss": 0.6069,
+      "nll_loss": 0.6007322072982788,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 560
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 5.71875,
+      "learning_rate": 5.50687806893139e-07,
+      "log_odds_chosen": 0.8616418838500977,
+      "log_odds_ratio": -0.44923096895217896,
+      "logits/chosen": -2.276895761489868,
+      "logits/rejected": -2.2783918380737305,
+      "logps/chosen": -0.37030458450317383,
+      "logps/rejected": -0.666776180267334,
+      "loss": 0.614,
+      "nll_loss": 0.6462346911430359,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 570
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 6.1875,
+      "learning_rate": 4.7130978646620807e-07,
+      "log_odds_chosen": 0.9556438326835632,
+      "log_odds_ratio": -0.4390975534915924,
+      "logits/chosen": -2.3128373622894287,
+      "logits/rejected": -2.2027499675750732,
+      "logps/chosen": -0.3874739706516266,
+      "logps/rejected": -0.7368007898330688,
+      "loss": 0.5845,
+      "nll_loss": 0.6006309986114502,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 580
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 5.5,
+      "learning_rate": 3.975108420793819e-07,
+      "log_odds_chosen": 0.8426518440246582,
+      "log_odds_ratio": -0.44841188192367554,
+      "logits/chosen": -2.2656946182250977,
+      "logits/rejected": -2.2316298484802246,
+      "logps/chosen": -0.3601939082145691,
+      "logps/rejected": -0.6829022765159607,
+      "loss": 0.6035,
+      "nll_loss": 0.5817385911941528,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 590
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 6.09375,
+      "learning_rate": 3.294939273032272e-07,
+      "log_odds_chosen": 0.9172550439834595,
+      "log_odds_ratio": -0.41558733582496643,
+      "logits/chosen": -2.251704692840576,
+      "logits/rejected": -2.228672742843628,
+      "logps/chosen": -0.3531130254268646,
+      "logps/rejected": -0.703464925289154,
+      "loss": 0.5694,
+      "nll_loss": 0.554794430732727,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 600
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 5.96875,
+      "learning_rate": 2.6744609461969523e-07,
+      "log_odds_chosen": 0.877754807472229,
+      "log_odds_ratio": -0.4350369870662689,
+      "logits/chosen": -2.3534913063049316,
+      "logits/rejected": -2.2918014526367188,
+      "logps/chosen": -0.37197771668434143,
+      "logps/rejected": -0.7188286781311035,
+      "loss": 0.594,
+      "nll_loss": 0.5847772359848022,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 610
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 5.84375,
+      "learning_rate": 2.1153798101138405e-07,
+      "log_odds_chosen": 0.9580931663513184,
+      "log_odds_ratio": -0.4062252938747406,
+      "logits/chosen": -2.2987186908721924,
+      "logits/rejected": -2.2561445236206055,
+      "logps/chosen": -0.37130284309387207,
+      "logps/rejected": -0.7186514735221863,
+      "loss": 0.6113,
+      "nll_loss": 0.6211617588996887,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 620
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.61923338694871e-07,
+      "log_odds_chosen": 0.8980989456176758,
+      "log_odds_ratio": -0.44537553191185,
+      "logits/chosen": -2.3041529655456543,
+      "logits/rejected": -2.246042490005493,
+      "logps/chosen": -0.3840414583683014,
+      "logps/rejected": -0.7412872910499573,
+      "loss": 0.6259,
+      "nll_loss": 0.6146137714385986,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 630
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 5.65625,
+      "learning_rate": 1.1873861228862998e-07,
+      "log_odds_chosen": 0.9331363439559937,
+      "log_odds_ratio": -0.4274202883243561,
+      "logits/chosen": -2.2929160594940186,
+      "logits/rejected": -2.2195255756378174,
+      "logps/chosen": -0.3945469558238983,
+      "logps/rejected": -0.7453508377075195,
+      "loss": 0.6133,
+      "nll_loss": 0.621927797794342,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 640
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 6.21875,
+      "learning_rate": 8.210256357836065e-08,
+      "log_odds_chosen": 1.0095536708831787,
+      "log_odds_ratio": -0.3956521451473236,
+      "logits/chosen": -2.263673782348633,
+      "logits/rejected": -2.20683217048645,
+      "logps/chosen": -0.3427670896053314,
+      "logps/rejected": -0.7156583070755005,
+      "loss": 0.574,
+      "nll_loss": 0.5451396703720093,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 650
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 5.8125,
+      "learning_rate": 5.21159449116615e-08,
+      "log_odds_chosen": 0.770971417427063,
+      "log_odds_ratio": -0.4887896180152893,
+      "logits/chosen": -2.284029006958008,
+      "logits/rejected": -2.214066982269287,
+      "logps/chosen": -0.4013218879699707,
+      "logps/rejected": -0.6818682551383972,
+      "loss": 0.5932,
+      "nll_loss": 0.5923025012016296,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 660
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 5.4375,
+      "learning_rate": 2.8861222120235845e-08,
+      "log_odds_chosen": 0.8828635215759277,
+      "log_odds_ratio": -0.4312991201877594,
+      "logits/chosen": -2.24355411529541,
+      "logits/rejected": -2.219613552093506,
+      "logps/chosen": -0.34176358580589294,
+      "logps/rejected": -0.6538265943527222,
+      "loss": 0.5864,
+      "nll_loss": 0.58399498462677,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 670
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 5.53125,
+      "learning_rate": 1.2402347731620412e-08,
+      "log_odds_chosen": 0.7922724485397339,
+      "log_odds_ratio": -0.4506749212741852,
+      "logits/chosen": -2.275207996368408,
+      "logits/rejected": -2.178473949432373,
+      "logps/chosen": -0.40824776887893677,
+      "logps/rejected": -0.7051072120666504,
+      "loss": 0.5958,
+      "nll_loss": 0.5631005167961121,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 680
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 6.15625,
+      "learning_rate": 2.7845850941254914e-09,
+      "log_odds_chosen": 0.8703590631484985,
+      "log_odds_ratio": -0.4304262101650238,
+      "logits/chosen": -2.343256711959839,
+      "logits/rejected": -2.2737724781036377,
+      "logps/chosen": -0.366679310798645,
+      "logps/rejected": -0.6560879349708557,
+      "loss": 0.6025,
+      "nll_loss": 0.5677012205123901,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 690
+    },
+    {
+      "epoch": 3.0,
+      "step": 699,
+      "total_flos": 0.0,
+      "train_loss": 0.7502862380468454,
+      "train_runtime": 21621.4849,
+      "train_samples_per_second": 2.071,
+      "train_steps_per_second": 0.032
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 699,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}