TX

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: zephyr-7b-MI-1-SELM
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-MI-1-SELM
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.8644854217705238,
+    "train_runtime": 7378.4782,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.286,
+    "train_steps_per_second": 0.065
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba08b20ac19ebf2b0d987ec66eaad2eda40493e5df15cf4f39139d09b6179a38
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21104ec7d9121ce0992c1759ff5ec736484b32277a8d683bc87e69814175007e
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5da6e5cac8df15c7062836ee413b92ea1fc71979d1b663c81022fd832319d0cd
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.8644854217705238,
+    "train_runtime": 7378.4782,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.286,
+    "train_steps_per_second": 0.065
+}

trainer_state.json

@@ -0,0 +1,1455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 15.141688725114767,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.770766496658325,
+      "logits/rejected": -2.728191375732422,
+      "logps/chosen": -1.027971625328064,
+      "logps/rejected": -1.1730705499649048,
+      "loss": 2.0284,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.027971625328064,
+      "rewards/margins": 0.14509883522987366,
+      "rewards/rejected": -1.1730705499649048,
+      "step": 5
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 13.969517964651633,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.7511062622070312,
+      "logits/rejected": -2.7253737449645996,
+      "logps/chosen": -0.9954013824462891,
+      "logps/rejected": -1.0719196796417236,
+      "loss": 2.002,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.9954013824462891,
+      "rewards/margins": 0.07651824504137039,
+      "rewards/rejected": -1.0719196796417236,
+      "step": 10
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 13.915856984059875,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.7306742668151855,
+      "logits/rejected": -2.697096347808838,
+      "logps/chosen": -0.9895243644714355,
+      "logps/rejected": -1.147679328918457,
+      "loss": 1.9874,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9895243644714355,
+      "rewards/margins": 0.15815496444702148,
+      "rewards/rejected": -1.147679328918457,
+      "step": 15
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 13.588156017248505,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.8030834197998047,
+      "logits/rejected": -2.709643602371216,
+      "logps/chosen": -1.0228091478347778,
+      "logps/rejected": -1.1238880157470703,
+      "loss": 2.0265,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.0228091478347778,
+      "rewards/margins": 0.10107902437448502,
+      "rewards/rejected": -1.1238880157470703,
+      "step": 20
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 14.042807702810773,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.7991013526916504,
+      "logits/rejected": -2.7758243083953857,
+      "logps/chosen": -1.0302048921585083,
+      "logps/rejected": -1.086944818496704,
+      "loss": 2.0091,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.0302048921585083,
+      "rewards/margins": 0.05673985555768013,
+      "rewards/rejected": -1.086944818496704,
+      "step": 25
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 15.069602245174066,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7733917236328125,
+      "logits/rejected": -2.7317938804626465,
+      "logps/chosen": -0.9855572581291199,
+      "logps/rejected": -1.0908451080322266,
+      "loss": 2.0118,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9855572581291199,
+      "rewards/margins": 0.1052878275513649,
+      "rewards/rejected": -1.0908451080322266,
+      "step": 30
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 14.14588992747414,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.7773802280426025,
+      "logits/rejected": -2.6910452842712402,
+      "logps/chosen": -1.0310332775115967,
+      "logps/rejected": -1.1711452007293701,
+      "loss": 1.9903,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0310332775115967,
+      "rewards/margins": 0.1401120126247406,
+      "rewards/rejected": -1.1711452007293701,
+      "step": 35
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 20.106972236529245,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.7931580543518066,
+      "logits/rejected": -2.6956787109375,
+      "logps/chosen": -1.011854887008667,
+      "logps/rejected": -1.2990570068359375,
+      "loss": 1.9463,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.011854887008667,
+      "rewards/margins": 0.28720206022262573,
+      "rewards/rejected": -1.2990570068359375,
+      "step": 40
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 15.19327904046142,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.8017444610595703,
+      "logits/rejected": -2.7550606727600098,
+      "logps/chosen": -0.9639647603034973,
+      "logps/rejected": -1.142878770828247,
+      "loss": 1.9512,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.9639647603034973,
+      "rewards/margins": 0.17891402542591095,
+      "rewards/rejected": -1.142878770828247,
+      "step": 45
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 20.91684390311358,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.7584781646728516,
+      "logits/rejected": -2.6904306411743164,
+      "logps/chosen": -1.0117145776748657,
+      "logps/rejected": -1.30343496799469,
+      "loss": 1.9653,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0117145776748657,
+      "rewards/margins": 0.2917203903198242,
+      "rewards/rejected": -1.30343496799469,
+      "step": 50
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 16.247858117319236,
+      "learning_rate": 4.996716052911017e-07,
+      "logits/chosen": -2.754804849624634,
+      "logits/rejected": -2.71933650970459,
+      "logps/chosen": -1.0166981220245361,
+      "logps/rejected": -1.268345832824707,
+      "loss": 1.9611,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.0166981220245361,
+      "rewards/margins": 0.25164780020713806,
+      "rewards/rejected": -1.268345832824707,
+      "step": 55
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 16.39781017532518,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.8165841102600098,
+      "logits/rejected": -2.7710437774658203,
+      "logps/chosen": -1.049938440322876,
+      "logps/rejected": -1.2433346509933472,
+      "loss": 1.9164,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.049938440322876,
+      "rewards/margins": 0.19339624047279358,
+      "rewards/rejected": -1.2433346509933472,
+      "step": 60
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 15.239254500208718,
+      "learning_rate": 4.980652179769217e-07,
+      "logits/chosen": -2.8142337799072266,
+      "logits/rejected": -2.6995322704315186,
+      "logps/chosen": -1.0191227197647095,
+      "logps/rejected": -1.544886827468872,
+      "loss": 1.9025,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0191227197647095,
+      "rewards/margins": 0.5257640480995178,
+      "rewards/rejected": -1.544886827468872,
+      "step": 65
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 21.0566472158153,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.7174973487854004,
+      "logits/rejected": -2.6667487621307373,
+      "logps/chosen": -1.0492310523986816,
+      "logps/rejected": -1.3482974767684937,
+      "loss": 1.9091,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.0492310523986816,
+      "rewards/margins": 0.2990665137767792,
+      "rewards/rejected": -1.3482974767684937,
+      "step": 70
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 30.761967865275885,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.773061752319336,
+      "logits/rejected": -2.698946952819824,
+      "logps/chosen": -0.9925507307052612,
+      "logps/rejected": -1.3563807010650635,
+      "loss": 1.8891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9925507307052612,
+      "rewards/margins": 0.36382991075515747,
+      "rewards/rejected": -1.3563807010650635,
+      "step": 75
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 31.297744502118277,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.738257884979248,
+      "logits/rejected": -2.6784729957580566,
+      "logps/chosen": -1.0087690353393555,
+      "logps/rejected": -1.4246900081634521,
+      "loss": 1.9382,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0087690353393555,
+      "rewards/margins": 0.4159211218357086,
+      "rewards/rejected": -1.4246900081634521,
+      "step": 80
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 24.407477818447635,
+      "learning_rate": 4.908790517010636e-07,
+      "logits/chosen": -2.747776508331299,
+      "logits/rejected": -2.7003962993621826,
+      "logps/chosen": -0.9973554611206055,
+      "logps/rejected": -1.3728947639465332,
+      "loss": 1.9613,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9973554611206055,
+      "rewards/margins": 0.3755393624305725,
+      "rewards/rejected": -1.3728947639465332,
+      "step": 85
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 31.20847987328103,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.705894708633423,
+      "logits/rejected": -2.6819891929626465,
+      "logps/chosen": -1.0237383842468262,
+      "logps/rejected": -1.404168725013733,
+      "loss": 1.88,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0237383842468262,
+      "rewards/margins": 0.3804302513599396,
+      "rewards/rejected": -1.404168725013733,
+      "step": 90
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 30.65914852598083,
+      "learning_rate": 4.853377929214243e-07,
+      "logits/chosen": -2.776717185974121,
+      "logits/rejected": -2.6825289726257324,
+      "logps/chosen": -1.038529872894287,
+      "logps/rejected": -1.3572471141815186,
+      "loss": 1.9062,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.038529872894287,
+      "rewards/margins": 0.31871724128723145,
+      "rewards/rejected": -1.3572471141815186,
+      "step": 95
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 34.08856475096774,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.74165678024292,
+      "logits/rejected": -2.6503825187683105,
+      "logps/chosen": -1.051953673362732,
+      "logps/rejected": -1.4617860317230225,
+      "loss": 1.9147,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.051953673362732,
+      "rewards/margins": 0.4098321497440338,
+      "rewards/rejected": -1.4617860317230225,
+      "step": 100
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 28.24345375738832,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.743473529815674,
+      "logits/rejected": -2.7197020053863525,
+      "logps/chosen": -0.931172251701355,
+      "logps/rejected": -1.453914999961853,
+      "loss": 1.8781,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.931172251701355,
+      "rewards/margins": 0.522742748260498,
+      "rewards/rejected": -1.453914999961853,
+      "step": 105
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 35.845899664243596,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.817007064819336,
+      "logits/rejected": -2.762336254119873,
+      "logps/chosen": -1.0500761270523071,
+      "logps/rejected": -1.4565006494522095,
+      "loss": 1.88,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0500761270523071,
+      "rewards/margins": 0.40642452239990234,
+      "rewards/rejected": -1.4565006494522095,
+      "step": 110
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 23.382933706781618,
+      "learning_rate": 4.70507279583015e-07,
+      "logits/chosen": -2.7697396278381348,
+      "logits/rejected": -2.710322856903076,
+      "logps/chosen": -1.0177266597747803,
+      "logps/rejected": -1.5666338205337524,
+      "loss": 1.868,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0177266597747803,
+      "rewards/margins": 0.5489073991775513,
+      "rewards/rejected": -1.5666338205337524,
+      "step": 115
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 29.59771539393517,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.7786221504211426,
+      "logits/rejected": -2.737684965133667,
+      "logps/chosen": -1.0584137439727783,
+      "logps/rejected": -1.4742605686187744,
+      "loss": 1.9062,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0584137439727783,
+      "rewards/margins": 0.41584667563438416,
+      "rewards/rejected": -1.4742605686187744,
+      "step": 120
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 24.555574989451102,
+      "learning_rate": 4.612975213859487e-07,
+      "logits/chosen": -2.8200080394744873,
+      "logits/rejected": -2.7997546195983887,
+      "logps/chosen": -1.058563470840454,
+      "logps/rejected": -1.3519017696380615,
+      "loss": 1.9069,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.058563470840454,
+      "rewards/margins": 0.29333844780921936,
+      "rewards/rejected": -1.3519017696380615,
+      "step": 125
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 26.634009671559898,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -2.788752794265747,
+      "logits/rejected": -2.754631519317627,
+      "logps/chosen": -1.0312286615371704,
+      "logps/rejected": -1.2794463634490967,
+      "loss": 1.8977,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.0312286615371704,
+      "rewards/margins": 0.24821774661540985,
+      "rewards/rejected": -1.2794463634490967,
+      "step": 130
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 32.240888174368,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -2.7877840995788574,
+      "logits/rejected": -2.7001712322235107,
+      "logps/chosen": -1.0880383253097534,
+      "logps/rejected": -1.7845348119735718,
+      "loss": 1.8912,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0880383253097534,
+      "rewards/margins": 0.696496307849884,
+      "rewards/rejected": -1.7845348119735718,
+      "step": 135
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 25.76204815720163,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -2.8313488960266113,
+      "logits/rejected": -2.780134677886963,
+      "logps/chosen": -0.9982128143310547,
+      "logps/rejected": -1.4504693746566772,
+      "loss": 1.8155,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9982128143310547,
+      "rewards/margins": 0.4522566795349121,
+      "rewards/rejected": -1.4504693746566772,
+      "step": 140
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 21.7794898263259,
+      "learning_rate": 4.395355737667985e-07,
+      "logits/chosen": -2.809255361557007,
+      "logits/rejected": -2.761308193206787,
+      "logps/chosen": -1.0206363201141357,
+      "logps/rejected": -1.419768214225769,
+      "loss": 1.8605,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0206363201141357,
+      "rewards/margins": 0.39913198351860046,
+      "rewards/rejected": -1.419768214225769,
+      "step": 145
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 26.507909597761785,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -2.849714517593384,
+      "logits/rejected": -2.8313772678375244,
+      "logps/chosen": -0.9701302647590637,
+      "logps/rejected": -1.3971799612045288,
+      "loss": 1.8345,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9701302647590637,
+      "rewards/margins": 0.42704978585243225,
+      "rewards/rejected": -1.3971799612045288,
+      "step": 150
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 28.367719379820283,
+      "learning_rate": 4.271000354423425e-07,
+      "logits/chosen": -2.7890210151672363,
+      "logits/rejected": -2.757753849029541,
+      "logps/chosen": -1.0669995546340942,
+      "logps/rejected": -1.4297670125961304,
+      "loss": 1.894,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0669995546340942,
+      "rewards/margins": 0.36276736855506897,
+      "rewards/rejected": -1.4297670125961304,
+      "step": 155
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 32.029509535293904,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -2.7710726261138916,
+      "logits/rejected": -2.7451701164245605,
+      "logps/chosen": -1.0376036167144775,
+      "logps/rejected": -1.4104793071746826,
+      "loss": 1.8838,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0376036167144775,
+      "rewards/margins": 0.37287575006484985,
+      "rewards/rejected": -1.4104793071746826,
+      "step": 160
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 25.20687040858121,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -2.7923319339752197,
+      "logits/rejected": -2.707219362258911,
+      "logps/chosen": -0.9815086126327515,
+      "logps/rejected": -1.4057986736297607,
+      "loss": 1.8435,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9815086126327515,
+      "rewards/margins": 0.4242900311946869,
+      "rewards/rejected": -1.4057986736297607,
+      "step": 165
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 28.006785803959723,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -2.8175511360168457,
+      "logits/rejected": -2.752450466156006,
+      "logps/chosen": -1.0611494779586792,
+      "logps/rejected": -1.5021220445632935,
+      "loss": 1.8575,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0611494779586792,
+      "rewards/margins": 0.4409725069999695,
+      "rewards/rejected": -1.5021220445632935,
+      "step": 170
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 25.596200434230976,
+      "learning_rate": 3.994527650465352e-07,
+      "logits/chosen": -2.7964577674865723,
+      "logits/rejected": -2.709230661392212,
+      "logps/chosen": -1.1404192447662354,
+      "logps/rejected": -1.6063992977142334,
+      "loss": 1.9076,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1404192447662354,
+      "rewards/margins": 0.4659802317619324,
+      "rewards/rejected": -1.6063992977142334,
+      "step": 175
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 25.892094106029592,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -2.7817273139953613,
+      "logits/rejected": -2.7617640495300293,
+      "logps/chosen": -1.0391079187393188,
+      "logps/rejected": -1.5876353979110718,
+      "loss": 1.8544,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0391079187393188,
+      "rewards/margins": 0.5485275387763977,
+      "rewards/rejected": -1.5876353979110718,
+      "step": 180
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 29.42463620034665,
+      "learning_rate": 3.8438923131177237e-07,
+      "logits/chosen": -2.7785096168518066,
+      "logits/rejected": -2.6860692501068115,
+      "logps/chosen": -1.0492454767227173,
+      "logps/rejected": -1.6226046085357666,
+      "loss": 1.8247,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0492454767227173,
+      "rewards/margins": 0.573358952999115,
+      "rewards/rejected": -1.6226046085357666,
+      "step": 185
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 27.921360264789193,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": -2.7853875160217285,
+      "logits/rejected": -2.7454705238342285,
+      "logps/chosen": -1.0294712781906128,
+      "logps/rejected": -1.4654343128204346,
+      "loss": 1.9244,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0294712781906128,
+      "rewards/margins": 0.4359629154205322,
+      "rewards/rejected": -1.4654343128204346,
+      "step": 190
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 29.87078620394206,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -2.701105833053589,
+      "logits/rejected": -2.6695313453674316,
+      "logps/chosen": -1.098097324371338,
+      "logps/rejected": -1.4831833839416504,
+      "loss": 1.899,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.098097324371338,
+      "rewards/margins": 0.3850858509540558,
+      "rewards/rejected": -1.4831833839416504,
+      "step": 195
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 28.048650935368237,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -2.738492250442505,
+      "logits/rejected": -2.7187023162841797,
+      "logps/chosen": -1.0665818452835083,
+      "logps/rejected": -1.46578049659729,
+      "loss": 1.8737,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0665818452835083,
+      "rewards/margins": 0.39919885993003845,
+      "rewards/rejected": -1.46578049659729,
+      "step": 200
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 27.850087928191183,
+      "learning_rate": 3.5218566107988867e-07,
+      "logits/chosen": -2.6582818031311035,
+      "logits/rejected": -2.5809996128082275,
+      "logps/chosen": -0.9890678524971008,
+      "logps/rejected": -1.4678758382797241,
+      "loss": 1.8403,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9890678524971008,
+      "rewards/margins": 0.47880783677101135,
+      "rewards/rejected": -1.4678758382797241,
+      "step": 205
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 29.109484926295075,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -2.6842644214630127,
+      "logits/rejected": -2.627836227416992,
+      "logps/chosen": -1.075455665588379,
+      "logps/rejected": -1.4202773571014404,
+      "loss": 1.8863,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.075455665588379,
+      "rewards/margins": 0.34482160210609436,
+      "rewards/rejected": -1.4202773571014404,
+      "step": 210
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 23.99752736324705,
+      "learning_rate": 3.3521824616429284e-07,
+      "logits/chosen": -2.716024875640869,
+      "logits/rejected": -2.6916396617889404,
+      "logps/chosen": -1.0032670497894287,
+      "logps/rejected": -1.5087072849273682,
+      "loss": 1.8264,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0032670497894287,
+      "rewards/margins": 0.5054404139518738,
+      "rewards/rejected": -1.5087072849273682,
+      "step": 215
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 31.21916182230841,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -2.742882013320923,
+      "logits/rejected": -2.658285617828369,
+      "logps/chosen": -1.033542275428772,
+      "logps/rejected": -1.5640771389007568,
+      "loss": 1.8307,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.033542275428772,
+      "rewards/margins": 0.5305347442626953,
+      "rewards/rejected": -1.5640771389007568,
+      "step": 220
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 26.98263419792695,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -2.7341504096984863,
+      "logits/rejected": -2.6276047229766846,
+      "logps/chosen": -0.988808810710907,
+      "logps/rejected": -1.5687158107757568,
+      "loss": 1.8334,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.988808810710907,
+      "rewards/margins": 0.5799070596694946,
+      "rewards/rejected": -1.5687158107757568,
+      "step": 225
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 29.311922113138767,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -2.74092698097229,
+      "logits/rejected": -2.6369271278381348,
+      "logps/chosen": -1.009143352508545,
+      "logps/rejected": -1.665408730506897,
+      "loss": 1.8283,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.009143352508545,
+      "rewards/margins": 0.6562654376029968,
+      "rewards/rejected": -1.665408730506897,
+      "step": 230
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 32.62916846952035,
+      "learning_rate": 3.000064234440111e-07,
+      "logits/chosen": -2.729424476623535,
+      "logits/rejected": -2.672799587249756,
+      "logps/chosen": -1.0022896528244019,
+      "logps/rejected": -1.5322816371917725,
+      "loss": 1.86,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0022896528244019,
+      "rewards/margins": 0.5299919247627258,
+      "rewards/rejected": -1.5322816371917725,
+      "step": 235
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 25.60849652215575,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -2.666738986968994,
+      "logits/rejected": -2.6147377490997314,
+      "logps/chosen": -1.0380260944366455,
+      "logps/rejected": -1.5499128103256226,
+      "loss": 1.8806,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0380260944366455,
+      "rewards/margins": 0.5118868947029114,
+      "rewards/rejected": -1.5499128103256226,
+      "step": 240
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 23.307769710548996,
+      "learning_rate": 2.8195076242990116e-07,
+      "logits/chosen": -2.7083492279052734,
+      "logits/rejected": -2.608227014541626,
+      "logps/chosen": -1.1102759838104248,
+      "logps/rejected": -1.5368731021881104,
+      "loss": 1.8599,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1102759838104248,
+      "rewards/margins": 0.42659682035446167,
+      "rewards/rejected": -1.5368731021881104,
+      "step": 245
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 30.713855522791444,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -2.621891498565674,
+      "logits/rejected": -2.572918653488159,
+      "logps/chosen": -1.0056602954864502,
+      "logps/rejected": -1.5289640426635742,
+      "loss": 1.8611,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0056602954864502,
+      "rewards/margins": 0.5233036875724792,
+      "rewards/rejected": -1.5289640426635742,
+      "step": 250
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 21.027582284131125,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -2.679741859436035,
+      "logits/rejected": -2.588026762008667,
+      "logps/chosen": -1.0767912864685059,
+      "logps/rejected": -1.6802423000335693,
+      "loss": 1.7962,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0767912864685059,
+      "rewards/margins": 0.603451132774353,
+      "rewards/rejected": -1.6802423000335693,
+      "step": 255
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 21.083992579626138,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -2.639758825302124,
+      "logits/rejected": -2.539482831954956,
+      "logps/chosen": -0.9617997407913208,
+      "logps/rejected": -1.643094778060913,
+      "loss": 1.8469,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9617997407913208,
+      "rewards/margins": 0.6812952160835266,
+      "rewards/rejected": -1.643094778060913,
+      "step": 260
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 34.336932322095805,
+      "learning_rate": 2.454233432955807e-07,
+      "logits/chosen": -2.6934094429016113,
+      "logits/rejected": -2.635524034500122,
+      "logps/chosen": -1.0399844646453857,
+      "logps/rejected": -1.3895108699798584,
+      "loss": 1.8185,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0399844646453857,
+      "rewards/margins": 0.3495265543460846,
+      "rewards/rejected": -1.3895108699798584,
+      "step": 265
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 35.56183499916552,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -2.678424596786499,
+      "logits/rejected": -2.6192078590393066,
+      "logps/chosen": -1.1347005367279053,
+      "logps/rejected": -1.5019694566726685,
+      "loss": 1.8876,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.1347005367279053,
+      "rewards/margins": 0.36726871132850647,
+      "rewards/rejected": -1.5019694566726685,
+      "step": 270
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 21.13133035084058,
+      "learning_rate": 2.2714738398943308e-07,
+      "logits/chosen": -2.749488592147827,
+      "logits/rejected": -2.629812717437744,
+      "logps/chosen": -1.0328668355941772,
+      "logps/rejected": -1.6207387447357178,
+      "loss": 1.833,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0328668355941772,
+      "rewards/margins": 0.5878719091415405,
+      "rewards/rejected": -1.6207387447357178,
+      "step": 275
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 27.44411354972676,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": -2.7630438804626465,
+      "logits/rejected": -2.692211866378784,
+      "logps/chosen": -0.968722939491272,
+      "logps/rejected": -1.3731743097305298,
+      "loss": 1.8453,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.968722939491272,
+      "rewards/margins": 0.40445131063461304,
+      "rewards/rejected": -1.3731743097305298,
+      "step": 280
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 32.499361452307625,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": -2.682573080062866,
+      "logits/rejected": -2.678396224975586,
+      "logps/chosen": -1.0675628185272217,
+      "logps/rejected": -1.5514373779296875,
+      "loss": 1.8267,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0675628185272217,
+      "rewards/margins": 0.4838746190071106,
+      "rewards/rejected": -1.5514373779296875,
+      "step": 285
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 34.51002212540645,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": -2.7195992469787598,
+      "logits/rejected": -2.6691336631774902,
+      "logps/chosen": -1.0737675428390503,
+      "logps/rejected": -1.6511163711547852,
+      "loss": 1.8055,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0737675428390503,
+      "rewards/margins": 0.5773489475250244,
+      "rewards/rejected": -1.6511163711547852,
+      "step": 290
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 34.81875700322843,
+      "learning_rate": 1.9106026612264315e-07,
+      "logits/chosen": -2.7475931644439697,
+      "logits/rejected": -2.661341667175293,
+      "logps/chosen": -1.0815327167510986,
+      "logps/rejected": -1.6415588855743408,
+      "loss": 1.8548,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0815327167510986,
+      "rewards/margins": 0.5600259900093079,
+      "rewards/rejected": -1.6415588855743408,
+      "step": 295
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 39.656088577829394,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -2.650444507598877,
+      "logits/rejected": -2.601776123046875,
+      "logps/chosen": -1.0726263523101807,
+      "logps/rejected": -1.5858640670776367,
+      "loss": 1.8352,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0726263523101807,
+      "rewards/margins": 0.5132376551628113,
+      "rewards/rejected": -1.5858640670776367,
+      "step": 300
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 34.266407421018876,
+      "learning_rate": 1.7344254621846017e-07,
+      "logits/chosen": -2.6358659267425537,
+      "logits/rejected": -2.589916944503784,
+      "logps/chosen": -1.0825355052947998,
+      "logps/rejected": -1.695123314857483,
+      "loss": 1.8122,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0825355052947998,
+      "rewards/margins": 0.6125878691673279,
+      "rewards/rejected": -1.695123314857483,
+      "step": 305
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 29.29392494715919,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": -2.6732287406921387,
+      "logits/rejected": -2.6671674251556396,
+      "logps/chosen": -0.9606215357780457,
+      "logps/rejected": -1.507188081741333,
+      "loss": 1.8154,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9606215357780457,
+      "rewards/margins": 0.5465664863586426,
+      "rewards/rejected": -1.507188081741333,
+      "step": 310
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 30.886215191002822,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": -2.6945061683654785,
+      "logits/rejected": -2.6447794437408447,
+      "logps/chosen": -0.9400749206542969,
+      "logps/rejected": -1.5918124914169312,
+      "loss": 1.8296,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9400749206542969,
+      "rewards/margins": 0.651737630367279,
+      "rewards/rejected": -1.5918124914169312,
+      "step": 315
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 40.31455490523793,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": -2.6877899169921875,
+      "logits/rejected": -2.6543657779693604,
+      "logps/chosen": -1.0309157371520996,
+      "logps/rejected": -1.5069587230682373,
+      "loss": 1.7854,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0309157371520996,
+      "rewards/margins": 0.47604283690452576,
+      "rewards/rejected": -1.5069587230682373,
+      "step": 320
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 33.29916453849858,
+      "learning_rate": 1.3953046172178413e-07,
+      "logits/chosen": -2.712068557739258,
+      "logits/rejected": -2.6253581047058105,
+      "logps/chosen": -1.0362852811813354,
+      "logps/rejected": -1.6363003253936768,
+      "loss": 1.7979,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0362852811813354,
+      "rewards/margins": 0.6000149846076965,
+      "rewards/rejected": -1.6363003253936768,
+      "step": 325
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 28.615351737244207,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -2.672591209411621,
+      "logits/rejected": -2.624178886413574,
+      "logps/chosen": -1.0226625204086304,
+      "logps/rejected": -1.4031275510787964,
+      "loss": 1.8499,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0226625204086304,
+      "rewards/margins": 0.38046500086784363,
+      "rewards/rejected": -1.4031275510787964,
+      "step": 330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 31.872437043858163,
+      "learning_rate": 1.2341787690142435e-07,
+      "logits/chosen": -2.783500909805298,
+      "logits/rejected": -2.7123568058013916,
+      "logps/chosen": -0.9628676176071167,
+      "logps/rejected": -1.5226846933364868,
+      "loss": 1.8281,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9628676176071167,
+      "rewards/margins": 0.5598170161247253,
+      "rewards/rejected": -1.5226846933364868,
+      "step": 335
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 27.878376041354297,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": -2.7173564434051514,
+      "logits/rejected": -2.6413800716400146,
+      "logps/chosen": -1.1935253143310547,
+      "logps/rejected": -1.5975086688995361,
+      "loss": 1.8896,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1935253143310547,
+      "rewards/margins": 0.40398329496383667,
+      "rewards/rejected": -1.5975086688995361,
+      "step": 340
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 36.976534101742224,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": -2.7495174407958984,
+      "logits/rejected": -2.700282573699951,
+      "logps/chosen": -1.0389212369918823,
+      "logps/rejected": -1.5958582162857056,
+      "loss": 1.8433,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0389212369918823,
+      "rewards/margins": 0.5569368600845337,
+      "rewards/rejected": -1.5958582162857056,
+      "step": 345
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 30.288124940758514,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": -2.7106757164001465,
+      "logits/rejected": -2.687342882156372,
+      "logps/chosen": -1.0641244649887085,
+      "logps/rejected": -1.5430529117584229,
+      "loss": 1.8735,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.0641244649887085,
+      "rewards/margins": 0.47892847657203674,
+      "rewards/rejected": -1.5430529117584229,
+      "step": 350
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 30.72874948597524,
+      "learning_rate": 9.331100255592436e-08,
+      "logits/chosen": -2.7169158458709717,
+      "logits/rejected": -2.66363787651062,
+      "logps/chosen": -0.9323965311050415,
+      "logps/rejected": -1.5993990898132324,
+      "loss": 1.7736,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9323965311050415,
+      "rewards/margins": 0.6670026779174805,
+      "rewards/rejected": -1.5993990898132324,
+      "step": 355
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 22.0079277817506,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -2.746021270751953,
+      "logits/rejected": -2.6950621604919434,
+      "logps/chosen": -0.9703041315078735,
+      "logps/rejected": -1.4327679872512817,
+      "loss": 1.783,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9703041315078735,
+      "rewards/margins": 0.4624637961387634,
+      "rewards/rejected": -1.4327679872512817,
+      "step": 360
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 33.139790420388316,
+      "learning_rate": 7.947809564230445e-08,
+      "logits/chosen": -2.6830432415008545,
+      "logits/rejected": -2.5992660522460938,
+      "logps/chosen": -1.0758744478225708,
+      "logps/rejected": -1.6359386444091797,
+      "loss": 1.8553,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0758744478225708,
+      "rewards/margins": 0.5600640773773193,
+      "rewards/rejected": -1.6359386444091797,
+      "step": 365
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 42.978648090018346,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": -2.7361717224121094,
+      "logits/rejected": -2.6721107959747314,
+      "logps/chosen": -1.0527479648590088,
+      "logps/rejected": -1.7067893743515015,
+      "loss": 1.8183,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0527479648590088,
+      "rewards/margins": 0.6540414094924927,
+      "rewards/rejected": -1.7067893743515015,
+      "step": 370
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 24.61410992203062,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": -2.715893268585205,
+      "logits/rejected": -2.7327394485473633,
+      "logps/chosen": -1.0623342990875244,
+      "logps/rejected": -1.5285212993621826,
+      "loss": 1.8189,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.0623342990875244,
+      "rewards/margins": 0.4661870002746582,
+      "rewards/rejected": -1.5285212993621826,
+      "step": 375
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 29.084948650681554,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": -2.7230355739593506,
+      "logits/rejected": -2.665907621383667,
+      "logps/chosen": -1.0696966648101807,
+      "logps/rejected": -1.5225203037261963,
+      "loss": 1.7794,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0696966648101807,
+      "rewards/margins": 0.4528239369392395,
+      "rewards/rejected": -1.5225203037261963,
+      "step": 380
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 25.930022257563827,
+      "learning_rate": 5.4623689209832484e-08,
+      "logits/chosen": -2.7370214462280273,
+      "logits/rejected": -2.6596484184265137,
+      "logps/chosen": -1.0190281867980957,
+      "logps/rejected": -1.4870203733444214,
+      "loss": 1.8012,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.0190281867980957,
+      "rewards/margins": 0.46799221634864807,
+      "rewards/rejected": -1.4870203733444214,
+      "step": 385
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 30.16253644602296,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -2.755403518676758,
+      "logits/rejected": -2.6998536586761475,
+      "logps/chosen": -1.0058339834213257,
+      "logps/rejected": -1.6741714477539062,
+      "loss": 1.7816,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0058339834213257,
+      "rewards/margins": 0.6683375239372253,
+      "rewards/rejected": -1.6741714477539062,
+      "step": 390
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 32.100503895364014,
+      "learning_rate": 4.373541737087263e-08,
+      "logits/chosen": -2.6867947578430176,
+      "logits/rejected": -2.6116786003112793,
+      "logps/chosen": -0.9958473443984985,
+      "logps/rejected": -1.5259888172149658,
+      "loss": 1.8098,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9958473443984985,
+      "rewards/margins": 0.5301414132118225,
+      "rewards/rejected": -1.5259888172149658,
+      "step": 395
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 39.50777886532678,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": -2.6500821113586426,
+      "logits/rejected": -2.587491273880005,
+      "logps/chosen": -0.9868978261947632,
+      "logps/rejected": -1.5105135440826416,
+      "loss": 1.81,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9868978261947632,
+      "rewards/margins": 0.523615837097168,
+      "rewards/rejected": -1.5105135440826416,
+      "step": 400
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 39.172472564486576,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": -2.7077910900115967,
+      "logits/rejected": -2.671496868133545,
+      "logps/chosen": -1.0807896852493286,
+      "logps/rejected": -1.6041944026947021,
+      "loss": 1.8475,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0807896852493286,
+      "rewards/margins": 0.5234047174453735,
+      "rewards/rejected": -1.6041944026947021,
+      "step": 405
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 34.16755927550673,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": -2.729513645172119,
+      "logits/rejected": -2.68589448928833,
+      "logps/chosen": -0.9558256268501282,
+      "logps/rejected": -1.4403427839279175,
+      "loss": 1.8083,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9558256268501282,
+      "rewards/margins": 0.4845171868801117,
+      "rewards/rejected": -1.4403427839279175,
+      "step": 410
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 27.600146859315487,
+      "learning_rate": 2.5328246937043525e-08,
+      "logits/chosen": -2.758931875228882,
+      "logits/rejected": -2.693338394165039,
+      "logps/chosen": -1.004372239112854,
+      "logps/rejected": -1.6196565628051758,
+      "loss": 1.7709,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.004372239112854,
+      "rewards/margins": 0.615284264087677,
+      "rewards/rejected": -1.6196565628051758,
+      "step": 415
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 26.767731856804396,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -2.7129735946655273,
+      "logits/rejected": -2.6572775840759277,
+      "logps/chosen": -0.9905831217765808,
+      "logps/rejected": -1.5554085969924927,
+      "loss": 1.8411,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.9905831217765808,
+      "rewards/margins": 0.5648255944252014,
+      "rewards/rejected": -1.5554085969924927,
+      "step": 420
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 32.1190541072846,
+      "learning_rate": 1.7908016745981856e-08,
+      "logits/chosen": -2.656250476837158,
+      "logits/rejected": -2.645900249481201,
+      "logps/chosen": -1.1461454629898071,
+      "logps/rejected": -1.604714035987854,
+      "loss": 1.8218,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1461454629898071,
+      "rewards/margins": 0.45856842398643494,
+      "rewards/rejected": -1.604714035987854,
+      "step": 425
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 38.87548467523877,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": -2.661616325378418,
+      "logits/rejected": -2.617453098297119,
+      "logps/chosen": -1.1022528409957886,
+      "logps/rejected": -1.628826379776001,
+      "loss": 1.8088,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1022528409957886,
+      "rewards/margins": 0.5265735387802124,
+      "rewards/rejected": -1.628826379776001,
+      "step": 430
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 34.292720836263,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": -2.7260875701904297,
+      "logits/rejected": -2.685364246368408,
+      "logps/chosen": -1.0480232238769531,
+      "logps/rejected": -1.6929914951324463,
+      "loss": 1.8315,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0480232238769531,
+      "rewards/margins": 0.6449683308601379,
+      "rewards/rejected": -1.6929914951324463,
+      "step": 435
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 36.60770449520838,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": -2.6789753437042236,
+      "logits/rejected": -2.6536145210266113,
+      "logps/chosen": -1.0216642618179321,
+      "logps/rejected": -1.547012448310852,
+      "loss": 1.8478,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0216642618179321,
+      "rewards/margins": 0.5253482460975647,
+      "rewards/rejected": -1.547012448310852,
+      "step": 440
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 34.43665423362751,
+      "learning_rate": 6.832927412229017e-09,
+      "logits/chosen": -2.721832752227783,
+      "logits/rejected": -2.660407781600952,
+      "logps/chosen": -1.0646559000015259,
+      "logps/rejected": -1.554300308227539,
+      "loss": 1.8533,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0646559000015259,
+      "rewards/margins": 0.4896443486213684,
+      "rewards/rejected": -1.554300308227539,
+      "step": 445
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 30.770535415455328,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -2.6228880882263184,
+      "logits/rejected": -2.5837976932525635,
+      "logps/chosen": -1.0409653186798096,
+      "logps/rejected": -1.537603735923767,
+      "loss": 1.7905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0409653186798096,
+      "rewards/margins": 0.4966384768486023,
+      "rewards/rejected": -1.537603735923767,
+      "step": 450
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 27.915122014592697,
+      "learning_rate": 3.2374343405217884e-09,
+      "logits/chosen": -2.6800639629364014,
+      "logits/rejected": -2.6128344535827637,
+      "logps/chosen": -1.132541298866272,
+      "logps/rejected": -1.5116033554077148,
+      "loss": 1.825,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.132541298866272,
+      "rewards/margins": 0.3790620267391205,
+      "rewards/rejected": -1.5116033554077148,
+      "step": 455
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 31.401214396279837,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": -2.6828560829162598,
+      "logits/rejected": -2.6430163383483887,
+      "logps/chosen": -0.9926983118057251,
+      "logps/rejected": -1.5756382942199707,
+      "loss": 1.7905,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9926983118057251,
+      "rewards/margins": 0.5829401612281799,
+      "rewards/rejected": -1.5756382942199707,
+      "step": 460
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 38.51782864695839,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": -2.7383384704589844,
+      "logits/rejected": -2.6524176597595215,
+      "logps/chosen": -1.0901256799697876,
+      "logps/rejected": -1.620060682296753,
+      "loss": 1.8476,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0901256799697876,
+      "rewards/margins": 0.5299350023269653,
+      "rewards/rejected": -1.620060682296753,
+      "step": 465
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 27.923910450585048,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": -2.7122910022735596,
+      "logits/rejected": -2.6457693576812744,
+      "logps/chosen": -0.9990774393081665,
+      "logps/rejected": -1.4476706981658936,
+      "loss": 1.8714,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.9990774393081665,
+      "rewards/margins": 0.4485931396484375,
+      "rewards/rejected": -1.4476706981658936,
+      "step": 470
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 33.978087151320835,
+      "learning_rate": 2.6813123097352287e-11,
+      "logits/chosen": -2.7624480724334717,
+      "logits/rejected": -2.6619315147399902,
+      "logps/chosen": -1.042180061340332,
+      "logps/rejected": -1.5233901739120483,
+      "loss": 1.8233,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.042180061340332,
+      "rewards/margins": 0.4812101721763611,
+      "rewards/rejected": -1.5233901739120483,
+      "step": 475
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 1.8644854217705238,
+      "train_runtime": 7378.4782,
+      "train_samples_per_second": 8.286,
+      "train_steps_per_second": 0.065
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1566c6850ed286617593bc0f0a3520f3b18f6ea3c6c7c83ad58bd9856178c1a5
+size 7160