Upload 18 files

README.md

@@ -1,3 +1,75 @@
----
-license: cc-by-4.0
----
+---
+base_model: Minbyul/selfbiorag-7b-wo-kqa_golden-iter-dpo-step3-filtered
+tags:
+- alignment-handbook
+- trl
+- dpo
+- generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: selfbiorag-7b-wo-kqa_golden-iter-dpo-step4-filtered
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# selfbiorag-7b-wo-kqa_golden-iter-dpo-step4-filtered
+
+This model is a fine-tuned version of [Minbyul/selfbiorag-7b-wo-kqa_golden-iter-dpo-step3-filtered](https://huggingface.co/Minbyul/selfbiorag-7b-wo-kqa_golden-iter-dpo-step3-filtered) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6766
+- Rewards/chosen: -0.0828
+- Rewards/rejected: -0.1144
+- Rewards/accuracies: 0.6319
+- Rewards/margins: 0.0316
+- Logps/rejected: -98.9601
+- Logps/chosen: -79.1920
+- Logits/rejected: -1.2073
+- Logits/chosen: -1.1930
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

added_tokens.json

@@ -0,0 +1,18 @@
+{
+  "</paragraph>": 32013,
+  "<pad>": 32015,
+  "<paragraph>": 32005,
+  "[Continue to Use Evidence]": 32006,
+  "[Fully supported]": 32014,
+  "[Irrelevant]": 32011,
+  "[No Retrieval]": 32010,
+  "[No support / Contradictory]": 32003,
+  "[Partially supported]": 32000,
+  "[Relevant]": 32007,
+  "[Retrieval]": 32004,
+  "[Utility:1]": 32001,
+  "[Utility:2]": 32002,
+  "[Utility:3]": 32008,
+  "[Utility:4]": 32012,
+  "[Utility:5]": 32009
+}

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.1930192708969116,
+    "eval_logits/rejected": -1.2072994709014893,
+    "eval_logps/chosen": -79.19204711914062,
+    "eval_logps/rejected": -98.96011352539062,
+    "eval_loss": 0.6765820384025574,
+    "eval_rewards/accuracies": 0.6319444179534912,
+    "eval_rewards/chosen": -0.08277318626642227,
+    "eval_rewards/margins": 0.03162948787212372,
+    "eval_rewards/rejected": -0.11440268903970718,
+    "eval_runtime": 48.68,
+    "eval_samples": 567,
+    "eval_samples_per_second": 11.647,
+    "eval_steps_per_second": 0.37,
+    "train_loss": 0.6676328546471066,
+    "train_runtime": 2437.1919,
+    "train_samples": 13823,
+    "train_samples_per_second": 5.672,
+    "train_steps_per_second": 0.089
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "Minbyul/selfbiorag-7b-wo-kqa_golden-iter-dpo-step3-filtered",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32016
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.1930192708969116,
+    "eval_logits/rejected": -1.2072994709014893,
+    "eval_logps/chosen": -79.19204711914062,
+    "eval_logps/rejected": -98.96011352539062,
+    "eval_loss": 0.6765820384025574,
+    "eval_rewards/accuracies": 0.6319444179534912,
+    "eval_rewards/chosen": -0.08277318626642227,
+    "eval_rewards/margins": 0.03162948787212372,
+    "eval_rewards/rejected": -0.11440268903970718,
+    "eval_runtime": 48.68,
+    "eval_samples": 567,
+    "eval_samples_per_second": 11.647,
+    "eval_steps_per_second": 0.37
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f4bf9a2645a9e1ccf6699d7c4f07bf489b6659f2826495ed4bf02a7ab3a8d3f
+size 4939116424

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cff5aae326ff1353f3e2b5093a8c9b4e0d5b5b4d1b6ef771930138b2bd229ef
+size 4947390880

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e946b73106cbafeeedd8c2e4e32fb82bd8aba15812b0001575f5abf801c807b0
+size 3590619888

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13477093376
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,47 @@
+{
+  "additional_special_tokens": [
+    "[Partially supported]",
+    "[Utility:1]",
+    "[Utility:2]",
+    "[No support / Contradictory]",
+    "[Retrieval]",
+    "<paragraph>",
+    "[Continue to Use Evidence]",
+    "[Relevant]",
+    "[Utility:3]",
+    "[Utility:5]",
+    "[No Retrieval]",
+    "[Irrelevant]",
+    "[Utility:4]",
+    "</paragraph>",
+    "[Fully supported]"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,188 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "[Partially supported]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "[Utility:1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "[Utility:2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "[No support / Contradictory]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "[Retrieval]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<paragraph>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "[Continue to Use Evidence]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "[Relevant]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "[Utility:3]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "[Utility:5]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "[No Retrieval]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32011": {
+      "content": "[Irrelevant]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "[Utility:4]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32013": {
+      "content": "</paragraph>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32014": {
+      "content": "[Fully supported]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32015": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "[Partially supported]",
+    "[Utility:1]",
+    "[Utility:2]",
+    "[No support / Contradictory]",
+    "[Retrieval]",
+    "<paragraph>",
+    "[Continue to Use Evidence]",
+    "[Relevant]",
+    "[Utility:3]",
+    "[Utility:5]",
+    "[No Retrieval]",
+    "[Irrelevant]",
+    "[Utility:4]",
+    "</paragraph>",
+    "[Fully supported]"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6676328546471066,
+    "train_runtime": 2437.1919,
+    "train_samples": 13823,
+    "train_samples_per_second": 5.672,
+    "train_steps_per_second": 0.089
+}

trainer_state.json

@@ -0,0 +1,360 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 216,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.33483565345429,
+      "learning_rate": 2.2727272727272725e-08,
+      "logits/chosen": -0.9294053316116333,
+      "logits/rejected": -1.2306766510009766,
+      "logps/chosen": -79.32324981689453,
+      "logps/rejected": -124.87494659423828,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 8.957121309516765,
+      "learning_rate": 2.2727272727272726e-07,
+      "logits/chosen": -0.5779850482940674,
+      "logits/rejected": -1.1789488792419434,
+      "logps/chosen": -83.05654907226562,
+      "logps/rejected": -135.26220703125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.0003059397859033197,
+      "rewards/margins": 0.00026924433768726885,
+      "rewards/rejected": 3.6695546441478655e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 8.37389940695367,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -0.621688723564148,
+      "logits/rejected": -1.1858972311019897,
+      "logps/chosen": -95.2262954711914,
+      "logps/rejected": -143.64205932617188,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.0002137714036507532,
+      "rewards/margins": -0.0006433525122702122,
+      "rewards/rejected": 0.00042958109406754375,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 8.286301306160889,
+      "learning_rate": 4.979050253066063e-07,
+      "logits/chosen": -0.6890290975570679,
+      "logits/rejected": -1.1448113918304443,
+      "logps/chosen": -88.85906982421875,
+      "logps/rejected": -132.80384826660156,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0019540651701390743,
+      "rewards/margins": 0.00375129166059196,
+      "rewards/rejected": -0.0017972266068682075,
+      "step": 30
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 8.434509757435025,
+      "learning_rate": 4.894543310469967e-07,
+      "logits/chosen": -0.5220860242843628,
+      "logits/rejected": -1.1176507472991943,
+      "logps/chosen": -91.24484252929688,
+      "logps/rejected": -126.5925521850586,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0025604977272450924,
+      "rewards/margins": 0.005932547152042389,
+      "rewards/rejected": -0.003372048959136009,
+      "step": 40
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 8.535183437584605,
+      "learning_rate": 4.747379352713488e-07,
+      "logits/chosen": -0.685869574546814,
+      "logits/rejected": -1.2057311534881592,
+      "logps/chosen": -96.0764389038086,
+      "logps/rejected": -127.81497955322266,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.003552153008058667,
+      "rewards/margins": 0.010397722944617271,
+      "rewards/rejected": -0.006845570169389248,
+      "step": 50
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 8.190633609066436,
+      "learning_rate": 4.541409157643027e-07,
+      "logits/chosen": -0.6526767015457153,
+      "logits/rejected": -1.1397744417190552,
+      "logps/chosen": -89.37004089355469,
+      "logps/rejected": -124.7223129272461,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.0007335222908295691,
+      "rewards/margins": 0.016615714877843857,
+      "rewards/rejected": -0.01588219217956066,
+      "step": 60
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 8.847754167480884,
+      "learning_rate": 4.282022261367073e-07,
+      "logits/chosen": -0.4965807795524597,
+      "logits/rejected": -1.2134766578674316,
+      "logps/chosen": -86.97933959960938,
+      "logps/rejected": -134.1288299560547,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.0005543470033444464,
+      "rewards/margins": 0.023061692714691162,
+      "rewards/rejected": -0.02250734530389309,
+      "step": 70
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 8.841125645791609,
+      "learning_rate": 3.9760059325148063e-07,
+      "logits/chosen": -0.6324743032455444,
+      "logits/rejected": -1.147215485572815,
+      "logps/chosen": -91.2511978149414,
+      "logps/rejected": -149.07229614257812,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.0010622118134051561,
+      "rewards/margins": 0.032594919204711914,
+      "rewards/rejected": -0.03153270110487938,
+      "step": 80
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 8.826466564074398,
+      "learning_rate": 3.6313675726113475e-07,
+      "logits/chosen": -0.6752112507820129,
+      "logits/rejected": -1.1697431802749634,
+      "logps/chosen": -83.96028900146484,
+      "logps/rejected": -135.6808624267578,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.007847856730222702,
+      "rewards/margins": 0.033581167459487915,
+      "rewards/rejected": -0.04142902046442032,
+      "step": 90
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 8.986918539516395,
+      "learning_rate": 3.2571251897448763e-07,
+      "logits/chosen": -0.7318952083587646,
+      "logits/rejected": -1.1481754779815674,
+      "logps/chosen": -104.59893798828125,
+      "logps/rejected": -132.4669952392578,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.011359195224940777,
+      "rewards/margins": 0.043643511831760406,
+      "rewards/rejected": -0.055002711713314056,
+      "step": 100
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 8.709487324526142,
+      "learning_rate": 2.863071428113726e-07,
+      "logits/chosen": -0.6006079316139221,
+      "logits/rejected": -1.0975641012191772,
+      "logps/chosen": -84.71573638916016,
+      "logps/rejected": -149.96249389648438,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.0164839755743742,
+      "rewards/margins": 0.04694540053606033,
+      "rewards/rejected": -0.06342937797307968,
+      "step": 110
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 9.237752042690602,
+      "learning_rate": 2.459517327993746e-07,
+      "logits/chosen": -0.6878473162651062,
+      "logits/rejected": -1.3123815059661865,
+      "logps/chosen": -90.21903991699219,
+      "logps/rejected": -151.0259552001953,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.019475463777780533,
+      "rewards/margins": 0.07015477865934372,
+      "rewards/rejected": -0.08963023871183395,
+      "step": 120
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 9.286029411981152,
+      "learning_rate": 2.0570225210519433e-07,
+      "logits/chosen": -0.790011465549469,
+      "logits/rejected": -1.2298551797866821,
+      "logps/chosen": -93.1588363647461,
+      "logps/rejected": -151.1782684326172,
+      "loss": 0.661,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.030178239569067955,
+      "rewards/margins": 0.07631012797355652,
+      "rewards/rejected": -0.10648836940526962,
+      "step": 130
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 10.19246601057813,
+      "learning_rate": 1.6661189208729489e-07,
+      "logits/chosen": -0.6443529725074768,
+      "logits/rejected": -1.133919596672058,
+      "logps/chosen": -84.59071350097656,
+      "logps/rejected": -138.85736083984375,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.04141147807240486,
+      "rewards/margins": 0.07676441967487335,
+      "rewards/rejected": -0.11817590147256851,
+      "step": 140
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 9.407025901804323,
+      "learning_rate": 1.2970351387729872e-07,
+      "logits/chosen": -0.73204106092453,
+      "logits/rejected": -1.1557518243789673,
+      "logps/chosen": -88.95694732666016,
+      "logps/rejected": -131.26490783691406,
+      "loss": 0.652,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.03920191153883934,
+      "rewards/margins": 0.08074145019054413,
+      "rewards/rejected": -0.11994334310293198,
+      "step": 150
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 8.654405006026039,
+      "learning_rate": 9.594288359976815e-08,
+      "logits/chosen": -0.7579759359359741,
+      "logits/rejected": -1.263896107673645,
+      "logps/chosen": -86.85200500488281,
+      "logps/rejected": -138.72409057617188,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.047876037657260895,
+      "rewards/margins": 0.09631510823965073,
+      "rewards/rejected": -0.14419114589691162,
+      "step": 160
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 8.716288122934651,
+      "learning_rate": 6.621340157319996e-08,
+      "logits/chosen": -0.736527681350708,
+      "logits/rejected": -1.3233287334442139,
+      "logps/chosen": -99.1356430053711,
+      "logps/rejected": -159.6076202392578,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.06683658063411713,
+      "rewards/margins": 0.10212481021881104,
+      "rewards/rejected": -0.16896137595176697,
+      "step": 170
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 9.622323797016332,
+      "learning_rate": 4.1292986742682254e-08,
+      "logits/chosen": -0.7446157932281494,
+      "logits/rejected": -1.380061149597168,
+      "logps/chosen": -96.81314849853516,
+      "logps/rejected": -152.16970825195312,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.06702615320682526,
+      "rewards/margins": 0.1122649684548378,
+      "rewards/rejected": -0.17929109930992126,
+      "step": 180
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 10.2499777892521,
+      "learning_rate": 2.183372119961499e-08,
+      "logits/chosen": -0.695759117603302,
+      "logits/rejected": -1.323012351989746,
+      "logps/chosen": -100.9570541381836,
+      "logps/rejected": -148.34474182128906,
+      "loss": 0.6454,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.07095570862293243,
+      "rewards/margins": 0.09609047323465347,
+      "rewards/rejected": -0.1670461893081665,
+      "step": 190
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 9.240245078846256,
+      "learning_rate": 8.344787421847216e-09,
+      "logits/chosen": -0.7457176446914673,
+      "logits/rejected": -1.3048522472381592,
+      "logps/chosen": -97.51776885986328,
+      "logps/rejected": -142.3980712890625,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.07124415785074234,
+      "rewards/margins": 0.09479977190494537,
+      "rewards/rejected": -0.16604390740394592,
+      "step": 200
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 9.838380839286996,
+      "learning_rate": 1.1791447083465133e-09,
+      "logits/chosen": -0.8287758827209473,
+      "logits/rejected": -1.403287649154663,
+      "logps/chosen": -104.15522766113281,
+      "logps/rejected": -157.41732788085938,
+      "loss": 0.6432,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.08230911195278168,
+      "rewards/margins": 0.10910389572381973,
+      "rewards/rejected": -0.191413015127182,
+      "step": 210
+    },
+    {
+      "epoch": 1.0,
+      "step": 216,
+      "total_flos": 0.0,
+      "train_loss": 0.6676328546471066,
+      "train_runtime": 2437.1919,
+      "train_samples_per_second": 5.672,
+      "train_steps_per_second": 0.089
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 216,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f148cca32b3315d543b99509396df03dac9d921a3ee1c4af94f03cf1ab332500
+size 6264