aaabiao commited on
Commit
b1baf0c
1 Parent(s): edfc00a

Add files using large-upload tool

Browse files
adapter_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/ML-A100/team/mm/zhangge/models/neo_pt2.6_phase2/sft_ckpts/neo_7B_sft_v0_1_plus/checkpoint-225",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 128,
13
+ "lora_dropout": 0.05,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 128,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "q_proj",
23
+ "k_proj",
24
+ "up_proj",
25
+ "o_proj",
26
+ "gate_proj",
27
+ "down_proj",
28
+ "v_proj"
29
+ ],
30
+ "task_type": "CAUSAL_LM"
31
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:572d1c5529e7a189a9ed68433d1d36377632c32d383d53273e40ae85dd0ab23e
3
+ size 770757112
added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|CLS|>": 64000,
3
+ "<|EOD|>": 64002,
4
+ "<|MASK|>": 64003,
5
+ "<|PAD|>": 64004,
6
+ "<|SEP|>": 64001
7
+ }
all_results.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "eval_logits/chosen": -3.394435405731201,
4
+ "eval_logits/rejected": -3.4397640228271484,
5
+ "eval_logps/chosen": -560.387451171875,
6
+ "eval_logps/rejected": -564.275390625,
7
+ "eval_loss": 0.7548772692680359,
8
+ "eval_rewards/accuracies": 0.46875,
9
+ "eval_rewards/chosen": 3.891627311706543,
10
+ "eval_rewards/diff": -0.19507551193237305,
11
+ "eval_rewards/diff_abs": 1.0197707414627075,
12
+ "eval_rewards/rejected": 3.839632511138916,
13
+ "eval_rewards/student_margin": 0.05199478566646576,
14
+ "eval_rewards/teacher_margin": 0.2470703125,
15
+ "eval_runtime": 26.2432,
16
+ "eval_samples": 1543,
17
+ "eval_samples_per_second": 58.796,
18
+ "eval_steps_per_second": 0.152,
19
+ "train_loss": 0.5412804069633256,
20
+ "train_runtime": 5959.7316,
21
+ "train_samples": 160261,
22
+ "train_samples_per_second": 26.891,
23
+ "train_steps_per_second": 0.14
24
+ }
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/ML-A100/team/mm/zhangge/models/neo_pt2.6_phase2/sft_ckpts/neo_7B_sft_v0_1_plus/checkpoint-225",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 3072,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 24576,
14
+ "max_position_embeddings": 8192,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 16,
17
+ "num_hidden_layers": 28,
18
+ "num_key_value_heads": 16,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_scaling": null,
22
+ "rope_theta": 10000.0,
23
+ "tie_word_embeddings": false,
24
+ "torch_dtype": "bfloat16",
25
+ "transformers_version": "4.39.0.dev0",
26
+ "use_cache": true,
27
+ "vocab_size": 64256
28
+ }
eval_results.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "eval_logits/chosen": -3.394435405731201,
4
+ "eval_logits/rejected": -3.4397640228271484,
5
+ "eval_logps/chosen": -560.387451171875,
6
+ "eval_logps/rejected": -564.275390625,
7
+ "eval_loss": 0.7548772692680359,
8
+ "eval_rewards/accuracies": 0.46875,
9
+ "eval_rewards/chosen": 3.891627311706543,
10
+ "eval_rewards/diff": -0.19507551193237305,
11
+ "eval_rewards/diff_abs": 1.0197707414627075,
12
+ "eval_rewards/rejected": 3.839632511138916,
13
+ "eval_rewards/student_margin": 0.05199478566646576,
14
+ "eval_rewards/teacher_margin": 0.2470703125,
15
+ "eval_runtime": 26.2432,
16
+ "eval_samples": 1543,
17
+ "eval_samples_per_second": 58.796,
18
+ "eval_steps_per_second": 0.152
19
+ }
runs/May20_13-38-05_t-20240520213106-dshv6-worker-0/events.out.tfevents.1716213768.t-20240520213106-dshv6-worker-0.6614.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cde8daa162be32359f4138c90388b85d5a21217a6c39d182022f41ddc18e345
3
+ size 79233
runs/May20_13-38-05_t-20240520213106-dshv6-worker-0/events.out.tfevents.1716219755.t-20240520213106-dshv6-worker-0.6614.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe725266237857a8524953acf2f73cc4c15b465e8c83531237a47b9ccaced3ab
3
+ size 1017
special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|CLS|>",
4
+ "<|SEP|>",
5
+ "<|EOD|>",
6
+ "<|MASK|>",
7
+ "<|PAD|>"
8
+ ],
9
+ "bos_token": {
10
+ "content": "<s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "eos_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": true
22
+ },
23
+ "pad_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": true
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": true,
34
+ "rstrip": false,
35
+ "single_word": true
36
+ }
37
+ }
tokenization_hkgpt.py ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 HKGAI Inc. All Rights Reserved.
2
+
3
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
6
+ # and OPT implementations in this library. It has been modified from its
7
+ # original forms to accommodate minor architectural differences compared
8
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the "License");
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at
13
+ #
14
+ # http://www.apache.org/licenses/LICENSE-2.0
15
+ #
16
+ # Unless required by applicable law or agreed to in writing, software
17
+ # distributed under the License is distributed on an "AS IS" BASIS,
18
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
+ # See the License for the specific language governing permissions and
20
+ # limitations under the License.
21
+
22
+ import os
23
+ from shutil import copyfile
24
+ from typing import Any, Dict, List, Optional, Tuple
25
+
26
+ import sentencepiece as spm
27
+
28
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
29
+ from transformers.utils import logging
30
+
31
+
32
+ logger = logging.get_logger(__name__)
33
+
34
+ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
35
+
36
+ PRETRAINED_VOCAB_FILES_MAP = {
37
+ "vocab_file": {},
38
+ "tokenizer_file": {},
39
+ }
40
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
41
+
42
+
43
+ class HKGPTTokenizer(PreTrainedTokenizer):
44
+ """
45
+ Construct a HKGPT tokenizer. Based on byte-level Byte-Pair-Encoding.
46
+
47
+ Args:
48
+ vocab_file (`str`):
49
+ Path to the vocabulary file.
50
+ """
51
+
52
+ vocab_files_names = VOCAB_FILES_NAMES
53
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
54
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
55
+ model_input_names = ["input_ids", "attention_mask"]
56
+
57
+ def __init__(
58
+ self,
59
+ vocab_file,
60
+ unk_token="<unk>",
61
+ bos_token="<s>",
62
+ eos_token="</s>",
63
+ pad_token=None,
64
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
65
+ add_bos_token=True,
66
+ add_eos_token=False,
67
+ clean_up_tokenization_spaces=False,
68
+ **kwargs,
69
+ ):
70
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
71
+ bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
72
+ eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
73
+ unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
74
+ pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
75
+
76
+ self.vocab_file = vocab_file
77
+ self.add_bos_token = add_bos_token
78
+ self.add_eos_token = add_eos_token
79
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
80
+ self.sp_model.Load(vocab_file)
81
+
82
+ super().__init__(
83
+ bos_token=bos_token,
84
+ eos_token=eos_token,
85
+ unk_token=unk_token,
86
+ pad_token=pad_token,
87
+ add_bos_token=add_bos_token,
88
+ add_eos_token=add_eos_token,
89
+ sp_model_kwargs=self.sp_model_kwargs,
90
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
91
+ **kwargs,
92
+ )
93
+
94
+ def __getstate__(self):
95
+ state = self.__dict__.copy()
96
+ state["sp_model"] = None
97
+ return state
98
+
99
+ def __setstate__(self, d):
100
+ self.__dict__ = d
101
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
102
+ self.sp_model.Load(self.vocab_file)
103
+
104
+ @property
105
+ def vocab_size(self):
106
+ """Returns vocab size"""
107
+ return self.sp_model.get_piece_size()
108
+
109
+ def get_vocab(self):
110
+ """Returns vocab as a dict"""
111
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
112
+ vocab.update(self.added_tokens_encoder)
113
+ return vocab
114
+
115
+ def _tokenize(self, text):
116
+ """Returns a tokenized string."""
117
+ return self.sp_model.encode(text, out_type=str)
118
+
119
+ def _convert_token_to_id(self, token):
120
+ """Converts a token (str) in an id using the vocab."""
121
+ return self.sp_model.piece_to_id(token)
122
+
123
+ def _convert_id_to_token(self, index):
124
+ """Converts an index (integer) in a token (str) using the vocab."""
125
+ token = self.sp_model.IdToPiece(index)
126
+ return token
127
+
128
+ def convert_tokens_to_string(self, tokens):
129
+ """Converts a sequence of tokens (string) in a single string."""
130
+ current_sub_tokens = []
131
+ out_string = ""
132
+ prev_is_special = False
133
+ for i, token in enumerate(tokens):
134
+ # make sure that special tokens are not decoded using sentencepiece model
135
+ if token in self.all_special_tokens:
136
+ if not prev_is_special and i != 0:
137
+ out_string += " "
138
+ out_string += self.sp_model.decode(current_sub_tokens) + token
139
+ prev_is_special = True
140
+ current_sub_tokens = []
141
+ else:
142
+ current_sub_tokens.append(token)
143
+ prev_is_special = False
144
+ out_string += self.sp_model.decode(current_sub_tokens)
145
+ return out_string
146
+
147
+ def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
148
+ """
149
+ Save the vocabulary and special tokens file to a directory.
150
+
151
+ Args:
152
+ save_directory (`str`):
153
+ The directory in which to save the vocabulary.
154
+
155
+ Returns:
156
+ `Tuple(str)`: Paths to the files saved.
157
+ """
158
+ if not os.path.isdir(save_directory):
159
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
160
+ return
161
+ out_vocab_file = os.path.join(
162
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
163
+ )
164
+
165
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
166
+ copyfile(self.vocab_file, out_vocab_file)
167
+ elif not os.path.isfile(self.vocab_file):
168
+ with open(out_vocab_file, "wb") as fi:
169
+ content_spiece_model = self.sp_model.serialized_model_proto()
170
+ fi.write(content_spiece_model)
171
+
172
+ return (out_vocab_file,)
173
+
174
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
175
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
176
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
177
+
178
+ output = bos_token_id + token_ids_0 + eos_token_id
179
+
180
+ if token_ids_1 is not None:
181
+ output = output + bos_token_id + token_ids_1 + eos_token_id
182
+
183
+ return output
184
+
185
+ def get_special_tokens_mask(
186
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
187
+ ) -> List[int]:
188
+ """
189
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
190
+ special tokens using the tokenizer `prepare_for_model` method.
191
+
192
+ Args:
193
+ token_ids_0 (`List[int]`):
194
+ List of IDs.
195
+ token_ids_1 (`List[int]`, *optional*):
196
+ Optional second list of IDs for sequence pairs.
197
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
198
+ Whether or not the token list is already formatted with special tokens for the model.
199
+
200
+ Returns:
201
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
202
+ """
203
+ if already_has_special_tokens:
204
+ return super().get_special_tokens_mask(
205
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
206
+ )
207
+
208
+ bos_token_id = [1] if self.add_bos_token else []
209
+ eos_token_id = [1] if self.add_eos_token else []
210
+
211
+ if token_ids_1 is None:
212
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
213
+ return (
214
+ bos_token_id
215
+ + ([0] * len(token_ids_0))
216
+ + eos_token_id
217
+ + bos_token_id
218
+ + ([0] * len(token_ids_1))
219
+ + eos_token_id
220
+ )
221
+
222
+ def create_token_type_ids_from_sequences(
223
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
224
+ ) -> List[int]:
225
+ """
226
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
227
+ sequence pair mask has the following format:
228
+
229
+ ```
230
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
231
+ | first sequence | second sequence |
232
+ ```
233
+
234
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
235
+
236
+ Args:
237
+ token_ids_0 (`List[int]`):
238
+ List of ids.
239
+ token_ids_1 (`List[int]`, *optional*):
240
+ Optional second list of IDs for sequence pairs.
241
+
242
+ Returns:
243
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
244
+ """
245
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
246
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
247
+
248
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
249
+
250
+ if token_ids_1 is not None:
251
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
252
+
253
+ return output
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6a2447b0e5664cabb2481587597102d82f42f0ccb7ef22e1c2d95494a8b03c5
3
+ size 1002561
tokenizer_config.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": true,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": true,
25
+ "rstrip": false,
26
+ "single_word": true,
27
+ "special": true
28
+ },
29
+ "64000": {
30
+ "content": "<|CLS|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "64001": {
38
+ "content": "<|SEP|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "64002": {
46
+ "content": "<|EOD|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "64003": {
54
+ "content": "<|MASK|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "64004": {
62
+ "content": "<|PAD|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ }
69
+ },
70
+ "additional_special_tokens": [
71
+ "<|CLS|>",
72
+ "<|SEP|>",
73
+ "<|EOD|>",
74
+ "<|MASK|>",
75
+ "<|PAD|>"
76
+ ],
77
+ "auto_map": {
78
+ "AutoTokenizer": [
79
+ "tokenization_hkgpt.HKGPTTokenizer",
80
+ null
81
+ ]
82
+ },
83
+ "bos_token": "<s>",
84
+ "chat_template": "{% set system_message = 'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\\n\\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don\\'t know the answer to a question, please don\\'t share false information.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '<s>' + '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
85
+ "clean_up_tokenization_spaces": false,
86
+ "eos_token": "</s>",
87
+ "model_max_length": 4096,
88
+ "pad_token": "<unk>",
89
+ "padding_side": "right",
90
+ "sp_model_kwargs": {},
91
+ "split_special_tokens": false,
92
+ "tokenizer_class": "HKGPTTokenizer",
93
+ "unk_token": "<unk>",
94
+ "use_fast": false
95
+ }
trainer_state.json ADDED
@@ -0,0 +1,1542 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 800000000,
6
+ "global_step": 835,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "grad_norm": 8.9375,
14
+ "learning_rate": 5.952380952380953e-08,
15
+ "logits/chosen": -3.4845848083496094,
16
+ "logits/rejected": -3.85036301612854,
17
+ "logps/chosen": -306.50885009765625,
18
+ "logps/rejected": -197.74395751953125,
19
+ "loss": 0.6931,
20
+ "rewards/accuracies": 0.0,
21
+ "rewards/chosen": 0.0,
22
+ "rewards/diff": -0.625,
23
+ "rewards/diff_abs": 0.625,
24
+ "rewards/rejected": 0.0,
25
+ "rewards/student_margin": 0.0,
26
+ "rewards/teacher_margin": 0.625,
27
+ "step": 1
28
+ },
29
+ {
30
+ "epoch": 0.01,
31
+ "grad_norm": 8.8125,
32
+ "learning_rate": 5.952380952380953e-07,
33
+ "logits/chosen": -3.454127788543701,
34
+ "logits/rejected": -3.5237815380096436,
35
+ "logps/chosen": -201.42767333984375,
36
+ "logps/rejected": -183.9016571044922,
37
+ "loss": 0.7039,
38
+ "rewards/accuracies": 0.4444444477558136,
39
+ "rewards/chosen": 0.005284797865897417,
40
+ "rewards/diff": -0.6895497441291809,
41
+ "rewards/diff_abs": 0.7025125026702881,
42
+ "rewards/rejected": 0.01312162820249796,
43
+ "rewards/student_margin": -0.007836826145648956,
44
+ "rewards/teacher_margin": 0.6817129254341125,
45
+ "step": 10
46
+ },
47
+ {
48
+ "epoch": 0.02,
49
+ "grad_norm": 8.5,
50
+ "learning_rate": 1.1904761904761906e-06,
51
+ "logits/chosen": -3.5940723419189453,
52
+ "logits/rejected": -3.5770275592803955,
53
+ "logps/chosen": -218.02499389648438,
54
+ "logps/rejected": -209.6902313232422,
55
+ "loss": 0.7101,
56
+ "rewards/accuracies": 0.4333333373069763,
57
+ "rewards/chosen": -0.025884132832288742,
58
+ "rewards/diff": -0.9974073171615601,
59
+ "rewards/diff_abs": 1.025899887084961,
60
+ "rewards/rejected": -0.01753927394747734,
61
+ "rewards/student_margin": -0.00834486074745655,
62
+ "rewards/teacher_margin": 0.9890626072883606,
63
+ "step": 20
64
+ },
65
+ {
66
+ "epoch": 0.04,
67
+ "grad_norm": 7.9375,
68
+ "learning_rate": 1.7857142857142859e-06,
69
+ "logits/chosen": -3.4890503883361816,
70
+ "logits/rejected": -3.6032581329345703,
71
+ "logps/chosen": -259.52838134765625,
72
+ "logps/rejected": -200.54518127441406,
73
+ "loss": 0.6961,
74
+ "rewards/accuracies": 0.5666666626930237,
75
+ "rewards/chosen": 0.11190159618854523,
76
+ "rewards/diff": -0.870490550994873,
77
+ "rewards/diff_abs": 0.910025417804718,
78
+ "rewards/rejected": 0.07041291892528534,
79
+ "rewards/student_margin": 0.04148866608738899,
80
+ "rewards/teacher_margin": 0.911979079246521,
81
+ "step": 30
82
+ },
83
+ {
84
+ "epoch": 0.05,
85
+ "grad_norm": 7.8125,
86
+ "learning_rate": 2.380952380952381e-06,
87
+ "logits/chosen": -3.4495646953582764,
88
+ "logits/rejected": -3.5306625366210938,
89
+ "logps/chosen": -296.196044921875,
90
+ "logps/rejected": -205.72494506835938,
91
+ "loss": 0.6763,
92
+ "rewards/accuracies": 0.5333333611488342,
93
+ "rewards/chosen": 0.27481913566589355,
94
+ "rewards/diff": -0.928841233253479,
95
+ "rewards/diff_abs": 0.9340232610702515,
96
+ "rewards/rejected": 0.1948060244321823,
97
+ "rewards/student_margin": 0.08001308888196945,
98
+ "rewards/teacher_margin": 1.0088541507720947,
99
+ "step": 40
100
+ },
101
+ {
102
+ "epoch": 0.06,
103
+ "grad_norm": 7.46875,
104
+ "learning_rate": 2.9761904761904763e-06,
105
+ "logits/chosen": -3.6242897510528564,
106
+ "logits/rejected": -3.6223366260528564,
107
+ "logps/chosen": -232.1892852783203,
108
+ "logps/rejected": -218.8447265625,
109
+ "loss": 0.6524,
110
+ "rewards/accuracies": 0.6333333253860474,
111
+ "rewards/chosen": 0.5324563980102539,
112
+ "rewards/diff": -1.090423583984375,
113
+ "rewards/diff_abs": 1.1463050842285156,
114
+ "rewards/rejected": 0.3765257000923157,
115
+ "rewards/student_margin": 0.15593069791793823,
116
+ "rewards/teacher_margin": 1.2463542222976685,
117
+ "step": 50
118
+ },
119
+ {
120
+ "epoch": 0.07,
121
+ "grad_norm": 7.5,
122
+ "learning_rate": 3.5714285714285718e-06,
123
+ "logits/chosen": -3.52375864982605,
124
+ "logits/rejected": -3.5178802013397217,
125
+ "logps/chosen": -278.49578857421875,
126
+ "logps/rejected": -227.9744110107422,
127
+ "loss": 0.6501,
128
+ "rewards/accuracies": 0.6999999284744263,
129
+ "rewards/chosen": 0.9247525930404663,
130
+ "rewards/diff": -0.6484954953193665,
131
+ "rewards/diff_abs": 0.8814946413040161,
132
+ "rewards/rejected": 0.6779355406761169,
133
+ "rewards/student_margin": 0.24681702256202698,
134
+ "rewards/teacher_margin": 0.895312488079071,
135
+ "step": 60
136
+ },
137
+ {
138
+ "epoch": 0.08,
139
+ "grad_norm": 7.28125,
140
+ "learning_rate": 4.166666666666667e-06,
141
+ "logits/chosen": -3.57179594039917,
142
+ "logits/rejected": -3.573483943939209,
143
+ "logps/chosen": -299.92742919921875,
144
+ "logps/rejected": -295.48846435546875,
145
+ "loss": 0.6298,
146
+ "rewards/accuracies": 0.6666666269302368,
147
+ "rewards/chosen": 1.0992387533187866,
148
+ "rewards/diff": -0.5216845273971558,
149
+ "rewards/diff_abs": 0.7238657474517822,
150
+ "rewards/rejected": 0.8657148480415344,
151
+ "rewards/student_margin": 0.23352384567260742,
152
+ "rewards/teacher_margin": 0.7552083730697632,
153
+ "step": 70
154
+ },
155
+ {
156
+ "epoch": 0.1,
157
+ "grad_norm": 7.15625,
158
+ "learning_rate": 4.761904761904762e-06,
159
+ "logits/chosen": -3.3535995483398438,
160
+ "logits/rejected": -3.4229187965393066,
161
+ "logps/chosen": -306.759521484375,
162
+ "logps/rejected": -193.92160034179688,
163
+ "loss": 0.6005,
164
+ "rewards/accuracies": 0.8333333730697632,
165
+ "rewards/chosen": 1.2742681503295898,
166
+ "rewards/diff": -0.2789258360862732,
167
+ "rewards/diff_abs": 0.8034403920173645,
168
+ "rewards/rejected": 0.5938189625740051,
169
+ "rewards/student_margin": 0.6804491281509399,
170
+ "rewards/teacher_margin": 0.9593750238418579,
171
+ "step": 80
172
+ },
173
+ {
174
+ "epoch": 0.11,
175
+ "grad_norm": 6.3125,
176
+ "learning_rate": 4.9992125742993825e-06,
177
+ "logits/chosen": -3.5169739723205566,
178
+ "logits/rejected": -3.478895664215088,
179
+ "logps/chosen": -305.4494323730469,
180
+ "logps/rejected": -259.5570373535156,
181
+ "loss": 0.5942,
182
+ "rewards/accuracies": 0.7333333492279053,
183
+ "rewards/chosen": 1.40164053440094,
184
+ "rewards/diff": -0.47005853056907654,
185
+ "rewards/diff_abs": 0.7208673357963562,
186
+ "rewards/rejected": 1.1060739755630493,
187
+ "rewards/student_margin": 0.29556649923324585,
188
+ "rewards/teacher_margin": 0.765625,
189
+ "step": 90
190
+ },
191
+ {
192
+ "epoch": 0.12,
193
+ "grad_norm": 7.0625,
194
+ "learning_rate": 4.994402324561469e-06,
195
+ "logits/chosen": -3.475271701812744,
196
+ "logits/rejected": -3.4696757793426514,
197
+ "logps/chosen": -290.073974609375,
198
+ "logps/rejected": -212.7600860595703,
199
+ "loss": 0.6009,
200
+ "rewards/accuracies": 0.800000011920929,
201
+ "rewards/chosen": 1.396482229232788,
202
+ "rewards/diff": -0.12815351784229279,
203
+ "rewards/diff_abs": 0.5424238443374634,
204
+ "rewards/rejected": 0.6892191171646118,
205
+ "rewards/student_margin": 0.7072631120681763,
206
+ "rewards/teacher_margin": 0.8354166746139526,
207
+ "step": 100
208
+ },
209
+ {
210
+ "epoch": 0.13,
211
+ "grad_norm": 7.03125,
212
+ "learning_rate": 4.985227689958313e-06,
213
+ "logits/chosen": -3.4492225646972656,
214
+ "logits/rejected": -3.490285873413086,
215
+ "logps/chosen": -309.30743408203125,
216
+ "logps/rejected": -202.38356018066406,
217
+ "loss": 0.5763,
218
+ "rewards/accuracies": 0.6666666269302368,
219
+ "rewards/chosen": 1.2701516151428223,
220
+ "rewards/diff": -0.4000614583492279,
221
+ "rewards/diff_abs": 0.8188215494155884,
222
+ "rewards/rejected": 0.8035463094711304,
223
+ "rewards/student_margin": 0.46660518646240234,
224
+ "rewards/teacher_margin": 0.8666666746139526,
225
+ "step": 110
226
+ },
227
+ {
228
+ "epoch": 0.14,
229
+ "grad_norm": 6.625,
230
+ "learning_rate": 4.97170472308737e-06,
231
+ "logits/chosen": -3.537369966506958,
232
+ "logits/rejected": -3.5341758728027344,
233
+ "logps/chosen": -238.89035034179688,
234
+ "logps/rejected": -219.8264617919922,
235
+ "loss": 0.5923,
236
+ "rewards/accuracies": 0.6333333253860474,
237
+ "rewards/chosen": 1.1539257764816284,
238
+ "rewards/diff": -0.407745361328125,
239
+ "rewards/diff_abs": 0.8213578462600708,
240
+ "rewards/rejected": 0.6814627051353455,
241
+ "rewards/student_margin": 0.47246304154396057,
242
+ "rewards/teacher_margin": 0.8802083134651184,
243
+ "step": 120
244
+ },
245
+ {
246
+ "epoch": 0.16,
247
+ "grad_norm": 6.3125,
248
+ "learning_rate": 4.953857084699501e-06,
249
+ "logits/chosen": -3.3898227214813232,
250
+ "logits/rejected": -3.445030689239502,
251
+ "logps/chosen": -237.6462860107422,
252
+ "logps/rejected": -189.9452667236328,
253
+ "loss": 0.5986,
254
+ "rewards/accuracies": 0.7666666507720947,
255
+ "rewards/chosen": 1.2968804836273193,
256
+ "rewards/diff": -0.1380801498889923,
257
+ "rewards/diff_abs": 0.6938132643699646,
258
+ "rewards/rejected": 0.6688148379325867,
259
+ "rewards/student_margin": 0.6280657649040222,
260
+ "rewards/teacher_margin": 0.7661458253860474,
261
+ "step": 130
262
+ },
263
+ {
264
+ "epoch": 0.17,
265
+ "grad_norm": 6.9375,
266
+ "learning_rate": 4.931716002300424e-06,
267
+ "logits/chosen": -3.4307568073272705,
268
+ "logits/rejected": -3.428516387939453,
269
+ "logps/chosen": -303.8690490722656,
270
+ "logps/rejected": -268.0577087402344,
271
+ "loss": 0.5674,
272
+ "rewards/accuracies": 0.76666659116745,
273
+ "rewards/chosen": 1.502533197402954,
274
+ "rewards/diff": -0.11084864288568497,
275
+ "rewards/diff_abs": 0.7653725743293762,
276
+ "rewards/rejected": 0.7842152714729309,
277
+ "rewards/student_margin": 0.7183180451393127,
278
+ "rewards/teacher_margin": 0.8291667103767395,
279
+ "step": 140
280
+ },
281
+ {
282
+ "epoch": 0.18,
283
+ "grad_norm": 5.90625,
284
+ "learning_rate": 4.905320215512843e-06,
285
+ "logits/chosen": -3.3582215309143066,
286
+ "logits/rejected": -3.445798397064209,
287
+ "logps/chosen": -272.33465576171875,
288
+ "logps/rejected": -241.3258819580078,
289
+ "loss": 0.5839,
290
+ "rewards/accuracies": 0.6666666865348816,
291
+ "rewards/chosen": 0.9148601293563843,
292
+ "rewards/diff": -0.40072593092918396,
293
+ "rewards/diff_abs": 0.7023404240608215,
294
+ "rewards/rejected": 0.5593360662460327,
295
+ "rewards/student_margin": 0.35552406311035156,
296
+ "rewards/teacher_margin": 0.7562500238418579,
297
+ "step": 150
298
+ },
299
+ {
300
+ "epoch": 0.19,
301
+ "grad_norm": 6.5625,
302
+ "learning_rate": 4.874715908294827e-06,
303
+ "logits/chosen": -3.4379913806915283,
304
+ "logits/rejected": -3.411599636077881,
305
+ "logps/chosen": -235.85824584960938,
306
+ "logps/rejected": -200.01751708984375,
307
+ "loss": 0.5673,
308
+ "rewards/accuracies": 0.7333332896232605,
309
+ "rewards/chosen": 0.91960209608078,
310
+ "rewards/diff": -0.3414815068244934,
311
+ "rewards/diff_abs": 0.8401057124137878,
312
+ "rewards/rejected": 0.38087528944015503,
313
+ "rewards/student_margin": 0.5387269258499146,
314
+ "rewards/teacher_margin": 0.8802083730697632,
315
+ "step": 160
316
+ },
317
+ {
318
+ "epoch": 0.2,
319
+ "grad_norm": 6.875,
320
+ "learning_rate": 4.839956628133049e-06,
321
+ "logits/chosen": -3.3944404125213623,
322
+ "logits/rejected": -3.4562854766845703,
323
+ "logps/chosen": -236.4658203125,
324
+ "logps/rejected": -207.5730438232422,
325
+ "loss": 0.5312,
326
+ "rewards/accuracies": 0.7333333492279053,
327
+ "rewards/chosen": 1.1229525804519653,
328
+ "rewards/diff": -0.3498944342136383,
329
+ "rewards/diff_abs": 0.8671269416809082,
330
+ "rewards/rejected": 0.43118032813072205,
331
+ "rewards/student_margin": 0.6917722821235657,
332
+ "rewards/teacher_margin": 1.0416667461395264,
333
+ "step": 170
334
+ },
335
+ {
336
+ "epoch": 0.22,
337
+ "grad_norm": 6.8125,
338
+ "learning_rate": 4.801103192352272e-06,
339
+ "logits/chosen": -3.5573208332061768,
340
+ "logits/rejected": -3.619119167327881,
341
+ "logps/chosen": -342.3301086425781,
342
+ "logps/rejected": -242.18148803710938,
343
+ "loss": 0.5428,
344
+ "rewards/accuracies": 0.699999988079071,
345
+ "rewards/chosen": 1.7506067752838135,
346
+ "rewards/diff": -0.05490243434906006,
347
+ "rewards/diff_abs": 1.2669219970703125,
348
+ "rewards/rejected": 0.852384090423584,
349
+ "rewards/student_margin": 0.8982225656509399,
350
+ "rewards/teacher_margin": 0.9531251192092896,
351
+ "step": 180
352
+ },
353
+ {
354
+ "epoch": 0.23,
355
+ "grad_norm": 6.75,
356
+ "learning_rate": 4.758223581705006e-06,
357
+ "logits/chosen": -3.493630886077881,
358
+ "logits/rejected": -3.531799793243408,
359
+ "logps/chosen": -242.2713623046875,
360
+ "logps/rejected": -195.7411346435547,
361
+ "loss": 0.5605,
362
+ "rewards/accuracies": 0.6333333253860474,
363
+ "rewards/chosen": 1.3145856857299805,
364
+ "rewards/diff": -0.18605072796344757,
365
+ "rewards/diff_abs": 0.9721413850784302,
366
+ "rewards/rejected": 0.7516780495643616,
367
+ "rewards/student_margin": 0.5629075765609741,
368
+ "rewards/teacher_margin": 0.7489583492279053,
369
+ "step": 190
370
+ },
371
+ {
372
+ "epoch": 0.24,
373
+ "grad_norm": 7.375,
374
+ "learning_rate": 4.711392821427515e-06,
375
+ "logits/chosen": -3.5924346446990967,
376
+ "logits/rejected": -3.6110892295837402,
377
+ "logps/chosen": -231.8784942626953,
378
+ "logps/rejected": -160.18458557128906,
379
+ "loss": 0.5528,
380
+ "rewards/accuracies": 0.76666659116745,
381
+ "rewards/chosen": 1.1364099979400635,
382
+ "rewards/diff": -0.20100148022174835,
383
+ "rewards/diff_abs": 0.8547786474227905,
384
+ "rewards/rejected": 0.13949476182460785,
385
+ "rewards/student_margin": 0.9969152212142944,
386
+ "rewards/teacher_margin": 1.1979167461395264,
387
+ "step": 200
388
+ },
389
+ {
390
+ "epoch": 0.25,
391
+ "grad_norm": 6.09375,
392
+ "learning_rate": 4.6606928499702905e-06,
393
+ "logits/chosen": -3.583310604095459,
394
+ "logits/rejected": -3.646390914916992,
395
+ "logps/chosen": -236.2954864501953,
396
+ "logps/rejected": -226.30050659179688,
397
+ "loss": 0.5455,
398
+ "rewards/accuracies": 0.699999988079071,
399
+ "rewards/chosen": 1.1413193941116333,
400
+ "rewards/diff": -0.6167432069778442,
401
+ "rewards/diff_abs": 0.9270604252815247,
402
+ "rewards/rejected": 0.8591042757034302,
403
+ "rewards/student_margin": 0.2822151482105255,
404
+ "rewards/teacher_margin": 0.8989583849906921,
405
+ "step": 210
406
+ },
407
+ {
408
+ "epoch": 0.26,
409
+ "grad_norm": 6.0625,
410
+ "learning_rate": 4.606212375632682e-06,
411
+ "logits/chosen": -3.3313984870910645,
412
+ "logits/rejected": -3.4001998901367188,
413
+ "logps/chosen": -241.50430297851562,
414
+ "logps/rejected": -185.7382049560547,
415
+ "loss": 0.5455,
416
+ "rewards/accuracies": 0.76666659116745,
417
+ "rewards/chosen": 1.245307207107544,
418
+ "rewards/diff": -0.19214758276939392,
419
+ "rewards/diff_abs": 0.9847003817558289,
420
+ "rewards/rejected": 0.4134964942932129,
421
+ "rewards/student_margin": 0.8318107724189758,
422
+ "rewards/teacher_margin": 1.023958444595337,
423
+ "step": 220
424
+ },
425
+ {
426
+ "epoch": 0.28,
427
+ "grad_norm": 7.59375,
428
+ "learning_rate": 4.5480467213524935e-06,
429
+ "logits/chosen": -3.4316277503967285,
430
+ "logits/rejected": -3.4833552837371826,
431
+ "logps/chosen": -258.82781982421875,
432
+ "logps/rejected": -248.5332489013672,
433
+ "loss": 0.5478,
434
+ "rewards/accuracies": 0.7333332896232605,
435
+ "rewards/chosen": 1.38356614112854,
436
+ "rewards/diff": -0.1715225726366043,
437
+ "rewards/diff_abs": 0.7107259631156921,
438
+ "rewards/rejected": 0.7259219288825989,
439
+ "rewards/student_margin": 0.6576440930366516,
440
+ "rewards/teacher_margin": 0.8291667699813843,
441
+ "step": 230
442
+ },
443
+ {
444
+ "epoch": 0.29,
445
+ "grad_norm": 6.40625,
446
+ "learning_rate": 4.4862976579221605e-06,
447
+ "logits/chosen": -3.3932158946990967,
448
+ "logits/rejected": -3.4250049591064453,
449
+ "logps/chosen": -303.9991149902344,
450
+ "logps/rejected": -221.3593292236328,
451
+ "loss": 0.5413,
452
+ "rewards/accuracies": 0.7666666507720947,
453
+ "rewards/chosen": 1.6610606908798218,
454
+ "rewards/diff": -0.16523823142051697,
455
+ "rewards/diff_abs": 0.8983039855957031,
456
+ "rewards/rejected": 0.6669239401817322,
457
+ "rewards/student_margin": 0.9941369295120239,
458
+ "rewards/teacher_margin": 1.1593749523162842,
459
+ "step": 240
460
+ },
461
+ {
462
+ "epoch": 0.3,
463
+ "grad_norm": 6.125,
464
+ "learning_rate": 4.421073225923276e-06,
465
+ "logits/chosen": -3.4080328941345215,
466
+ "logits/rejected": -3.545672655105591,
467
+ "logps/chosen": -302.5151062011719,
468
+ "logps/rejected": -223.77474975585938,
469
+ "loss": 0.5379,
470
+ "rewards/accuracies": 0.6666666269302368,
471
+ "rewards/chosen": 1.6596599817276,
472
+ "rewards/diff": -0.011551931500434875,
473
+ "rewards/diff_abs": 0.9808802604675293,
474
+ "rewards/rejected": 0.7274617552757263,
475
+ "rewards/student_margin": 0.9321980476379395,
476
+ "rewards/teacher_margin": 0.9437500238418579,
477
+ "step": 250
478
+ },
479
+ {
480
+ "epoch": 0.31,
481
+ "grad_norm": 7.28125,
482
+ "learning_rate": 4.3524875466910634e-06,
483
+ "logits/chosen": -3.377882719039917,
484
+ "logits/rejected": -3.380521059036255,
485
+ "logps/chosen": -247.70703125,
486
+ "logps/rejected": -240.69363403320312,
487
+ "loss": 0.5479,
488
+ "rewards/accuracies": 0.5666667222976685,
489
+ "rewards/chosen": 0.9718164205551147,
490
+ "rewards/diff": -0.17575177550315857,
491
+ "rewards/diff_abs": 0.8040043711662292,
492
+ "rewards/rejected": 0.5954850316047668,
493
+ "rewards/student_margin": 0.3763315677642822,
494
+ "rewards/teacher_margin": 0.5520833730697632,
495
+ "step": 260
496
+ },
497
+ {
498
+ "epoch": 0.32,
499
+ "grad_norm": 6.4375,
500
+ "learning_rate": 4.280660622639513e-06,
501
+ "logits/chosen": -3.5067367553710938,
502
+ "logits/rejected": -3.5205013751983643,
503
+ "logps/chosen": -237.2410125732422,
504
+ "logps/rejected": -190.3438720703125,
505
+ "loss": 0.5352,
506
+ "rewards/accuracies": 0.7333333492279053,
507
+ "rewards/chosen": 1.3214976787567139,
508
+ "rewards/diff": -0.030149614438414574,
509
+ "rewards/diff_abs": 0.9027697443962097,
510
+ "rewards/rejected": 0.4870639443397522,
511
+ "rewards/student_margin": 0.8344337344169617,
512
+ "rewards/teacher_margin": 0.8645833730697632,
513
+ "step": 270
514
+ },
515
+ {
516
+ "epoch": 0.34,
517
+ "grad_norm": 6.0625,
518
+ "learning_rate": 4.205718127296574e-06,
519
+ "logits/chosen": -3.5430946350097656,
520
+ "logits/rejected": -3.5217278003692627,
521
+ "logps/chosen": -239.96188354492188,
522
+ "logps/rejected": -210.2650604248047,
523
+ "loss": 0.5277,
524
+ "rewards/accuracies": 0.7333332896232605,
525
+ "rewards/chosen": 1.366132140159607,
526
+ "rewards/diff": -0.2834340035915375,
527
+ "rewards/diff_abs": 1.1607670783996582,
528
+ "rewards/rejected": 0.7917537689208984,
529
+ "rewards/student_margin": 0.5743785500526428,
530
+ "rewards/teacher_margin": 0.8578125238418579,
531
+ "step": 280
532
+ },
533
+ {
534
+ "epoch": 0.35,
535
+ "grad_norm": 6.90625,
536
+ "learning_rate": 4.127791185416747e-06,
537
+ "logits/chosen": -3.410996675491333,
538
+ "logits/rejected": -3.428239345550537,
539
+ "logps/chosen": -218.684326171875,
540
+ "logps/rejected": -173.12939453125,
541
+ "loss": 0.5492,
542
+ "rewards/accuracies": 0.7000000476837158,
543
+ "rewards/chosen": 1.1556932926177979,
544
+ "rewards/diff": -0.2741561830043793,
545
+ "rewards/diff_abs": 0.9800466299057007,
546
+ "rewards/rejected": 0.501724362373352,
547
+ "rewards/student_margin": 0.6539688110351562,
548
+ "rewards/teacher_margin": 0.9281250834465027,
549
+ "step": 290
550
+ },
551
+ {
552
+ "epoch": 0.36,
553
+ "grad_norm": 5.6875,
554
+ "learning_rate": 4.047016143555834e-06,
555
+ "logits/chosen": -3.4146499633789062,
556
+ "logits/rejected": -3.4334769248962402,
557
+ "logps/chosen": -246.2406005859375,
558
+ "logps/rejected": -207.5984344482422,
559
+ "loss": 0.5396,
560
+ "rewards/accuracies": 0.7333332896232605,
561
+ "rewards/chosen": 1.4659597873687744,
562
+ "rewards/diff": -0.0349099263548851,
563
+ "rewards/diff_abs": 0.7459529042243958,
564
+ "rewards/rejected": 0.5878490209579468,
565
+ "rewards/student_margin": 0.8781110048294067,
566
+ "rewards/teacher_margin": 0.91302090883255,
567
+ "step": 300
568
+ },
569
+ {
570
+ "epoch": 0.37,
571
+ "grad_norm": 6.75,
572
+ "learning_rate": 3.9635343315092374e-06,
573
+ "logits/chosen": -3.3409626483917236,
574
+ "logits/rejected": -3.4818501472473145,
575
+ "logps/chosen": -242.3018341064453,
576
+ "logps/rejected": -209.86740112304688,
577
+ "loss": 0.5499,
578
+ "rewards/accuracies": 0.800000011920929,
579
+ "rewards/chosen": 1.246741533279419,
580
+ "rewards/diff": -0.13802729547023773,
581
+ "rewards/diff_abs": 0.9692344665527344,
582
+ "rewards/rejected": 0.40664371848106384,
583
+ "rewards/student_margin": 0.8400977253913879,
584
+ "rewards/teacher_margin": 0.9781249165534973,
585
+ "step": 310
586
+ },
587
+ {
588
+ "epoch": 0.38,
589
+ "grad_norm": 6.78125,
590
+ "learning_rate": 3.877491815031241e-06,
591
+ "logits/chosen": -3.50838041305542,
592
+ "logits/rejected": -3.6322741508483887,
593
+ "logps/chosen": -257.099609375,
594
+ "logps/rejected": -179.91046142578125,
595
+ "loss": 0.526,
596
+ "rewards/accuracies": 0.9666666984558105,
597
+ "rewards/chosen": 1.3539568185806274,
598
+ "rewards/diff": 0.1423492729663849,
599
+ "rewards/diff_abs": 0.7491869926452637,
600
+ "rewards/rejected": 0.354315847158432,
601
+ "rewards/student_margin": 0.999640941619873,
602
+ "rewards/teacher_margin": 0.8572916984558105,
603
+ "step": 320
604
+ },
605
+ {
606
+ "epoch": 0.4,
607
+ "grad_norm": 6.4375,
608
+ "learning_rate": 3.789039140267903e-06,
609
+ "logits/chosen": -3.6152091026306152,
610
+ "logits/rejected": -3.6335906982421875,
611
+ "logps/chosen": -238.04483032226562,
612
+ "logps/rejected": -203.68545532226562,
613
+ "loss": 0.5211,
614
+ "rewards/accuracies": 0.7666667103767395,
615
+ "rewards/chosen": 1.1535335779190063,
616
+ "rewards/diff": -0.22724106907844543,
617
+ "rewards/diff_abs": 0.8418729901313782,
618
+ "rewards/rejected": 0.3599412739276886,
619
+ "rewards/student_margin": 0.7935922741889954,
620
+ "rewards/teacher_margin": 1.0208333730697632,
621
+ "step": 330
622
+ },
623
+ {
624
+ "epoch": 0.41,
625
+ "grad_norm": 6.25,
626
+ "learning_rate": 3.6983310703507475e-06,
627
+ "logits/chosen": -3.474027633666992,
628
+ "logits/rejected": -3.618129253387451,
629
+ "logps/chosen": -314.650390625,
630
+ "logps/rejected": -292.22796630859375,
631
+ "loss": 0.5083,
632
+ "rewards/accuracies": 0.7333333492279053,
633
+ "rewards/chosen": 1.6438385248184204,
634
+ "rewards/diff": -0.008717024698853493,
635
+ "rewards/diff_abs": 0.9126062393188477,
636
+ "rewards/rejected": 0.937972366809845,
637
+ "rewards/student_margin": 0.7058663368225098,
638
+ "rewards/teacher_margin": 0.7145833969116211,
639
+ "step": 340
640
+ },
641
+ {
642
+ "epoch": 0.42,
643
+ "grad_norm": 6.84375,
644
+ "learning_rate": 3.6055263146121062e-06,
645
+ "logits/chosen": -3.4695258140563965,
646
+ "logits/rejected": -3.544586658477783,
647
+ "logps/chosen": -241.87686157226562,
648
+ "logps/rejected": -190.71157836914062,
649
+ "loss": 0.5241,
650
+ "rewards/accuracies": 0.7333333492279053,
651
+ "rewards/chosen": 1.3957545757293701,
652
+ "rewards/diff": -0.020256221294403076,
653
+ "rewards/diff_abs": 1.0595004558563232,
654
+ "rewards/rejected": 0.6003857851028442,
655
+ "rewards/student_margin": 0.7953688502311707,
656
+ "rewards/teacher_margin": 0.815625011920929,
657
+ "step": 350
658
+ },
659
+ {
660
+ "epoch": 0.43,
661
+ "grad_norm": 6.9375,
662
+ "learning_rate": 3.5107872508959144e-06,
663
+ "logits/chosen": -3.5332858562469482,
664
+ "logits/rejected": -3.658419370651245,
665
+ "logps/chosen": -301.9664001464844,
666
+ "logps/rejected": -229.6895294189453,
667
+ "loss": 0.5268,
668
+ "rewards/accuracies": 0.699999988079071,
669
+ "rewards/chosen": 1.3080447912216187,
670
+ "rewards/diff": 0.03206339478492737,
671
+ "rewards/diff_abs": 1.0055023431777954,
672
+ "rewards/rejected": 0.4827522337436676,
673
+ "rewards/student_margin": 0.8252925872802734,
674
+ "rewards/teacher_margin": 0.7932292222976685,
675
+ "step": 360
676
+ },
677
+ {
678
+ "epoch": 0.44,
679
+ "grad_norm": 6.5625,
680
+ "learning_rate": 3.414279641449809e-06,
681
+ "logits/chosen": -3.4194533824920654,
682
+ "logits/rejected": -3.459688901901245,
683
+ "logps/chosen": -293.80865478515625,
684
+ "logps/rejected": -236.77560424804688,
685
+ "loss": 0.512,
686
+ "rewards/accuracies": 0.7666666507720947,
687
+ "rewards/chosen": 1.4415782690048218,
688
+ "rewards/diff": -0.22951290011405945,
689
+ "rewards/diff_abs": 0.9373496174812317,
690
+ "rewards/rejected": 0.7585911750793457,
691
+ "rewards/student_margin": 0.6829870939254761,
692
+ "rewards/teacher_margin": 0.9125000238418579,
693
+ "step": 370
694
+ },
695
+ {
696
+ "epoch": 0.46,
697
+ "grad_norm": 6.125,
698
+ "learning_rate": 3.3161723428956356e-06,
699
+ "logits/chosen": -3.329397201538086,
700
+ "logits/rejected": -3.4820456504821777,
701
+ "logps/chosen": -303.4757385253906,
702
+ "logps/rejected": -242.2552032470703,
703
+ "loss": 0.5127,
704
+ "rewards/accuracies": 0.699999988079071,
705
+ "rewards/chosen": 1.638362169265747,
706
+ "rewards/diff": -0.17444480955600739,
707
+ "rewards/diff_abs": 0.9962782859802246,
708
+ "rewards/rejected": 0.7367652654647827,
709
+ "rewards/student_margin": 0.9015968441963196,
710
+ "rewards/teacher_margin": 1.0760416984558105,
711
+ "step": 380
712
+ },
713
+ {
714
+ "epoch": 0.47,
715
+ "grad_norm": 6.21875,
716
+ "learning_rate": 3.216637010785813e-06,
717
+ "logits/chosen": -3.547212600708008,
718
+ "logits/rejected": -3.542712688446045,
719
+ "logps/chosen": -321.4063720703125,
720
+ "logps/rejected": -284.3674621582031,
721
+ "loss": 0.5144,
722
+ "rewards/accuracies": 0.7666666507720947,
723
+ "rewards/chosen": 1.7464603185653687,
724
+ "rewards/diff": 0.06983740627765656,
725
+ "rewards/diff_abs": 0.938123881816864,
726
+ "rewards/rejected": 0.7797478437423706,
727
+ "rewards/student_margin": 0.9667123556137085,
728
+ "rewards/teacher_margin": 0.8968750238418579,
729
+ "step": 390
730
+ },
731
+ {
732
+ "epoch": 0.48,
733
+ "grad_norm": 7.40625,
734
+ "learning_rate": 3.115847799262494e-06,
735
+ "logits/chosen": -3.4556503295898438,
736
+ "logits/rejected": -3.5828518867492676,
737
+ "logps/chosen": -256.46868896484375,
738
+ "logps/rejected": -220.39010620117188,
739
+ "loss": 0.5092,
740
+ "rewards/accuracies": 0.8333333730697632,
741
+ "rewards/chosen": 1.3820217847824097,
742
+ "rewards/diff": 0.07980125397443771,
743
+ "rewards/diff_abs": 0.8547611236572266,
744
+ "rewards/rejected": 0.42253294587135315,
745
+ "rewards/student_margin": 0.9594887495040894,
746
+ "rewards/teacher_margin": 0.879687488079071,
747
+ "step": 400
748
+ },
749
+ {
750
+ "epoch": 0.49,
751
+ "grad_norm": 6.03125,
752
+ "learning_rate": 3.0139810563450094e-06,
753
+ "logits/chosen": -3.592397689819336,
754
+ "logits/rejected": -3.6688952445983887,
755
+ "logps/chosen": -292.0425720214844,
756
+ "logps/rejected": -234.98208618164062,
757
+ "loss": 0.5161,
758
+ "rewards/accuracies": 0.7999999523162842,
759
+ "rewards/chosen": 1.5078362226486206,
760
+ "rewards/diff": 0.07032543420791626,
761
+ "rewards/diff_abs": 0.7481400370597839,
762
+ "rewards/rejected": 0.6864690184593201,
763
+ "rewards/student_margin": 0.821367084980011,
764
+ "rewards/teacher_margin": 0.7510417103767395,
765
+ "step": 410
766
+ },
767
+ {
768
+ "epoch": 0.5,
769
+ "grad_norm": 5.65625,
770
+ "learning_rate": 2.911215015378752e-06,
771
+ "logits/chosen": -3.552057981491089,
772
+ "logits/rejected": -3.6183040142059326,
773
+ "logps/chosen": -224.40554809570312,
774
+ "logps/rejected": -186.09158325195312,
775
+ "loss": 0.5053,
776
+ "rewards/accuracies": 0.7333332896232605,
777
+ "rewards/chosen": 1.171048641204834,
778
+ "rewards/diff": 0.06066560745239258,
779
+ "rewards/diff_abs": 1.058672547340393,
780
+ "rewards/rejected": 0.3551747798919678,
781
+ "rewards/student_margin": 0.8158739805221558,
782
+ "rewards/teacher_margin": 0.7552083730697632,
783
+ "step": 420
784
+ },
785
+ {
786
+ "epoch": 0.51,
787
+ "grad_norm": 6.1875,
788
+ "learning_rate": 2.8077294831853547e-06,
789
+ "logits/chosen": -3.4315121173858643,
790
+ "logits/rejected": -3.4911365509033203,
791
+ "logps/chosen": -285.84918212890625,
792
+ "logps/rejected": -214.66140747070312,
793
+ "loss": 0.5183,
794
+ "rewards/accuracies": 0.7333332896232605,
795
+ "rewards/chosen": 1.376267671585083,
796
+ "rewards/diff": -0.3253711462020874,
797
+ "rewards/diff_abs": 0.9324856996536255,
798
+ "rewards/rejected": 0.7354929447174072,
799
+ "rewards/student_margin": 0.6407747268676758,
800
+ "rewards/teacher_margin": 0.9661458134651184,
801
+ "step": 430
802
+ },
803
+ {
804
+ "epoch": 0.53,
805
+ "grad_norm": 6.40625,
806
+ "learning_rate": 2.703705525459806e-06,
807
+ "logits/chosen": -3.5061888694763184,
808
+ "logits/rejected": -3.5336086750030518,
809
+ "logps/chosen": -219.6090087890625,
810
+ "logps/rejected": -203.97415161132812,
811
+ "loss": 0.5235,
812
+ "rewards/accuracies": 0.76666659116745,
813
+ "rewards/chosen": 1.4297001361846924,
814
+ "rewards/diff": 0.08995727449655533,
815
+ "rewards/diff_abs": 0.5762092471122742,
816
+ "rewards/rejected": 0.5720344185829163,
817
+ "rewards/student_margin": 0.8576656579971313,
818
+ "rewards/teacher_margin": 0.767708420753479,
819
+ "step": 440
820
+ },
821
+ {
822
+ "epoch": 0.54,
823
+ "grad_norm": 6.78125,
824
+ "learning_rate": 2.599325149964946e-06,
825
+ "logits/chosen": -3.4120395183563232,
826
+ "logits/rejected": -3.5835208892822266,
827
+ "logps/chosen": -336.3391418457031,
828
+ "logps/rejected": -304.1842956542969,
829
+ "loss": 0.5215,
830
+ "rewards/accuracies": 0.7999999523162842,
831
+ "rewards/chosen": 1.8122284412384033,
832
+ "rewards/diff": -0.12476543337106705,
833
+ "rewards/diff_abs": 0.7393094897270203,
834
+ "rewards/rejected": 1.2078273296356201,
835
+ "rewards/student_margin": 0.604401171207428,
836
+ "rewards/teacher_margin": 0.7291667461395264,
837
+ "step": 450
838
+ },
839
+ {
840
+ "epoch": 0.55,
841
+ "grad_norm": 6.53125,
842
+ "learning_rate": 2.4947709880776607e-06,
843
+ "logits/chosen": -3.4514999389648438,
844
+ "logits/rejected": -3.581846237182617,
845
+ "logps/chosen": -248.5549774169922,
846
+ "logps/rejected": -214.66116333007812,
847
+ "loss": 0.5098,
848
+ "rewards/accuracies": 0.7000000476837158,
849
+ "rewards/chosen": 1.2313129901885986,
850
+ "rewards/diff": 0.08004424721002579,
851
+ "rewards/diff_abs": 1.226075530052185,
852
+ "rewards/rejected": 0.3127269446849823,
853
+ "rewards/student_margin": 0.9185859560966492,
854
+ "rewards/teacher_margin": 0.8385416865348816,
855
+ "step": 460
856
+ },
857
+ {
858
+ "epoch": 0.56,
859
+ "grad_norm": 6.125,
860
+ "learning_rate": 2.3902259752439462e-06,
861
+ "logits/chosen": -3.492166042327881,
862
+ "logits/rejected": -3.5663814544677734,
863
+ "logps/chosen": -278.4722595214844,
864
+ "logps/rejected": -242.76022338867188,
865
+ "loss": 0.5051,
866
+ "rewards/accuracies": 0.7999999523162842,
867
+ "rewards/chosen": 1.3407200574874878,
868
+ "rewards/diff": -0.000378596771042794,
869
+ "rewards/diff_abs": 1.141404390335083,
870
+ "rewards/rejected": 0.5020361542701721,
871
+ "rewards/student_margin": 0.8386839628219604,
872
+ "rewards/teacher_margin": 0.839062511920929,
873
+ "step": 470
874
+ },
875
+ {
876
+ "epoch": 0.57,
877
+ "grad_norm": 6.25,
878
+ "learning_rate": 2.2858730309019594e-06,
879
+ "logits/chosen": -3.388932704925537,
880
+ "logits/rejected": -3.441415309906006,
881
+ "logps/chosen": -331.4084777832031,
882
+ "logps/rejected": -241.6312713623047,
883
+ "loss": 0.5096,
884
+ "rewards/accuracies": 0.800000011920929,
885
+ "rewards/chosen": 1.8032405376434326,
886
+ "rewards/diff": 0.10180602222681046,
887
+ "rewards/diff_abs": 1.0163486003875732,
888
+ "rewards/rejected": 0.8545595407485962,
889
+ "rewards/student_margin": 0.9486810564994812,
890
+ "rewards/teacher_margin": 0.846875011920929,
891
+ "step": 480
892
+ },
893
+ {
894
+ "epoch": 0.59,
895
+ "grad_norm": 7.28125,
896
+ "learning_rate": 2.181894738433076e-06,
897
+ "logits/chosen": -3.532305955886841,
898
+ "logits/rejected": -3.5801339149475098,
899
+ "logps/chosen": -246.8173065185547,
900
+ "logps/rejected": -220.6660614013672,
901
+ "loss": 0.5419,
902
+ "rewards/accuracies": 0.7666666507720947,
903
+ "rewards/chosen": 1.4681113958358765,
904
+ "rewards/diff": -0.13037186861038208,
905
+ "rewards/diff_abs": 0.9364659190177917,
906
+ "rewards/rejected": 0.711243748664856,
907
+ "rewards/student_margin": 0.7568677067756653,
908
+ "rewards/teacher_margin": 0.8872395753860474,
909
+ "step": 490
910
+ },
911
+ {
912
+ "epoch": 0.6,
913
+ "grad_norm": 6.3125,
914
+ "learning_rate": 2.078473025700937e-06,
915
+ "logits/chosen": -3.536620616912842,
916
+ "logits/rejected": -3.610663652420044,
917
+ "logps/chosen": -196.84896850585938,
918
+ "logps/rejected": -168.04319763183594,
919
+ "loss": 0.5438,
920
+ "rewards/accuracies": 0.6333333253860474,
921
+ "rewards/chosen": 0.8065615892410278,
922
+ "rewards/diff": -0.20639605820178986,
923
+ "rewards/diff_abs": 1.1502354145050049,
924
+ "rewards/rejected": 0.2853534519672394,
925
+ "rewards/student_margin": 0.5212081670761108,
926
+ "rewards/teacher_margin": 0.7276042103767395,
927
+ "step": 500
928
+ },
929
+ {
930
+ "epoch": 0.61,
931
+ "grad_norm": 6.625,
932
+ "learning_rate": 1.975788846737431e-06,
933
+ "logits/chosen": -3.4818530082702637,
934
+ "logits/rejected": -3.5163490772247314,
935
+ "logps/chosen": -223.56863403320312,
936
+ "logps/rejected": -223.87515258789062,
937
+ "loss": 0.5176,
938
+ "rewards/accuracies": 0.6666666269302368,
939
+ "rewards/chosen": 1.066042423248291,
940
+ "rewards/diff": -0.1881529539823532,
941
+ "rewards/diff_abs": 0.9577949643135071,
942
+ "rewards/rejected": 0.4776328504085541,
943
+ "rewards/student_margin": 0.5884095430374146,
944
+ "rewards/teacher_margin": 0.7765625715255737,
945
+ "step": 510
946
+ },
947
+ {
948
+ "epoch": 0.62,
949
+ "grad_norm": 6.03125,
950
+ "learning_rate": 1.8740218651325714e-06,
951
+ "logits/chosen": -3.465400218963623,
952
+ "logits/rejected": -3.4614810943603516,
953
+ "logps/chosen": -256.56890869140625,
954
+ "logps/rejected": -236.2727813720703,
955
+ "loss": 0.5154,
956
+ "rewards/accuracies": 0.7999999523162842,
957
+ "rewards/chosen": 1.6230605840682983,
958
+ "rewards/diff": 0.13750340044498444,
959
+ "rewards/diff_abs": 0.9272276163101196,
960
+ "rewards/rejected": 0.6319113969802856,
961
+ "rewards/student_margin": 0.9911492466926575,
962
+ "rewards/teacher_margin": 0.853645920753479,
963
+ "step": 520
964
+ },
965
+ {
966
+ "epoch": 0.63,
967
+ "grad_norm": 6.9375,
968
+ "learning_rate": 1.7733501396822178e-06,
969
+ "logits/chosen": -3.588365077972412,
970
+ "logits/rejected": -3.5591952800750732,
971
+ "logps/chosen": -199.75267028808594,
972
+ "logps/rejected": -181.21066284179688,
973
+ "loss": 0.5305,
974
+ "rewards/accuracies": 0.6666666269302368,
975
+ "rewards/chosen": 1.041146993637085,
976
+ "rewards/diff": -0.35296258330345154,
977
+ "rewards/diff_abs": 1.0002224445343018,
978
+ "rewards/rejected": 0.4019221365451813,
979
+ "rewards/student_margin": 0.6392248868942261,
980
+ "rewards/teacher_margin": 0.9921875,
981
+ "step": 530
982
+ },
983
+ {
984
+ "epoch": 0.65,
985
+ "grad_norm": 6.375,
986
+ "learning_rate": 1.6739498128436563e-06,
987
+ "logits/chosen": -3.5126869678497314,
988
+ "logits/rejected": -3.5697379112243652,
989
+ "logps/chosen": -275.712158203125,
990
+ "logps/rejected": -249.92800903320312,
991
+ "loss": 0.5093,
992
+ "rewards/accuracies": 0.7666666507720947,
993
+ "rewards/chosen": 1.5757232904434204,
994
+ "rewards/diff": 0.19753125309944153,
995
+ "rewards/diff_abs": 1.0118718147277832,
996
+ "rewards/rejected": 0.4016294479370117,
997
+ "rewards/student_margin": 1.1740937232971191,
998
+ "rewards/teacher_margin": 0.9765625,
999
+ "step": 540
1000
+ },
1001
+ {
1002
+ "epoch": 0.66,
1003
+ "grad_norm": 6.0,
1004
+ "learning_rate": 1.5759948025441535e-06,
1005
+ "logits/chosen": -3.370077610015869,
1006
+ "logits/rejected": -3.4373347759246826,
1007
+ "logps/chosen": -266.87689208984375,
1008
+ "logps/rejected": -229.03158569335938,
1009
+ "loss": 0.5216,
1010
+ "rewards/accuracies": 0.7666666507720947,
1011
+ "rewards/chosen": 1.2909305095672607,
1012
+ "rewards/diff": -0.09488488733768463,
1013
+ "rewards/diff_abs": 1.032594084739685,
1014
+ "rewards/rejected": 0.4243570864200592,
1015
+ "rewards/student_margin": 0.8665734529495239,
1016
+ "rewards/teacher_margin": 0.9614583849906921,
1017
+ "step": 550
1018
+ },
1019
+ {
1020
+ "epoch": 0.67,
1021
+ "grad_norm": 5.75,
1022
+ "learning_rate": 1.479656497881698e-06,
1023
+ "logits/chosen": -3.55267071723938,
1024
+ "logits/rejected": -3.6114087104797363,
1025
+ "logps/chosen": -230.6641082763672,
1026
+ "logps/rejected": -188.7877655029297,
1027
+ "loss": 0.4984,
1028
+ "rewards/accuracies": 0.6000000238418579,
1029
+ "rewards/chosen": 1.1030203104019165,
1030
+ "rewards/diff": -0.43049484491348267,
1031
+ "rewards/diff_abs": 1.0176421403884888,
1032
+ "rewards/rejected": 0.7288275957107544,
1033
+ "rewards/student_margin": 0.37419265508651733,
1034
+ "rewards/teacher_margin": 0.8046875,
1035
+ "step": 560
1036
+ },
1037
+ {
1038
+ "epoch": 0.68,
1039
+ "grad_norm": 6.28125,
1040
+ "learning_rate": 1.3851034592503648e-06,
1041
+ "logits/chosen": -3.3889052867889404,
1042
+ "logits/rejected": -3.5159294605255127,
1043
+ "logps/chosen": -272.55511474609375,
1044
+ "logps/rejected": -199.54537963867188,
1045
+ "loss": 0.5254,
1046
+ "rewards/accuracies": 0.7999999523162842,
1047
+ "rewards/chosen": 1.3238239288330078,
1048
+ "rewards/diff": 0.10217878967523575,
1049
+ "rewards/diff_abs": 0.8275870084762573,
1050
+ "rewards/rejected": 0.3945617377758026,
1051
+ "rewards/student_margin": 0.9292620420455933,
1052
+ "rewards/teacher_margin": 0.82708340883255,
1053
+ "step": 570
1054
+ },
1055
+ {
1056
+ "epoch": 0.69,
1057
+ "grad_norm": 6.78125,
1058
+ "learning_rate": 1.2925011234149859e-06,
1059
+ "logits/chosen": -3.478515148162842,
1060
+ "logits/rejected": -3.606118679046631,
1061
+ "logps/chosen": -204.07174682617188,
1062
+ "logps/rejected": -156.8729705810547,
1063
+ "loss": 0.5088,
1064
+ "rewards/accuracies": 0.6666666269302368,
1065
+ "rewards/chosen": 1.1958485841751099,
1066
+ "rewards/diff": 0.014614415355026722,
1067
+ "rewards/diff_abs": 1.1147606372833252,
1068
+ "rewards/rejected": 0.2895674705505371,
1069
+ "rewards/student_margin": 0.9062811136245728,
1070
+ "rewards/teacher_margin": 0.8916667699813843,
1071
+ "step": 580
1072
+ },
1073
+ {
1074
+ "epoch": 0.71,
1075
+ "grad_norm": 6.21875,
1076
+ "learning_rate": 1.2020115140511436e-06,
1077
+ "logits/chosen": -3.372546434402466,
1078
+ "logits/rejected": -3.3879222869873047,
1079
+ "logps/chosen": -285.7796936035156,
1080
+ "logps/rejected": -257.11016845703125,
1081
+ "loss": 0.5148,
1082
+ "rewards/accuracies": 0.7333332896232605,
1083
+ "rewards/chosen": 1.2352849245071411,
1084
+ "rewards/diff": -0.10239236056804657,
1085
+ "rewards/diff_abs": 0.7884107828140259,
1086
+ "rewards/rejected": 0.5449690818786621,
1087
+ "rewards/student_margin": 0.6903160214424133,
1088
+ "rewards/teacher_margin": 0.7927082777023315,
1089
+ "step": 590
1090
+ },
1091
+ {
1092
+ "epoch": 0.72,
1093
+ "grad_norm": 6.25,
1094
+ "learning_rate": 1.11379295825695e-06,
1095
+ "logits/chosen": -3.4046216011047363,
1096
+ "logits/rejected": -3.449857711791992,
1097
+ "logps/chosen": -274.1463317871094,
1098
+ "logps/rejected": -247.4075927734375,
1099
+ "loss": 0.5252,
1100
+ "rewards/accuracies": 0.6333333253860474,
1101
+ "rewards/chosen": 1.4221832752227783,
1102
+ "rewards/diff": -0.02768692374229431,
1103
+ "rewards/diff_abs": 0.8126093745231628,
1104
+ "rewards/rejected": 0.7457036972045898,
1105
+ "rewards/student_margin": 0.676479697227478,
1106
+ "rewards/teacher_margin": 0.7041667699813843,
1107
+ "step": 600
1108
+ },
1109
+ {
1110
+ "epoch": 0.73,
1111
+ "grad_norm": 6.625,
1112
+ "learning_rate": 1.0279998095326188e-06,
1113
+ "logits/chosen": -3.5202414989471436,
1114
+ "logits/rejected": -3.6290194988250732,
1115
+ "logps/chosen": -281.3245849609375,
1116
+ "logps/rejected": -231.3522186279297,
1117
+ "loss": 0.5181,
1118
+ "rewards/accuracies": 0.7333332896232605,
1119
+ "rewards/chosen": 1.2789822816848755,
1120
+ "rewards/diff": -0.12169651687145233,
1121
+ "rewards/diff_abs": 0.7308156490325928,
1122
+ "rewards/rejected": 0.6079703569412231,
1123
+ "rewards/student_margin": 0.6710118055343628,
1124
+ "rewards/teacher_margin": 0.7927082777023315,
1125
+ "step": 610
1126
+ },
1127
+ {
1128
+ "epoch": 0.74,
1129
+ "grad_norm": 6.15625,
1130
+ "learning_rate": 9.447821777125376e-07,
1131
+ "logits/chosen": -3.484200954437256,
1132
+ "logits/rejected": -3.4762959480285645,
1133
+ "logps/chosen": -234.85791015625,
1134
+ "logps/rejected": -222.82534790039062,
1135
+ "loss": 0.5155,
1136
+ "rewards/accuracies": 0.699999988079071,
1137
+ "rewards/chosen": 0.9543665647506714,
1138
+ "rewards/diff": -0.3350816071033478,
1139
+ "rewards/diff_abs": 1.0735111236572266,
1140
+ "rewards/rejected": 0.3592398166656494,
1141
+ "rewards/student_margin": 0.595126748085022,
1142
+ "rewards/teacher_margin": 0.9302083849906921,
1143
+ "step": 620
1144
+ },
1145
+ {
1146
+ "epoch": 0.75,
1147
+ "grad_norm": 7.375,
1148
+ "learning_rate": 8.642856663223537e-07,
1149
+ "logits/chosen": -3.6152985095977783,
1150
+ "logits/rejected": -3.6913936138153076,
1151
+ "logps/chosen": -278.0227355957031,
1152
+ "logps/rejected": -192.86460876464844,
1153
+ "loss": 0.5314,
1154
+ "rewards/accuracies": 0.7999999523162842,
1155
+ "rewards/chosen": 1.4040509462356567,
1156
+ "rewards/diff": -0.1374289095401764,
1157
+ "rewards/diff_abs": 0.7850462198257446,
1158
+ "rewards/rejected": 0.5149174928665161,
1159
+ "rewards/student_margin": 0.889133632183075,
1160
+ "rewards/teacher_margin": 1.0265624523162842,
1161
+ "step": 630
1162
+ },
1163
+ {
1164
+ "epoch": 0.77,
1165
+ "grad_norm": 5.71875,
1166
+ "learning_rate": 7.866511178206202e-07,
1167
+ "logits/chosen": -3.5455310344696045,
1168
+ "logits/rejected": -3.4960360527038574,
1169
+ "logps/chosen": -288.90374755859375,
1170
+ "logps/rejected": -259.2998046875,
1171
+ "loss": 0.5068,
1172
+ "rewards/accuracies": 0.7000000476837158,
1173
+ "rewards/chosen": 1.5496443510055542,
1174
+ "rewards/diff": -0.2831823229789734,
1175
+ "rewards/diff_abs": 1.0701242685317993,
1176
+ "rewards/rejected": 0.9109517335891724,
1177
+ "rewards/student_margin": 0.6386927366256714,
1178
+ "rewards/teacher_margin": 0.921875,
1179
+ "step": 640
1180
+ },
1181
+ {
1182
+ "epoch": 0.78,
1183
+ "grad_norm": 6.5625,
1184
+ "learning_rate": 7.120143671707535e-07,
1185
+ "logits/chosen": -3.624680995941162,
1186
+ "logits/rejected": -3.571241855621338,
1187
+ "logps/chosen": -238.3637237548828,
1188
+ "logps/rejected": -190.933349609375,
1189
+ "loss": 0.5136,
1190
+ "rewards/accuracies": 0.6999999284744263,
1191
+ "rewards/chosen": 1.272955060005188,
1192
+ "rewards/diff": -0.05736231803894043,
1193
+ "rewards/diff_abs": 0.828034520149231,
1194
+ "rewards/rejected": 0.6131298542022705,
1195
+ "rewards/student_margin": 0.6598252058029175,
1196
+ "rewards/teacher_margin": 0.7171874642372131,
1197
+ "step": 650
1198
+ },
1199
+ {
1200
+ "epoch": 0.79,
1201
+ "grad_norm": 5.9375,
1202
+ "learning_rate": 6.405060041744557e-07,
1203
+ "logits/chosen": -3.3889694213867188,
1204
+ "logits/rejected": -3.4272830486297607,
1205
+ "logps/chosen": -314.39337158203125,
1206
+ "logps/rejected": -279.32037353515625,
1207
+ "loss": 0.5242,
1208
+ "rewards/accuracies": 0.6333333253860474,
1209
+ "rewards/chosen": 1.6729543209075928,
1210
+ "rewards/diff": -0.12356214225292206,
1211
+ "rewards/diff_abs": 1.1017476320266724,
1212
+ "rewards/rejected": 0.8991208076477051,
1213
+ "rewards/student_margin": 0.7738337516784668,
1214
+ "rewards/teacher_margin": 0.8973957896232605,
1215
+ "step": 660
1216
+ },
1217
+ {
1218
+ "epoch": 0.8,
1219
+ "grad_norm": 7.0,
1220
+ "learning_rate": 5.72251144982447e-07,
1221
+ "logits/chosen": -3.5143237113952637,
1222
+ "logits/rejected": -3.4414215087890625,
1223
+ "logps/chosen": -255.2972869873047,
1224
+ "logps/rejected": -279.54595947265625,
1225
+ "loss": 0.4898,
1226
+ "rewards/accuracies": 0.699999988079071,
1227
+ "rewards/chosen": 1.4696900844573975,
1228
+ "rewards/diff": 0.25774964690208435,
1229
+ "rewards/diff_abs": 1.2428072690963745,
1230
+ "rewards/rejected": 0.48954445123672485,
1231
+ "rewards/student_margin": 0.9801454544067383,
1232
+ "rewards/teacher_margin": 0.7223958969116211,
1233
+ "step": 670
1234
+ },
1235
+ {
1236
+ "epoch": 0.81,
1237
+ "grad_norm": 6.4375,
1238
+ "learning_rate": 5.07369213182295e-07,
1239
+ "logits/chosen": -3.437652111053467,
1240
+ "logits/rejected": -3.513336181640625,
1241
+ "logps/chosen": -256.0563659667969,
1242
+ "logps/rejected": -192.80081176757812,
1243
+ "loss": 0.5181,
1244
+ "rewards/accuracies": 0.7333332896232605,
1245
+ "rewards/chosen": 0.9916725158691406,
1246
+ "rewards/diff": -0.04445856809616089,
1247
+ "rewards/diff_abs": 1.0562130212783813,
1248
+ "rewards/rejected": 0.035089436918497086,
1249
+ "rewards/student_margin": 0.9565832018852234,
1250
+ "rewards/teacher_margin": 1.0010416507720947,
1251
+ "step": 680
1252
+ },
1253
+ {
1254
+ "epoch": 0.83,
1255
+ "grad_norm": 5.75,
1256
+ "learning_rate": 4.4597373084635717e-07,
1257
+ "logits/chosen": -3.4108052253723145,
1258
+ "logits/rejected": -3.403064727783203,
1259
+ "logps/chosen": -295.178955078125,
1260
+ "logps/rejected": -241.88851928710938,
1261
+ "loss": 0.5054,
1262
+ "rewards/accuracies": 0.5666666626930237,
1263
+ "rewards/chosen": 1.1539413928985596,
1264
+ "rewards/diff": -0.40385159850120544,
1265
+ "rewards/diff_abs": 1.0642129182815552,
1266
+ "rewards/rejected": 0.7130011320114136,
1267
+ "rewards/student_margin": 0.44094014167785645,
1268
+ "rewards/teacher_margin": 0.8447917103767395,
1269
+ "step": 690
1270
+ },
1271
+ {
1272
+ "epoch": 0.84,
1273
+ "grad_norm": 6.4375,
1274
+ "learning_rate": 3.88172119905435e-07,
1275
+ "logits/chosen": -3.562473773956299,
1276
+ "logits/rejected": -3.469137668609619,
1277
+ "logps/chosen": -264.6433410644531,
1278
+ "logps/rejected": -231.51602172851562,
1279
+ "loss": 0.5061,
1280
+ "rewards/accuracies": 0.7666667103767395,
1281
+ "rewards/chosen": 1.1783157587051392,
1282
+ "rewards/diff": -0.00968353170901537,
1283
+ "rewards/diff_abs": 0.7437331080436707,
1284
+ "rewards/rejected": 0.3218533396720886,
1285
+ "rewards/student_margin": 0.856462299823761,
1286
+ "rewards/teacher_margin": 0.86614590883255,
1287
+ "step": 700
1288
+ },
1289
+ {
1290
+ "epoch": 0.85,
1291
+ "grad_norm": 5.25,
1292
+ "learning_rate": 3.3406551419567584e-07,
1293
+ "logits/chosen": -3.484909772872925,
1294
+ "logits/rejected": -3.444756269454956,
1295
+ "logps/chosen": -285.2689208984375,
1296
+ "logps/rejected": -289.5472106933594,
1297
+ "loss": 0.4931,
1298
+ "rewards/accuracies": 0.6666666269302368,
1299
+ "rewards/chosen": 1.4713407754898071,
1300
+ "rewards/diff": 0.38614755868911743,
1301
+ "rewards/diff_abs": 1.1154059171676636,
1302
+ "rewards/rejected": 0.4737350344657898,
1303
+ "rewards/student_margin": 0.9976059198379517,
1304
+ "rewards/teacher_margin": 0.6114583611488342,
1305
+ "step": 710
1306
+ },
1307
+ {
1308
+ "epoch": 0.86,
1309
+ "grad_norm": 6.15625,
1310
+ "learning_rate": 2.837485825075728e-07,
1311
+ "logits/chosen": -3.577286958694458,
1312
+ "logits/rejected": -3.652881145477295,
1313
+ "logps/chosen": -301.7745361328125,
1314
+ "logps/rejected": -229.53173828125,
1315
+ "loss": 0.5191,
1316
+ "rewards/accuracies": 0.6666666865348816,
1317
+ "rewards/chosen": 1.0710813999176025,
1318
+ "rewards/diff": -0.43389981985092163,
1319
+ "rewards/diff_abs": 1.1230199337005615,
1320
+ "rewards/rejected": 0.5716478228569031,
1321
+ "rewards/student_margin": 0.4994335174560547,
1322
+ "rewards/teacher_margin": 0.9333332777023315,
1323
+ "step": 720
1324
+ },
1325
+ {
1326
+ "epoch": 0.87,
1327
+ "grad_norm": 5.90625,
1328
+ "learning_rate": 2.37309362946673e-07,
1329
+ "logits/chosen": -3.4588115215301514,
1330
+ "logits/rejected": -3.5218307971954346,
1331
+ "logps/chosen": -200.84402465820312,
1332
+ "logps/rejected": -166.37826538085938,
1333
+ "loss": 0.513,
1334
+ "rewards/accuracies": 0.8999999761581421,
1335
+ "rewards/chosen": 0.8319117426872253,
1336
+ "rewards/diff": -0.07107441127300262,
1337
+ "rewards/diff_abs": 0.7676871418952942,
1338
+ "rewards/rejected": 0.08527780324220657,
1339
+ "rewards/student_margin": 0.7466338872909546,
1340
+ "rewards/teacher_margin": 0.8177083134651184,
1341
+ "step": 730
1342
+ },
1343
+ {
1344
+ "epoch": 0.89,
1345
+ "grad_norm": 5.375,
1346
+ "learning_rate": 1.948291088958032e-07,
1347
+ "logits/chosen": -3.379662275314331,
1348
+ "logits/rejected": -3.4146881103515625,
1349
+ "logps/chosen": -259.4773864746094,
1350
+ "logps/rejected": -210.60165405273438,
1351
+ "loss": 0.5071,
1352
+ "rewards/accuracies": 0.6000000238418579,
1353
+ "rewards/chosen": 0.9233649373054504,
1354
+ "rewards/diff": -0.2959494888782501,
1355
+ "rewards/diff_abs": 1.0808264017105103,
1356
+ "rewards/rejected": 0.5380643010139465,
1357
+ "rewards/student_margin": 0.38530051708221436,
1358
+ "rewards/teacher_margin": 0.6812499761581421,
1359
+ "step": 740
1360
+ },
1361
+ {
1362
+ "epoch": 0.9,
1363
+ "grad_norm": 7.15625,
1364
+ "learning_rate": 1.5638214684833923e-07,
1365
+ "logits/chosen": -3.3812708854675293,
1366
+ "logits/rejected": -3.489490032196045,
1367
+ "logps/chosen": -282.34906005859375,
1368
+ "logps/rejected": -206.3470458984375,
1369
+ "loss": 0.5175,
1370
+ "rewards/accuracies": 0.6666666865348816,
1371
+ "rewards/chosen": 1.5291283130645752,
1372
+ "rewards/diff": -0.04967302083969116,
1373
+ "rewards/diff_abs": 1.0276174545288086,
1374
+ "rewards/rejected": 0.6121346354484558,
1375
+ "rewards/student_margin": 0.9169937372207642,
1376
+ "rewards/teacher_margin": 0.9666666984558105,
1377
+ "step": 750
1378
+ },
1379
+ {
1380
+ "epoch": 0.91,
1381
+ "grad_norm": 7.0625,
1382
+ "learning_rate": 1.220357463612501e-07,
1383
+ "logits/chosen": -3.5278987884521484,
1384
+ "logits/rejected": -3.4870636463165283,
1385
+ "logps/chosen": -262.7372131347656,
1386
+ "logps/rejected": -204.89157104492188,
1387
+ "loss": 0.5368,
1388
+ "rewards/accuracies": 0.800000011920929,
1389
+ "rewards/chosen": 1.4189398288726807,
1390
+ "rewards/diff": 0.041193410754203796,
1391
+ "rewards/diff_abs": 0.6765682101249695,
1392
+ "rewards/rejected": 0.6954547166824341,
1393
+ "rewards/student_margin": 0.7234851121902466,
1394
+ "rewards/teacher_margin": 0.6822917461395264,
1395
+ "step": 760
1396
+ },
1397
+ {
1398
+ "epoch": 0.92,
1399
+ "grad_norm": 6.34375,
1400
+ "learning_rate": 9.185000235546443e-08,
1401
+ "logits/chosen": -3.531663417816162,
1402
+ "logits/rejected": -3.5207691192626953,
1403
+ "logps/chosen": -221.78579711914062,
1404
+ "logps/rejected": -199.05947875976562,
1405
+ "loss": 0.5111,
1406
+ "rewards/accuracies": 0.6999999284744263,
1407
+ "rewards/chosen": 1.0741617679595947,
1408
+ "rewards/diff": -0.33916252851486206,
1409
+ "rewards/diff_abs": 0.7832191586494446,
1410
+ "rewards/rejected": 0.7206159830093384,
1411
+ "rewards/student_margin": 0.35354581475257874,
1412
+ "rewards/teacher_margin": 0.6927083730697632,
1413
+ "step": 770
1414
+ },
1415
+ {
1416
+ "epoch": 0.93,
1417
+ "grad_norm": 6.78125,
1418
+ "learning_rate": 6.587772996949876e-08,
1419
+ "logits/chosen": -3.4602973461151123,
1420
+ "logits/rejected": -3.5840487480163574,
1421
+ "logps/chosen": -273.8219299316406,
1422
+ "logps/rejected": -187.69760131835938,
1423
+ "loss": 0.5151,
1424
+ "rewards/accuracies": 0.7666666507720947,
1425
+ "rewards/chosen": 1.2822003364562988,
1426
+ "rewards/diff": 0.012847900390625,
1427
+ "rewards/diff_abs": 0.7760865688323975,
1428
+ "rewards/rejected": 0.34331077337265015,
1429
+ "rewards/student_margin": 0.9388895034790039,
1430
+ "rewards/teacher_margin": 0.9260417819023132,
1431
+ "step": 780
1432
+ },
1433
+ {
1434
+ "epoch": 0.95,
1435
+ "grad_norm": 6.5625,
1436
+ "learning_rate": 4.416437215030628e-08,
1437
+ "logits/chosen": -3.357858657836914,
1438
+ "logits/rejected": -3.428370237350464,
1439
+ "logps/chosen": -231.9785614013672,
1440
+ "logps/rejected": -208.6261749267578,
1441
+ "loss": 0.5225,
1442
+ "rewards/accuracies": 0.7666666507720947,
1443
+ "rewards/chosen": 1.1795800924301147,
1444
+ "rewards/diff": -0.32492926716804504,
1445
+ "rewards/diff_abs": 1.1657856702804565,
1446
+ "rewards/rejected": 0.5086759328842163,
1447
+ "rewards/student_margin": 0.6709040403366089,
1448
+ "rewards/teacher_margin": 0.9958333969116211,
1449
+ "step": 790
1450
+ },
1451
+ {
1452
+ "epoch": 0.96,
1453
+ "grad_norm": 6.84375,
1454
+ "learning_rate": 2.6747920143047056e-08,
1455
+ "logits/chosen": -3.574307680130005,
1456
+ "logits/rejected": -3.662809371948242,
1457
+ "logps/chosen": -242.3976593017578,
1458
+ "logps/rejected": -184.373291015625,
1459
+ "loss": 0.4997,
1460
+ "rewards/accuracies": 0.8333333134651184,
1461
+ "rewards/chosen": 1.215039610862732,
1462
+ "rewards/diff": 0.025632739067077637,
1463
+ "rewards/diff_abs": 0.862860381603241,
1464
+ "rewards/rejected": 0.06649022549390793,
1465
+ "rewards/student_margin": 1.1485494375228882,
1466
+ "rewards/teacher_margin": 1.1229166984558105,
1467
+ "step": 800
1468
+ },
1469
+ {
1470
+ "epoch": 0.97,
1471
+ "grad_norm": 7.0,
1472
+ "learning_rate": 1.3658847018884758e-08,
1473
+ "logits/chosen": -3.3837954998016357,
1474
+ "logits/rejected": -3.477294445037842,
1475
+ "logps/chosen": -303.0797119140625,
1476
+ "logps/rejected": -258.445068359375,
1477
+ "loss": 0.5211,
1478
+ "rewards/accuracies": 0.6666666269302368,
1479
+ "rewards/chosen": 1.3037405014038086,
1480
+ "rewards/diff": -0.3252946734428406,
1481
+ "rewards/diff_abs": 1.0890836715698242,
1482
+ "rewards/rejected": 0.933201789855957,
1483
+ "rewards/student_margin": 0.3705386519432068,
1484
+ "rewards/teacher_margin": 0.6958333849906921,
1485
+ "step": 810
1486
+ },
1487
+ {
1488
+ "epoch": 0.98,
1489
+ "grad_norm": 5.96875,
1490
+ "learning_rate": 4.920054357119841e-09,
1491
+ "logits/chosen": -3.4326694011688232,
1492
+ "logits/rejected": -3.4905331134796143,
1493
+ "logps/chosen": -251.43948364257812,
1494
+ "logps/rejected": -198.3247833251953,
1495
+ "loss": 0.512,
1496
+ "rewards/accuracies": 0.8666666746139526,
1497
+ "rewards/chosen": 1.450547695159912,
1498
+ "rewards/diff": -0.012740576639771461,
1499
+ "rewards/diff_abs": 0.7532329559326172,
1500
+ "rewards/rejected": 0.5716216564178467,
1501
+ "rewards/student_margin": 0.878926157951355,
1502
+ "rewards/teacher_margin": 0.8916667699813843,
1503
+ "step": 820
1504
+ },
1505
+ {
1506
+ "epoch": 0.99,
1507
+ "grad_norm": 5.96875,
1508
+ "learning_rate": 5.468321749468875e-10,
1509
+ "logits/chosen": -3.446951389312744,
1510
+ "logits/rejected": -3.5641331672668457,
1511
+ "logps/chosen": -233.23678588867188,
1512
+ "logps/rejected": -200.827880859375,
1513
+ "loss": 0.5075,
1514
+ "rewards/accuracies": 0.6666666269302368,
1515
+ "rewards/chosen": 0.8117042779922485,
1516
+ "rewards/diff": -0.23592355847358704,
1517
+ "rewards/diff_abs": 0.7656908631324768,
1518
+ "rewards/rejected": 0.2184610813856125,
1519
+ "rewards/student_margin": 0.5932431817054749,
1520
+ "rewards/teacher_margin": 0.8291667103767395,
1521
+ "step": 830
1522
+ },
1523
+ {
1524
+ "epoch": 1.0,
1525
+ "step": 835,
1526
+ "total_flos": 0.0,
1527
+ "train_loss": 0.5412804069633256,
1528
+ "train_runtime": 5959.7316,
1529
+ "train_samples_per_second": 26.891,
1530
+ "train_steps_per_second": 0.14
1531
+ }
1532
+ ],
1533
+ "logging_steps": 10,
1534
+ "max_steps": 835,
1535
+ "num_input_tokens_seen": 0,
1536
+ "num_train_epochs": 1,
1537
+ "save_steps": 100000000000000000000000000000000,
1538
+ "total_flos": 0.0,
1539
+ "train_batch_size": 3,
1540
+ "trial_name": null,
1541
+ "trial_params": null
1542
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc0ec2eae238819ce43250e3cdb080c742b037f60c89b365116208731a6fc8a0
3
+ size 5240