Hugging Face's logo

Ritvik19
/
zephyr-tinyllama-sft-qlora

PEFT
Safetensors
llama
alignment-handbook
trl
sft
Generated from Trainer
Model card Files Community
Ritvik19 commited on
Commit
002c335
•
1 Parent(s): 251227d

Upload 13 files

Browse files
Files changed (13) hide show
  1. README.md +54 -3
  2. adapter_config.json +31 -0
  3. adapter_model.safetensors +3 -0
  4. all_results.json +14 -0
  5. config.json +28 -0
  6. eval_results.json +8 -0
  7. special_tokens_map.json +24 -0
  8. tokenizer.json +0 -0
  9. tokenizer.model +3 -0
  10. tokenizer_config.json +42 -0
  11. train_results.json +9 -0
  12. trainer_state.json +843 -0
  13. training_args.bin +3 -0
README.md CHANGED
@@ -1,3 +1,54 @@
1
- ---
2
- license: apache-2.0
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - alignment-handbook
6
+ - trl
7
+ - sft
8
+ - generated_from_trainer
9
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
10
+ datasets:
11
+ - HuggingFaceH4/ultrachat_200k
12
+ model-index:
13
+ - name: zephyr-tinyllama-sft-qlora
14
+ results: []
15
+ ---
16
+
17
+ **Note**: This model card has been generated automatically according to the information the Trainer had access to.
18
+ Visit the [model card](https://ritvik19.github.io/zephyr-mini/) to see the full description.
19
+
20
+ # zephyr-tinyllama-sft-qlora
21
+
22
+ This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T](https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T) on the HuggingFaceH4/ultrachat_200k dataset.
23
+ It achieves the following results on the evaluation set:
24
+ - Loss: 1.1943
25
+
26
+ ### Training hyperparameters
27
+
28
+ The following hyperparameters were used during training:
29
+ - learning_rate: 0.0002
30
+ - train_batch_size: 2
31
+ - eval_batch_size: 4
32
+ - seed: 42
33
+ - distributed_type: multi-GPU
34
+ - gradient_accumulation_steps: 128
35
+ - total_train_batch_size: 256
36
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
37
+ - lr_scheduler_type: cosine
38
+ - lr_scheduler_warmup_ratio: 0.1
39
+ - num_epochs: 1
40
+
41
+ ### Training results
42
+
43
+ | Training Loss | Epoch | Step | Validation Loss |
44
+ |:-------------:|:------:|:----:|:---------------:|
45
+ | 1.1908 | 0.9991 | 570 | 1.1943 |
46
+
47
+
48
+ ### Framework versions
49
+
50
+ - PEFT 0.7.1
51
+ - Transformers 4.40.1
52
+ - Pytorch 2.1.2+cu121
53
+ - Datasets 2.19.0
54
+ - Tokenizers 0.19.1
adapter_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 16,
13
+ "lora_dropout": 0.05,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 16,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "q_proj",
23
+ "v_proj",
24
+ "down_proj",
25
+ "k_proj",
26
+ "up_proj",
27
+ "o_proj",
28
+ "gate_proj"
29
+ ],
30
+ "task_type": "CAUSAL_LM"
31
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fcff03da247869bc2c5fcfee099df660bf55ba205f7ebf9890d8110e4395186
3
+ size 25272360
all_results.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.999068850303993,
3
+ "eval_loss": 1.1943339109420776,
4
+ "eval_runtime": 1454.6537,
5
+ "eval_samples": 23109,
6
+ "eval_samples_per_second": 11.113,
7
+ "eval_steps_per_second": 2.779,
8
+ "total_flos": 1.8775686401910374e+18,
9
+ "train_loss": 0.1465015503398159,
10
+ "train_runtime": 6527.562,
11
+ "train_samples": 207864,
12
+ "train_samples_per_second": 22.375,
13
+ "train_steps_per_second": 0.087
14
+ }
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 5632,
14
+ "max_position_embeddings": 2048,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 32,
17
+ "num_hidden_layers": 22,
18
+ "num_key_value_heads": 4,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_scaling": null,
22
+ "rope_theta": 10000.0,
23
+ "tie_word_embeddings": false,
24
+ "torch_dtype": "bfloat16",
25
+ "transformers_version": "4.40.1",
26
+ "use_cache": true,
27
+ "vocab_size": 32000
28
+ }
eval_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.999068850303993,
3
+ "eval_loss": 1.1943339109420776,
4
+ "eval_runtime": 1454.6537,
5
+ "eval_samples": 23109,
6
+ "eval_samples_per_second": 11.113,
7
+ "eval_steps_per_second": 2.779
8
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "bos_token": "<s>",
31
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 2048,
36
+ "pad_token": "</s>",
37
+ "padding_side": "right",
38
+ "sp_model_kwargs": {},
39
+ "tokenizer_class": "LlamaTokenizer",
40
+ "unk_token": "<unk>",
41
+ "use_default_system_prompt": false
42
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.999068850303993,
3
+ "total_flos": 1.8775686401910374e+18,
4
+ "train_loss": 0.1465015503398159,
5
+ "train_runtime": 6527.562,
6
+ "train_samples": 207864,
7
+ "train_samples_per_second": 22.375,
8
+ "train_steps_per_second": 0.087
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,843 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.999068850303993,
5
+ "eval_steps": 500,
6
+ "global_step": 570,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0017527523689543736,
13
+ "grad_norm": 0.435546875,
14
+ "learning_rate": 3.5087719298245615e-06,
15
+ "loss": 1.6619,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.008763761844771869,
20
+ "grad_norm": 0.455078125,
21
+ "learning_rate": 1.7543859649122806e-05,
22
+ "loss": 1.6809,
23
+ "step": 5
24
+ },
25
+ {
26
+ "epoch": 0.017527523689543738,
27
+ "grad_norm": 0.455078125,
28
+ "learning_rate": 3.508771929824561e-05,
29
+ "loss": 1.6799,
30
+ "step": 10
31
+ },
32
+ {
33
+ "epoch": 0.026291285534315607,
34
+ "grad_norm": 0.4375,
35
+ "learning_rate": 5.2631578947368424e-05,
36
+ "loss": 1.6325,
37
+ "step": 15
38
+ },
39
+ {
40
+ "epoch": 0.035055047379087476,
41
+ "grad_norm": 0.40234375,
42
+ "learning_rate": 7.017543859649122e-05,
43
+ "loss": 1.5568,
44
+ "step": 20
45
+ },
46
+ {
47
+ "epoch": 0.04381880922385934,
48
+ "grad_norm": 0.2236328125,
49
+ "learning_rate": 8.771929824561403e-05,
50
+ "loss": 1.4713,
51
+ "step": 25
52
+ },
53
+ {
54
+ "epoch": 0.05258257106863121,
55
+ "grad_norm": 0.1572265625,
56
+ "learning_rate": 0.00010526315789473685,
57
+ "loss": 1.397,
58
+ "step": 30
59
+ },
60
+ {
61
+ "epoch": 0.06134633291340308,
62
+ "grad_norm": 0.11474609375,
63
+ "learning_rate": 0.00012280701754385965,
64
+ "loss": 1.3622,
65
+ "step": 35
66
+ },
67
+ {
68
+ "epoch": 0.07011009475817495,
69
+ "grad_norm": 0.0966796875,
70
+ "learning_rate": 0.00014035087719298245,
71
+ "loss": 1.338,
72
+ "step": 40
73
+ },
74
+ {
75
+ "epoch": 0.07887385660294681,
76
+ "grad_norm": 0.072265625,
77
+ "learning_rate": 0.00015789473684210527,
78
+ "loss": 1.3272,
79
+ "step": 45
80
+ },
81
+ {
82
+ "epoch": 0.08763761844771868,
83
+ "grad_norm": 0.04833984375,
84
+ "learning_rate": 0.00017543859649122806,
85
+ "loss": 1.3016,
86
+ "step": 50
87
+ },
88
+ {
89
+ "epoch": 0.09640138029249055,
90
+ "grad_norm": 0.043701171875,
91
+ "learning_rate": 0.00019298245614035088,
92
+ "loss": 1.2995,
93
+ "step": 55
94
+ },
95
+ {
96
+ "epoch": 0.10516514213726243,
97
+ "grad_norm": 0.046142578125,
98
+ "learning_rate": 0.00019998312416333227,
99
+ "loss": 1.2801,
100
+ "step": 60
101
+ },
102
+ {
103
+ "epoch": 0.11392890398203429,
104
+ "grad_norm": 0.036376953125,
105
+ "learning_rate": 0.0001998800146766861,
106
+ "loss": 1.2739,
107
+ "step": 65
108
+ },
109
+ {
110
+ "epoch": 0.12269266582680616,
111
+ "grad_norm": 0.032470703125,
112
+ "learning_rate": 0.00019968326771610797,
113
+ "loss": 1.2633,
114
+ "step": 70
115
+ },
116
+ {
117
+ "epoch": 0.13145642767157803,
118
+ "grad_norm": 0.029541015625,
119
+ "learning_rate": 0.00019939306773179497,
120
+ "loss": 1.2596,
121
+ "step": 75
122
+ },
123
+ {
124
+ "epoch": 0.1402201895163499,
125
+ "grad_norm": 0.030029296875,
126
+ "learning_rate": 0.00019900968678611666,
127
+ "loss": 1.2441,
128
+ "step": 80
129
+ },
130
+ {
131
+ "epoch": 0.14898395136112177,
132
+ "grad_norm": 0.0308837890625,
133
+ "learning_rate": 0.00019853348429855672,
134
+ "loss": 1.2414,
135
+ "step": 85
136
+ },
137
+ {
138
+ "epoch": 0.15774771320589362,
139
+ "grad_norm": 0.02978515625,
140
+ "learning_rate": 0.0001979649067087574,
141
+ "loss": 1.2428,
142
+ "step": 90
143
+ },
144
+ {
145
+ "epoch": 0.1665114750506655,
146
+ "grad_norm": 0.035888671875,
147
+ "learning_rate": 0.00019730448705798239,
148
+ "loss": 1.2387,
149
+ "step": 95
150
+ },
151
+ {
152
+ "epoch": 0.17527523689543736,
153
+ "grad_norm": 0.03125,
154
+ "learning_rate": 0.00019655284448939094,
155
+ "loss": 1.2568,
156
+ "step": 100
157
+ },
158
+ {
159
+ "epoch": 0.18403899874020924,
160
+ "grad_norm": 0.03173828125,
161
+ "learning_rate": 0.00019571068366759143,
162
+ "loss": 1.2212,
163
+ "step": 105
164
+ },
165
+ {
166
+ "epoch": 0.1928027605849811,
167
+ "grad_norm": 0.0306396484375,
168
+ "learning_rate": 0.00019477879411801844,
169
+ "loss": 1.24,
170
+ "step": 110
171
+ },
172
+ {
173
+ "epoch": 0.20156652242975298,
174
+ "grad_norm": 0.0341796875,
175
+ "learning_rate": 0.00019375804948675306,
176
+ "loss": 1.2301,
177
+ "step": 115
178
+ },
179
+ {
180
+ "epoch": 0.21033028427452485,
181
+ "grad_norm": 0.03564453125,
182
+ "learning_rate": 0.00019264940672148018,
183
+ "loss": 1.2178,
184
+ "step": 120
185
+ },
186
+ {
187
+ "epoch": 0.2190940461192967,
188
+ "grad_norm": 0.03955078125,
189
+ "learning_rate": 0.00019145390517435012,
190
+ "loss": 1.218,
191
+ "step": 125
192
+ },
193
+ {
194
+ "epoch": 0.22785780796406857,
195
+ "grad_norm": 0.036865234375,
196
+ "learning_rate": 0.00019017266562758659,
197
+ "loss": 1.2244,
198
+ "step": 130
199
+ },
200
+ {
201
+ "epoch": 0.23662156980884044,
202
+ "grad_norm": 0.040771484375,
203
+ "learning_rate": 0.00018880688924275378,
204
+ "loss": 1.2208,
205
+ "step": 135
206
+ },
207
+ {
208
+ "epoch": 0.24538533165361232,
209
+ "grad_norm": 0.0380859375,
210
+ "learning_rate": 0.00018735785643466784,
211
+ "loss": 1.221,
212
+ "step": 140
213
+ },
214
+ {
215
+ "epoch": 0.25414909349838416,
216
+ "grad_norm": 0.036865234375,
217
+ "learning_rate": 0.00018582692567100867,
218
+ "loss": 1.2084,
219
+ "step": 145
220
+ },
221
+ {
222
+ "epoch": 0.26291285534315606,
223
+ "grad_norm": 0.03759765625,
224
+ "learning_rate": 0.00018421553219875658,
225
+ "loss": 1.2209,
226
+ "step": 150
227
+ },
228
+ {
229
+ "epoch": 0.2716766171879279,
230
+ "grad_norm": 0.038330078125,
231
+ "learning_rate": 0.00018252518669864936,
232
+ "loss": 1.2136,
233
+ "step": 155
234
+ },
235
+ {
236
+ "epoch": 0.2804403790326998,
237
+ "grad_norm": 0.0361328125,
238
+ "learning_rate": 0.0001807574738689193,
239
+ "loss": 1.2214,
240
+ "step": 160
241
+ },
242
+ {
243
+ "epoch": 0.28920414087747165,
244
+ "grad_norm": 0.035888671875,
245
+ "learning_rate": 0.00017891405093963938,
246
+ "loss": 1.2213,
247
+ "step": 165
248
+ },
249
+ {
250
+ "epoch": 0.29796790272224355,
251
+ "grad_norm": 0.034912109375,
252
+ "learning_rate": 0.00017699664611907072,
253
+ "loss": 1.2084,
254
+ "step": 170
255
+ },
256
+ {
257
+ "epoch": 0.3067316645670154,
258
+ "grad_norm": 0.040771484375,
259
+ "learning_rate": 0.0001750070569734681,
260
+ "loss": 1.2028,
261
+ "step": 175
262
+ },
263
+ {
264
+ "epoch": 0.31549542641178724,
265
+ "grad_norm": 0.036865234375,
266
+ "learning_rate": 0.0001729471487418621,
267
+ "loss": 1.2065,
268
+ "step": 180
269
+ },
270
+ {
271
+ "epoch": 0.32425918825655914,
272
+ "grad_norm": 0.035888671875,
273
+ "learning_rate": 0.00017081885258739846,
274
+ "loss": 1.2002,
275
+ "step": 185
276
+ },
277
+ {
278
+ "epoch": 0.333022950101331,
279
+ "grad_norm": 0.037841796875,
280
+ "learning_rate": 0.0001686241637868734,
281
+ "loss": 1.2058,
282
+ "step": 190
283
+ },
284
+ {
285
+ "epoch": 0.3417867119461029,
286
+ "grad_norm": 0.0361328125,
287
+ "learning_rate": 0.00016636513986016213,
288
+ "loss": 1.2084,
289
+ "step": 195
290
+ },
291
+ {
292
+ "epoch": 0.35055047379087473,
293
+ "grad_norm": 0.03759765625,
294
+ "learning_rate": 0.00016404389864129533,
295
+ "loss": 1.199,
296
+ "step": 200
297
+ },
298
+ {
299
+ "epoch": 0.35931423563564663,
300
+ "grad_norm": 0.042724609375,
301
+ "learning_rate": 0.00016166261629298995,
302
+ "loss": 1.2023,
303
+ "step": 205
304
+ },
305
+ {
306
+ "epoch": 0.3680779974804185,
307
+ "grad_norm": 0.044189453125,
308
+ "learning_rate": 0.00015922352526649803,
309
+ "loss": 1.1999,
310
+ "step": 210
311
+ },
312
+ {
313
+ "epoch": 0.3768417593251903,
314
+ "grad_norm": 0.036376953125,
315
+ "learning_rate": 0.00015672891220868432,
316
+ "loss": 1.1882,
317
+ "step": 215
318
+ },
319
+ {
320
+ "epoch": 0.3856055211699622,
321
+ "grad_norm": 0.041748046875,
322
+ "learning_rate": 0.00015418111581829574,
323
+ "loss": 1.2016,
324
+ "step": 220
325
+ },
326
+ {
327
+ "epoch": 0.39436928301473406,
328
+ "grad_norm": 0.041748046875,
329
+ "learning_rate": 0.00015158252465343242,
330
+ "loss": 1.1994,
331
+ "step": 225
332
+ },
333
+ {
334
+ "epoch": 0.40313304485950596,
335
+ "grad_norm": 0.048828125,
336
+ "learning_rate": 0.00014893557489227517,
337
+ "loss": 1.2054,
338
+ "step": 230
339
+ },
340
+ {
341
+ "epoch": 0.4118968067042778,
342
+ "grad_norm": 0.0419921875,
343
+ "learning_rate": 0.00014624274804916958,
344
+ "loss": 1.1868,
345
+ "step": 235
346
+ },
347
+ {
348
+ "epoch": 0.4206605685490497,
349
+ "grad_norm": 0.041259765625,
350
+ "learning_rate": 0.00014350656864820733,
351
+ "loss": 1.2056,
352
+ "step": 240
353
+ },
354
+ {
355
+ "epoch": 0.42942433039382155,
356
+ "grad_norm": 0.039306640625,
357
+ "learning_rate": 0.00014072960185648577,
358
+ "loss": 1.2042,
359
+ "step": 245
360
+ },
361
+ {
362
+ "epoch": 0.4381880922385934,
363
+ "grad_norm": 0.037353515625,
364
+ "learning_rate": 0.00013791445107926478,
365
+ "loss": 1.1992,
366
+ "step": 250
367
+ },
368
+ {
369
+ "epoch": 0.4469518540833653,
370
+ "grad_norm": 0.0380859375,
371
+ "learning_rate": 0.00013506375551927547,
372
+ "loss": 1.2025,
373
+ "step": 255
374
+ },
375
+ {
376
+ "epoch": 0.45571561592813714,
377
+ "grad_norm": 0.036865234375,
378
+ "learning_rate": 0.00013218018770246858,
379
+ "loss": 1.2102,
380
+ "step": 260
381
+ },
382
+ {
383
+ "epoch": 0.46447937777290904,
384
+ "grad_norm": 0.036865234375,
385
+ "learning_rate": 0.0001292664509725226,
386
+ "loss": 1.2105,
387
+ "step": 265
388
+ },
389
+ {
390
+ "epoch": 0.4732431396176809,
391
+ "grad_norm": 0.044189453125,
392
+ "learning_rate": 0.00012632527695645993,
393
+ "loss": 1.1963,
394
+ "step": 270
395
+ },
396
+ {
397
+ "epoch": 0.48200690146245273,
398
+ "grad_norm": 0.0380859375,
399
+ "learning_rate": 0.00012335942300374788,
400
+ "loss": 1.1911,
401
+ "step": 275
402
+ },
403
+ {
404
+ "epoch": 0.49077066330722463,
405
+ "grad_norm": 0.037109375,
406
+ "learning_rate": 0.00012037166960128443,
407
+ "loss": 1.1901,
408
+ "step": 280
409
+ },
410
+ {
411
+ "epoch": 0.4995344251519965,
412
+ "grad_norm": 0.035888671875,
413
+ "learning_rate": 0.00011736481776669306,
414
+ "loss": 1.1952,
415
+ "step": 285
416
+ },
417
+ {
418
+ "epoch": 0.5082981869967683,
419
+ "grad_norm": 0.0361328125,
420
+ "learning_rate": 0.00011434168642236964,
421
+ "loss": 1.1864,
422
+ "step": 290
423
+ },
424
+ {
425
+ "epoch": 0.5170619488415402,
426
+ "grad_norm": 0.0390625,
427
+ "learning_rate": 0.00011130510975274409,
428
+ "loss": 1.1909,
429
+ "step": 295
430
+ },
431
+ {
432
+ "epoch": 0.5258257106863121,
433
+ "grad_norm": 0.039794921875,
434
+ "learning_rate": 0.00010825793454723325,
435
+ "loss": 1.1998,
436
+ "step": 300
437
+ },
438
+ {
439
+ "epoch": 0.534589472531084,
440
+ "grad_norm": 0.043212890625,
441
+ "learning_rate": 0.00010520301753137724,
442
+ "loss": 1.1918,
443
+ "step": 305
444
+ },
445
+ {
446
+ "epoch": 0.5433532343758558,
447
+ "grad_norm": 0.037109375,
448
+ "learning_rate": 0.00010214322268866032,
449
+ "loss": 1.1821,
450
+ "step": 310
451
+ },
452
+ {
453
+ "epoch": 0.5521169962206277,
454
+ "grad_norm": 0.044189453125,
455
+ "learning_rate": 9.908141857552737e-05,
456
+ "loss": 1.1964,
457
+ "step": 315
458
+ },
459
+ {
460
+ "epoch": 0.5608807580653996,
461
+ "grad_norm": 0.037109375,
462
+ "learning_rate": 9.602047563211359e-05,
463
+ "loss": 1.1802,
464
+ "step": 320
465
+ },
466
+ {
467
+ "epoch": 0.5696445199101714,
468
+ "grad_norm": 0.038818359375,
469
+ "learning_rate": 9.296326349120785e-05,
470
+ "loss": 1.1869,
471
+ "step": 325
472
+ },
473
+ {
474
+ "epoch": 0.5784082817549433,
475
+ "grad_norm": 0.039306640625,
476
+ "learning_rate": 8.991264828797319e-05,
477
+ "loss": 1.2014,
478
+ "step": 330
479
+ },
480
+ {
481
+ "epoch": 0.5871720435997152,
482
+ "grad_norm": 0.03515625,
483
+ "learning_rate": 8.687148997294621e-05,
484
+ "loss": 1.1903,
485
+ "step": 335
486
+ },
487
+ {
488
+ "epoch": 0.5959358054444871,
489
+ "grad_norm": 0.03857421875,
490
+ "learning_rate": 8.384263963083453e-05,
491
+ "loss": 1.1862,
492
+ "step": 340
493
+ },
494
+ {
495
+ "epoch": 0.6046995672892589,
496
+ "grad_norm": 0.037353515625,
497
+ "learning_rate": 8.082893680762619e-05,
498
+ "loss": 1.1919,
499
+ "step": 345
500
+ },
501
+ {
502
+ "epoch": 0.6134633291340308,
503
+ "grad_norm": 0.03662109375,
504
+ "learning_rate": 7.783320684851614e-05,
505
+ "loss": 1.1874,
506
+ "step": 350
507
+ },
508
+ {
509
+ "epoch": 0.6222270909788027,
510
+ "grad_norm": 0.0380859375,
511
+ "learning_rate": 7.485825824914659e-05,
512
+ "loss": 1.2009,
513
+ "step": 355
514
+ },
515
+ {
516
+ "epoch": 0.6309908528235745,
517
+ "grad_norm": 0.03955078125,
518
+ "learning_rate": 7.190688002264308e-05,
519
+ "loss": 1.1853,
520
+ "step": 360
521
+ },
522
+ {
523
+ "epoch": 0.6397546146683464,
524
+ "grad_norm": 0.037841796875,
525
+ "learning_rate": 6.898183908491617e-05,
526
+ "loss": 1.1868,
527
+ "step": 365
528
+ },
529
+ {
530
+ "epoch": 0.6485183765131183,
531
+ "grad_norm": 0.039306640625,
532
+ "learning_rate": 6.608587766067852e-05,
533
+ "loss": 1.1845,
534
+ "step": 370
535
+ },
536
+ {
537
+ "epoch": 0.6572821383578902,
538
+ "grad_norm": 0.038818359375,
539
+ "learning_rate": 6.322171071261071e-05,
540
+ "loss": 1.1923,
541
+ "step": 375
542
+ },
543
+ {
544
+ "epoch": 0.666045900202662,
545
+ "grad_norm": 0.0380859375,
546
+ "learning_rate": 6.039202339608432e-05,
547
+ "loss": 1.1929,
548
+ "step": 380
549
+ },
550
+ {
551
+ "epoch": 0.6748096620474339,
552
+ "grad_norm": 0.038330078125,
553
+ "learning_rate": 5.7599468541830356e-05,
554
+ "loss": 1.1872,
555
+ "step": 385
556
+ },
557
+ {
558
+ "epoch": 0.6835734238922058,
559
+ "grad_norm": 0.03759765625,
560
+ "learning_rate": 5.484666416891109e-05,
561
+ "loss": 1.1766,
562
+ "step": 390
563
+ },
564
+ {
565
+ "epoch": 0.6923371857369776,
566
+ "grad_norm": 0.036865234375,
567
+ "learning_rate": 5.2136191030328455e-05,
568
+ "loss": 1.1938,
569
+ "step": 395
570
+ },
571
+ {
572
+ "epoch": 0.7011009475817495,
573
+ "grad_norm": 0.038330078125,
574
+ "learning_rate": 4.9470590193569044e-05,
575
+ "loss": 1.1953,
576
+ "step": 400
577
+ },
578
+ {
579
+ "epoch": 0.7098647094265214,
580
+ "grad_norm": 0.037353515625,
581
+ "learning_rate": 4.685236065835443e-05,
582
+ "loss": 1.1926,
583
+ "step": 405
584
+ },
585
+ {
586
+ "epoch": 0.7186284712712933,
587
+ "grad_norm": 0.03466796875,
588
+ "learning_rate": 4.4283957013829846e-05,
589
+ "loss": 1.1892,
590
+ "step": 410
591
+ },
592
+ {
593
+ "epoch": 0.727392233116065,
594
+ "grad_norm": 0.037841796875,
595
+ "learning_rate": 4.176778713738787e-05,
596
+ "loss": 1.1871,
597
+ "step": 415
598
+ },
599
+ {
600
+ "epoch": 0.736155994960837,
601
+ "grad_norm": 0.035888671875,
602
+ "learning_rate": 3.9306209937284346e-05,
603
+ "loss": 1.1879,
604
+ "step": 420
605
+ },
606
+ {
607
+ "epoch": 0.7449197568056088,
608
+ "grad_norm": 0.034423828125,
609
+ "learning_rate": 3.69015331411628e-05,
610
+ "loss": 1.2001,
611
+ "step": 425
612
+ },
613
+ {
614
+ "epoch": 0.7536835186503806,
615
+ "grad_norm": 0.03662109375,
616
+ "learning_rate": 3.455601113256073e-05,
617
+ "loss": 1.1817,
618
+ "step": 430
619
+ },
620
+ {
621
+ "epoch": 0.7624472804951525,
622
+ "grad_norm": 0.036865234375,
623
+ "learning_rate": 3.227184283742591e-05,
624
+ "loss": 1.19,
625
+ "step": 435
626
+ },
627
+ {
628
+ "epoch": 0.7712110423399244,
629
+ "grad_norm": 0.036376953125,
630
+ "learning_rate": 3.0051169662624225e-05,
631
+ "loss": 1.201,
632
+ "step": 440
633
+ },
634
+ {
635
+ "epoch": 0.7799748041846963,
636
+ "grad_norm": 0.035888671875,
637
+ "learning_rate": 2.789607348837153e-05,
638
+ "loss": 1.1962,
639
+ "step": 445
640
+ },
641
+ {
642
+ "epoch": 0.7887385660294681,
643
+ "grad_norm": 0.035400390625,
644
+ "learning_rate": 2.5808574716471856e-05,
645
+ "loss": 1.1805,
646
+ "step": 450
647
+ },
648
+ {
649
+ "epoch": 0.79750232787424,
650
+ "grad_norm": 0.036865234375,
651
+ "learning_rate": 2.379063037619146e-05,
652
+ "loss": 1.1879,
653
+ "step": 455
654
+ },
655
+ {
656
+ "epoch": 0.8062660897190119,
657
+ "grad_norm": 0.0361328125,
658
+ "learning_rate": 2.184413228954468e-05,
659
+ "loss": 1.1902,
660
+ "step": 460
661
+ },
662
+ {
663
+ "epoch": 0.8150298515637837,
664
+ "grad_norm": 0.035888671875,
665
+ "learning_rate": 1.9970905297711606e-05,
666
+ "loss": 1.1754,
667
+ "step": 465
668
+ },
669
+ {
670
+ "epoch": 0.8237936134085556,
671
+ "grad_norm": 0.033935546875,
672
+ "learning_rate": 1.8172705550250092e-05,
673
+ "loss": 1.1926,
674
+ "step": 470
675
+ },
676
+ {
677
+ "epoch": 0.8325573752533275,
678
+ "grad_norm": 0.034912109375,
679
+ "learning_rate": 1.6451218858706374e-05,
680
+ "loss": 1.1832,
681
+ "step": 475
682
+ },
683
+ {
684
+ "epoch": 0.8413211370980994,
685
+ "grad_norm": 0.033935546875,
686
+ "learning_rate": 1.4808059116167305e-05,
687
+ "loss": 1.1767,
688
+ "step": 480
689
+ },
690
+ {
691
+ "epoch": 0.8500848989428712,
692
+ "grad_norm": 0.03515625,
693
+ "learning_rate": 1.3244766784236307e-05,
694
+ "loss": 1.1978,
695
+ "step": 485
696
+ },
697
+ {
698
+ "epoch": 0.8588486607876431,
699
+ "grad_norm": 0.035888671875,
700
+ "learning_rate": 1.176280744885121e-05,
701
+ "loss": 1.1975,
702
+ "step": 490
703
+ },
704
+ {
705
+ "epoch": 0.867612422632415,
706
+ "grad_norm": 0.03466796875,
707
+ "learning_rate": 1.0363570446297999e-05,
708
+ "loss": 1.1886,
709
+ "step": 495
710
+ },
711
+ {
712
+ "epoch": 0.8763761844771868,
713
+ "grad_norm": 0.03466796875,
714
+ "learning_rate": 9.048367560708604e-06,
715
+ "loss": 1.1954,
716
+ "step": 500
717
+ },
718
+ {
719
+ "epoch": 0.8851399463219587,
720
+ "grad_norm": 0.036376953125,
721
+ "learning_rate": 7.818431794263836e-06,
722
+ "loss": 1.1965,
723
+ "step": 505
724
+ },
725
+ {
726
+ "epoch": 0.8939037081667306,
727
+ "grad_norm": 0.03515625,
728
+ "learning_rate": 6.674916211254289e-06,
729
+ "loss": 1.2116,
730
+ "step": 510
731
+ },
732
+ {
733
+ "epoch": 0.9026674700115024,
734
+ "grad_norm": 0.033935546875,
735
+ "learning_rate": 5.618892857083069e-06,
736
+ "loss": 1.1851,
737
+ "step": 515
738
+ },
739
+ {
740
+ "epoch": 0.9114312318562743,
741
+ "grad_norm": 0.035400390625,
742
+ "learning_rate": 4.65135175322361e-06,
743
+ "loss": 1.1957,
744
+ "step": 520
745
+ },
746
+ {
747
+ "epoch": 0.9201949937010462,
748
+ "grad_norm": 0.0361328125,
749
+ "learning_rate": 3.7731999690749585e-06,
750
+ "loss": 1.1806,
751
+ "step": 525
752
+ },
753
+ {
754
+ "epoch": 0.9289587555458181,
755
+ "grad_norm": 0.034423828125,
756
+ "learning_rate": 2.9852607715846193e-06,
757
+ "loss": 1.1924,
758
+ "step": 530
759
+ },
760
+ {
761
+ "epoch": 0.9377225173905899,
762
+ "grad_norm": 0.03564453125,
763
+ "learning_rate": 2.288272853436013e-06,
764
+ "loss": 1.1911,
765
+ "step": 535
766
+ },
767
+ {
768
+ "epoch": 0.9464862792353618,
769
+ "grad_norm": 0.033935546875,
770
+ "learning_rate": 1.6828896405244988e-06,
771
+ "loss": 1.1815,
772
+ "step": 540
773
+ },
774
+ {
775
+ "epoch": 0.9552500410801337,
776
+ "grad_norm": 0.034423828125,
777
+ "learning_rate": 1.1696786793707781e-06,
778
+ "loss": 1.2068,
779
+ "step": 545
780
+ },
781
+ {
782
+ "epoch": 0.9640138029249055,
783
+ "grad_norm": 0.03515625,
784
+ "learning_rate": 7.491211050462798e-07,
785
+ "loss": 1.1929,
786
+ "step": 550
787
+ },
788
+ {
789
+ "epoch": 0.9727775647696774,
790
+ "grad_norm": 0.035400390625,
791
+ "learning_rate": 4.216111901092501e-07,
792
+ "loss": 1.1962,
793
+ "step": 555
794
+ },
795
+ {
796
+ "epoch": 0.9815413266144493,
797
+ "grad_norm": 0.036865234375,
798
+ "learning_rate": 1.8745597497433765e-07,
799
+ "loss": 1.1793,
800
+ "step": 560
801
+ },
802
+ {
803
+ "epoch": 0.9903050884592212,
804
+ "grad_norm": 0.03515625,
805
+ "learning_rate": 4.687498006236135e-08,
806
+ "loss": 1.2006,
807
+ "step": 565
808
+ },
809
+ {
810
+ "epoch": 0.999068850303993,
811
+ "grad_norm": 0.034912109375,
812
+ "learning_rate": 0.0,
813
+ "loss": 1.1908,
814
+ "step": 570
815
+ },
816
+ {
817
+ "epoch": 0.999068850303993,
818
+ "eval_loss": 1.1943339109420776,
819
+ "eval_runtime": 1451.0124,
820
+ "eval_samples_per_second": 11.141,
821
+ "eval_steps_per_second": 2.786,
822
+ "step": 570
823
+ },
824
+ {
825
+ "epoch": 0.999068850303993,
826
+ "step": 570,
827
+ "total_flos": 1.8775686401910374e+18,
828
+ "train_loss": 0.1465015503398159,
829
+ "train_runtime": 6527.562,
830
+ "train_samples_per_second": 22.375,
831
+ "train_steps_per_second": 0.087
832
+ }
833
+ ],
834
+ "logging_steps": 5,
835
+ "max_steps": 570,
836
+ "num_input_tokens_seen": 0,
837
+ "num_train_epochs": 1,
838
+ "save_steps": 100,
839
+ "total_flos": 1.8775686401910374e+18,
840
+ "train_batch_size": 2,
841
+ "trial_name": null,
842
+ "trial_params": null
843
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c1fafd059d5231fec0896e6333676e22a05f7cf60638f8ec22769f4ad806b20
3
+ size 5176