EvgeniyZh commited on
Commit
a008024
1 Parent(s): 9a78b83

Model save

Browse files
README.md ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: mistralai/Mistral-7B-v0.1
4
+ tags:
5
+ - generated_from_trainer
6
+ model-index:
7
+ - name: zephyr-7b-sft-lora
8
+ results: []
9
+ ---
10
+
11
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
12
+ should probably proofread and complete it, then remove this comment. -->
13
+
14
+ # zephyr-7b-sft-lora
15
+
16
+ This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
17
+ It achieves the following results on the evaluation set:
18
+ - Loss: 1.0667
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 2e-05
38
+ - train_batch_size: 16
39
+ - eval_batch_size: 32
40
+ - seed: 42
41
+ - distributed_type: multi-GPU
42
+ - num_devices: 2
43
+ - gradient_accumulation_steps: 16
44
+ - total_train_batch_size: 512
45
+ - total_eval_batch_size: 64
46
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
+ - lr_scheduler_type: cosine
48
+ - num_epochs: 1
49
+
50
+ ### Training results
51
+
52
+ | Training Loss | Epoch | Step | Validation Loss |
53
+ |:-------------:|:-----:|:----:|:---------------:|
54
+ | 1.063 | 0.67 | 272 | 1.0667 |
55
+
56
+
57
+ ### Framework versions
58
+
59
+ - Transformers 4.35.0
60
+ - Pytorch 2.1.1+cu121
61
+ - Datasets 2.14.6
62
+ - Tokenizers 0.14.1
adapter_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.1,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 64,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "v_proj",
21
+ "o_proj",
22
+ "k_proj"
23
+ ],
24
+ "task_type": "CAUSAL_LM"
25
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38c4cc3db797cde9c3e4214f38f663a87bbca856e02bb4d2a95bb006aca3741f
3
+ size 109086672
all_results.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.67,
3
+ "eval_loss": 1.0666559934616089,
4
+ "eval_runtime": 2404.6287,
5
+ "eval_samples": 23110,
6
+ "eval_samples_per_second": 9.611,
7
+ "eval_steps_per_second": 0.151,
8
+ "train_loss": 1.1599171240540112,
9
+ "train_runtime": 73159.9281,
10
+ "train_samples": 207865,
11
+ "train_samples_per_second": 2.841,
12
+ "train_steps_per_second": 0.006
13
+ }
eval_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.67,
3
+ "eval_loss": 1.0666559934616089,
4
+ "eval_runtime": 2404.6287,
5
+ "eval_samples": 23110,
6
+ "eval_samples_per_second": 9.611,
7
+ "eval_steps_per_second": 0.151
8
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ }
27
+ },
28
+ "additional_special_tokens": [],
29
+ "bos_token": "<s>",
30
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
31
+ "clean_up_tokenization_spaces": false,
32
+ "eos_token": "</s>",
33
+ "legacy": true,
34
+ "model_max_length": 2048,
35
+ "pad_token": "</s>",
36
+ "sp_model_kwargs": {},
37
+ "spaces_between_special_tokens": false,
38
+ "tokenizer_class": "LlamaTokenizer",
39
+ "unk_token": "<unk>",
40
+ "use_default_system_prompt": true
41
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.67,
3
+ "train_loss": 1.1599171240540112,
4
+ "train_runtime": 73159.9281,
5
+ "train_samples": 207865,
6
+ "train_samples_per_second": 2.841,
7
+ "train_steps_per_second": 0.006
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,366 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.6699507389162561,
5
+ "eval_steps": 500,
6
+ "global_step": 272,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 1.9999700625010444e-05,
14
+ "loss": 1.6931,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.01,
19
+ "learning_rate": 1.999251652147735e-05,
20
+ "loss": 1.6625,
21
+ "step": 5
22
+ },
23
+ {
24
+ "epoch": 0.02,
25
+ "learning_rate": 1.997007728639956e-05,
26
+ "loss": 1.5879,
27
+ "step": 10
28
+ },
29
+ {
30
+ "epoch": 0.04,
31
+ "learning_rate": 1.9932715879473385e-05,
32
+ "loss": 1.5023,
33
+ "step": 15
34
+ },
35
+ {
36
+ "epoch": 0.05,
37
+ "learning_rate": 1.9880488219356086e-05,
38
+ "loss": 1.4524,
39
+ "step": 20
40
+ },
41
+ {
42
+ "epoch": 0.06,
43
+ "learning_rate": 1.981347247496222e-05,
44
+ "loss": 1.3857,
45
+ "step": 25
46
+ },
47
+ {
48
+ "epoch": 0.07,
49
+ "learning_rate": 1.973176894846855e-05,
50
+ "loss": 1.348,
51
+ "step": 30
52
+ },
53
+ {
54
+ "epoch": 0.09,
55
+ "learning_rate": 1.963549992519223e-05,
56
+ "loss": 1.3286,
57
+ "step": 35
58
+ },
59
+ {
60
+ "epoch": 0.1,
61
+ "learning_rate": 1.9524809490566878e-05,
62
+ "loss": 1.2921,
63
+ "step": 40
64
+ },
65
+ {
66
+ "epoch": 0.11,
67
+ "learning_rate": 1.939986331449053e-05,
68
+ "loss": 1.2736,
69
+ "step": 45
70
+ },
71
+ {
72
+ "epoch": 0.12,
73
+ "learning_rate": 1.926084840336821e-05,
74
+ "loss": 1.2586,
75
+ "step": 50
76
+ },
77
+ {
78
+ "epoch": 0.14,
79
+ "learning_rate": 1.910797282022027e-05,
80
+ "loss": 1.2334,
81
+ "step": 55
82
+ },
83
+ {
84
+ "epoch": 0.15,
85
+ "learning_rate": 1.894146537327533e-05,
86
+ "loss": 1.2086,
87
+ "step": 60
88
+ },
89
+ {
90
+ "epoch": 0.16,
91
+ "learning_rate": 1.8761575273514005e-05,
92
+ "loss": 1.189,
93
+ "step": 65
94
+ },
95
+ {
96
+ "epoch": 0.17,
97
+ "learning_rate": 1.8568571761675893e-05,
98
+ "loss": 1.1707,
99
+ "step": 70
100
+ },
101
+ {
102
+ "epoch": 0.18,
103
+ "learning_rate": 1.8362743705288127e-05,
104
+ "loss": 1.1654,
105
+ "step": 75
106
+ },
107
+ {
108
+ "epoch": 0.2,
109
+ "learning_rate": 1.814439916631857e-05,
110
+ "loss": 1.1668,
111
+ "step": 80
112
+ },
113
+ {
114
+ "epoch": 0.21,
115
+ "learning_rate": 1.791386494010081e-05,
116
+ "loss": 1.1569,
117
+ "step": 85
118
+ },
119
+ {
120
+ "epoch": 0.22,
121
+ "learning_rate": 1.7671486066220965e-05,
122
+ "loss": 1.1376,
123
+ "step": 90
124
+ },
125
+ {
126
+ "epoch": 0.23,
127
+ "learning_rate": 1.7417625312098453e-05,
128
+ "loss": 1.1222,
129
+ "step": 95
130
+ },
131
+ {
132
+ "epoch": 0.25,
133
+ "learning_rate": 1.7152662630033506e-05,
134
+ "loss": 1.1304,
135
+ "step": 100
136
+ },
137
+ {
138
+ "epoch": 0.26,
139
+ "learning_rate": 1.6876994588534234e-05,
140
+ "loss": 1.1425,
141
+ "step": 105
142
+ },
143
+ {
144
+ "epoch": 0.27,
145
+ "learning_rate": 1.659103377877423e-05,
146
+ "loss": 1.1127,
147
+ "step": 110
148
+ },
149
+ {
150
+ "epoch": 0.28,
151
+ "learning_rate": 1.629520819706912e-05,
152
+ "loss": 1.1069,
153
+ "step": 115
154
+ },
155
+ {
156
+ "epoch": 0.3,
157
+ "learning_rate": 1.598996060429634e-05,
158
+ "loss": 1.1078,
159
+ "step": 120
160
+ },
161
+ {
162
+ "epoch": 0.31,
163
+ "learning_rate": 1.56757478632168e-05,
164
+ "loss": 1.103,
165
+ "step": 125
166
+ },
167
+ {
168
+ "epoch": 0.32,
169
+ "learning_rate": 1.5353040254690396e-05,
170
+ "loss": 1.1107,
171
+ "step": 130
172
+ },
173
+ {
174
+ "epoch": 0.33,
175
+ "learning_rate": 1.5022320773808612e-05,
176
+ "loss": 1.1062,
177
+ "step": 135
178
+ },
179
+ {
180
+ "epoch": 0.34,
181
+ "learning_rate": 1.4684084406997903e-05,
182
+ "loss": 1.0909,
183
+ "step": 140
184
+ },
185
+ {
186
+ "epoch": 0.36,
187
+ "learning_rate": 1.4338837391175582e-05,
188
+ "loss": 1.0942,
189
+ "step": 145
190
+ },
191
+ {
192
+ "epoch": 0.37,
193
+ "learning_rate": 1.3987096456067236e-05,
194
+ "loss": 1.0887,
195
+ "step": 150
196
+ },
197
+ {
198
+ "epoch": 0.38,
199
+ "learning_rate": 1.3629388050819547e-05,
200
+ "loss": 1.0813,
201
+ "step": 155
202
+ },
203
+ {
204
+ "epoch": 0.39,
205
+ "learning_rate": 1.3266247556066122e-05,
206
+ "loss": 1.0699,
207
+ "step": 160
208
+ },
209
+ {
210
+ "epoch": 0.41,
211
+ "learning_rate": 1.2898218482625606e-05,
212
+ "loss": 1.0875,
213
+ "step": 165
214
+ },
215
+ {
216
+ "epoch": 0.42,
217
+ "learning_rate": 1.252585165803135e-05,
218
+ "loss": 1.0872,
219
+ "step": 170
220
+ },
221
+ {
222
+ "epoch": 0.43,
223
+ "learning_rate": 1.2149704402110243e-05,
224
+ "loss": 1.0709,
225
+ "step": 175
226
+ },
227
+ {
228
+ "epoch": 0.44,
229
+ "learning_rate": 1.1770339692844484e-05,
230
+ "loss": 1.0772,
231
+ "step": 180
232
+ },
233
+ {
234
+ "epoch": 0.46,
235
+ "learning_rate": 1.1388325323764889e-05,
236
+ "loss": 1.0847,
237
+ "step": 185
238
+ },
239
+ {
240
+ "epoch": 0.47,
241
+ "learning_rate": 1.1004233054136726e-05,
242
+ "loss": 1.0778,
243
+ "step": 190
244
+ },
245
+ {
246
+ "epoch": 0.48,
247
+ "learning_rate": 1.0618637753210086e-05,
248
+ "loss": 1.0798,
249
+ "step": 195
250
+ },
251
+ {
252
+ "epoch": 0.49,
253
+ "learning_rate": 1.0232116539815558e-05,
254
+ "loss": 1.0671,
255
+ "step": 200
256
+ },
257
+ {
258
+ "epoch": 0.5,
259
+ "learning_rate": 9.845247918592937e-06,
260
+ "loss": 1.0715,
261
+ "step": 205
262
+ },
263
+ {
264
+ "epoch": 0.52,
265
+ "learning_rate": 9.458610914145826e-06,
266
+ "loss": 1.07,
267
+ "step": 210
268
+ },
269
+ {
270
+ "epoch": 0.53,
271
+ "learning_rate": 9.072784204417995e-06,
272
+ "loss": 1.0795,
273
+ "step": 215
274
+ },
275
+ {
276
+ "epoch": 0.54,
277
+ "learning_rate": 8.688345254588579e-06,
278
+ "loss": 1.0686,
279
+ "step": 220
280
+ },
281
+ {
282
+ "epoch": 0.55,
283
+ "learning_rate": 8.305869452782446e-06,
284
+ "loss": 1.0716,
285
+ "step": 225
286
+ },
287
+ {
288
+ "epoch": 0.57,
289
+ "learning_rate": 7.92592924888925e-06,
290
+ "loss": 1.0707,
291
+ "step": 230
292
+ },
293
+ {
294
+ "epoch": 0.58,
295
+ "learning_rate": 7.549093297780133e-06,
296
+ "loss": 1.067,
297
+ "step": 235
298
+ },
299
+ {
300
+ "epoch": 0.59,
301
+ "learning_rate": 7.175925608204428e-06,
302
+ "loss": 1.0732,
303
+ "step": 240
304
+ },
305
+ {
306
+ "epoch": 0.6,
307
+ "learning_rate": 6.806984698640202e-06,
308
+ "loss": 1.0746,
309
+ "step": 245
310
+ },
311
+ {
312
+ "epoch": 0.62,
313
+ "learning_rate": 6.442822761362015e-06,
314
+ "loss": 1.0622,
315
+ "step": 250
316
+ },
317
+ {
318
+ "epoch": 0.63,
319
+ "learning_rate": 6.083984835977154e-06,
320
+ "loss": 1.0552,
321
+ "step": 255
322
+ },
323
+ {
324
+ "epoch": 0.64,
325
+ "learning_rate": 5.731007993667155e-06,
326
+ "loss": 1.0697,
327
+ "step": 260
328
+ },
329
+ {
330
+ "epoch": 0.65,
331
+ "learning_rate": 5.38442053335571e-06,
332
+ "loss": 1.0599,
333
+ "step": 265
334
+ },
335
+ {
336
+ "epoch": 0.67,
337
+ "learning_rate": 5.044741191005908e-06,
338
+ "loss": 1.063,
339
+ "step": 270
340
+ },
341
+ {
342
+ "epoch": 0.67,
343
+ "eval_loss": 1.066710114479065,
344
+ "eval_runtime": 2407.1881,
345
+ "eval_samples_per_second": 9.6,
346
+ "eval_steps_per_second": 0.15,
347
+ "step": 272
348
+ },
349
+ {
350
+ "epoch": 0.67,
351
+ "step": 272,
352
+ "total_flos": 4.932776606721638e+16,
353
+ "train_loss": 1.1599171240540112,
354
+ "train_runtime": 73159.9281,
355
+ "train_samples_per_second": 2.841,
356
+ "train_steps_per_second": 0.006
357
+ }
358
+ ],
359
+ "logging_steps": 5,
360
+ "max_steps": 406,
361
+ "num_train_epochs": 1,
362
+ "save_steps": 500,
363
+ "total_flos": 4.932776606721638e+16,
364
+ "trial_name": null,
365
+ "trial_params": null
366
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:894ab50c9898c2dee04f953d44234befab17dc2499229944290efadb69ed352d
3
+ size 5624