tuanio commited on
Commit
86e414a
1 Parent(s): ea649a0

Upload folder using huggingface_hub

Browse files
added_tokens.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "<|endoftext|>": 151643,
3
+ "<|extra_0|>": 151646,
4
+ "<|im_end|>": 151645,
5
+ "<|im_start|>": 151644
6
+ }
config.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./checkpoints/ft-llava-qwen1.5-0.5b-vista_reasoning-merged-4ep",
3
+ "architectures": [
4
+ "MoELLaVAQwen1_5ForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "eos_token_id": 151643,
9
+ "freeze_mm_mlp_adapter": false,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 1024,
12
+ "image_aspect_ratio": "pad",
13
+ "image_projector_type": "mlp2x_gelu",
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 2816,
16
+ "lora": {},
17
+ "max_position_embeddings": 32768,
18
+ "max_window_layers": 21,
19
+ "mm_hidden_size": 768,
20
+ "mm_image_tower": "google/siglip-base-patch16-256-multilingual",
21
+ "mm_projector_lr": null,
22
+ "mm_use_im_patch_token": false,
23
+ "mm_use_im_start_end": false,
24
+ "mm_video_tower": null,
25
+ "mm_vision_select_feature": "patch",
26
+ "mm_vision_select_layer": -2,
27
+ "model_type": "moe_llava_qwen1_5",
28
+ "moe": {
29
+ "capacity_factor": 1.5,
30
+ "ep_size": 1,
31
+ "eval_capacity_factor": 2.0,
32
+ "min_capacity": 0,
33
+ "moe_enable": true,
34
+ "moe_layers_idx": [
35
+ 0,
36
+ 2,
37
+ 4,
38
+ 6,
39
+ 8,
40
+ 10,
41
+ 12,
42
+ 14,
43
+ 16,
44
+ 18,
45
+ 20,
46
+ 22
47
+ ],
48
+ "moe_mode": "sparse",
49
+ "num_experts": [
50
+ 4,
51
+ 4,
52
+ 4,
53
+ 4,
54
+ 4,
55
+ 4,
56
+ 4,
57
+ 4,
58
+ 4,
59
+ 4,
60
+ 4,
61
+ 4
62
+ ],
63
+ "router_aux_loss_coef": 0.01,
64
+ "top_k_experts": 2,
65
+ "train_modules": [
66
+ "mlp.gate_proj",
67
+ "mlp.up_proj",
68
+ "mlp.down_proj",
69
+ "wg"
70
+ ],
71
+ "use_residual": false
72
+ },
73
+ "num_attention_heads": 16,
74
+ "num_hidden_layers": 24,
75
+ "num_key_value_heads": 16,
76
+ "pad_token_id": 151646,
77
+ "rms_norm_eps": 1e-06,
78
+ "rope_theta": 1000000.0,
79
+ "sliding_window": 32768,
80
+ "tie_word_embeddings": true,
81
+ "tokenizer_padding_side": "right",
82
+ "torch_dtype": "bfloat16",
83
+ "transformers_version": "4.37.0",
84
+ "tune_mm_mlp_adapter": false,
85
+ "use_cache": true,
86
+ "use_mm_proj": true,
87
+ "use_sliding_window": false,
88
+ "video_global_proj": false,
89
+ "video_projector_type": "linear",
90
+ "video_spatial_proj": false,
91
+ "video_temproal_proj": false,
92
+ "vocab_size": 151646
93
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "eos_token_id": 151643,
4
+ "max_new_tokens": 2048,
5
+ "transformers_version": "4.37.0"
6
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:edd31310b237533cc56559a926fc3e3f4db8e098f6d97bb7608f50953e0ce55b
3
+ size 2050524576
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06739df7f61af3c0df008bff0dd27d1144a1083f6b3ae366c3352409942a3141
3
+ size 1740053129
special_tokens_map.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>"
5
+ ],
6
+ "eos_token": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "pad_token": "<|extra_0|>",
14
+ "unk_token": {
15
+ "content": "<|extra_0|>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ }
21
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "151646": {
29
+ "content": "<|extra_0|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ }
36
+ },
37
+ "additional_special_tokens": [
38
+ "<|im_start|>",
39
+ "<|im_end|>"
40
+ ],
41
+ "bos_token": null,
42
+ "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
43
+ "clean_up_tokenization_spaces": false,
44
+ "eos_token": "<|endoftext|>",
45
+ "errors": "replace",
46
+ "model_max_length": 2048,
47
+ "pad_token": "<|extra_0|>",
48
+ "padding_side": "right",
49
+ "split_special_tokens": false,
50
+ "tokenizer_class": "Qwen2Tokenizer",
51
+ "unk_token": "<|extra_0|>"
52
+ }
trainer_state.json ADDED
@@ -0,0 +1,366 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.9992900248491303,
5
+ "eval_steps": 500,
6
+ "global_step": 2816,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.04,
13
+ "learning_rate": 1.1764705882352942e-05,
14
+ "loss": 2.8037,
15
+ "step": 50
16
+ },
17
+ {
18
+ "epoch": 0.07,
19
+ "learning_rate": 1.999851133118102e-05,
20
+ "loss": 2.4343,
21
+ "step": 100
22
+ },
23
+ {
24
+ "epoch": 0.11,
25
+ "learning_rate": 1.9972058436024328e-05,
26
+ "loss": 2.233,
27
+ "step": 150
28
+ },
29
+ {
30
+ "epoch": 0.14,
31
+ "learning_rate": 1.9912624721856506e-05,
32
+ "loss": 2.1259,
33
+ "step": 200
34
+ },
35
+ {
36
+ "epoch": 0.18,
37
+ "learning_rate": 1.9820406755172106e-05,
38
+ "loss": 2.0545,
39
+ "step": 250
40
+ },
41
+ {
42
+ "epoch": 0.21,
43
+ "learning_rate": 1.9695709530580642e-05,
44
+ "loss": 1.9909,
45
+ "step": 300
46
+ },
47
+ {
48
+ "epoch": 0.25,
49
+ "learning_rate": 1.9538945462090903e-05,
50
+ "loss": 1.9676,
51
+ "step": 350
52
+ },
53
+ {
54
+ "epoch": 0.28,
55
+ "learning_rate": 1.9350633019124565e-05,
56
+ "loss": 1.9318,
57
+ "step": 400
58
+ },
59
+ {
60
+ "epoch": 0.32,
61
+ "learning_rate": 1.913139501177033e-05,
62
+ "loss": 1.897,
63
+ "step": 450
64
+ },
65
+ {
66
+ "epoch": 0.35,
67
+ "learning_rate": 1.888195653094972e-05,
68
+ "loss": 1.8821,
69
+ "step": 500
70
+ },
71
+ {
72
+ "epoch": 0.39,
73
+ "learning_rate": 1.860314255030716e-05,
74
+ "loss": 1.8568,
75
+ "step": 550
76
+ },
77
+ {
78
+ "epoch": 0.43,
79
+ "learning_rate": 1.8295875197755593e-05,
80
+ "loss": 1.8416,
81
+ "step": 600
82
+ },
83
+ {
84
+ "epoch": 0.46,
85
+ "learning_rate": 1.796117070570155e-05,
86
+ "loss": 1.8278,
87
+ "step": 650
88
+ },
89
+ {
90
+ "epoch": 0.5,
91
+ "learning_rate": 1.7600136050036265e-05,
92
+ "loss": 1.8107,
93
+ "step": 700
94
+ },
95
+ {
96
+ "epoch": 0.53,
97
+ "learning_rate": 1.7213965289008755e-05,
98
+ "loss": 1.799,
99
+ "step": 750
100
+ },
101
+ {
102
+ "epoch": 0.57,
103
+ "learning_rate": 1.6803935614089426e-05,
104
+ "loss": 1.7788,
105
+ "step": 800
106
+ },
107
+ {
108
+ "epoch": 0.6,
109
+ "learning_rate": 1.637140312588523e-05,
110
+ "loss": 1.7629,
111
+ "step": 850
112
+ },
113
+ {
114
+ "epoch": 0.64,
115
+ "learning_rate": 1.5917798349076846e-05,
116
+ "loss": 1.7656,
117
+ "step": 900
118
+ },
119
+ {
120
+ "epoch": 0.67,
121
+ "learning_rate": 1.5444621501211386e-05,
122
+ "loss": 1.7521,
123
+ "step": 950
124
+ },
125
+ {
126
+ "epoch": 0.71,
127
+ "learning_rate": 1.4953437530998262e-05,
128
+ "loss": 1.738,
129
+ "step": 1000
130
+ },
131
+ {
132
+ "epoch": 0.75,
133
+ "learning_rate": 1.4445870942518226e-05,
134
+ "loss": 1.729,
135
+ "step": 1050
136
+ },
137
+ {
138
+ "epoch": 0.78,
139
+ "learning_rate": 1.3923600422463505e-05,
140
+ "loss": 1.7321,
141
+ "step": 1100
142
+ },
143
+ {
144
+ "epoch": 0.82,
145
+ "learning_rate": 1.3388353288178595e-05,
146
+ "loss": 1.7203,
147
+ "step": 1150
148
+ },
149
+ {
150
+ "epoch": 0.85,
151
+ "learning_rate": 1.284189977486371e-05,
152
+ "loss": 1.7134,
153
+ "step": 1200
154
+ },
155
+ {
156
+ "epoch": 0.89,
157
+ "learning_rate": 1.228604718083506e-05,
158
+ "loss": 1.7147,
159
+ "step": 1250
160
+ },
161
+ {
162
+ "epoch": 0.92,
163
+ "learning_rate": 1.1722633890205434e-05,
164
+ "loss": 1.6945,
165
+ "step": 1300
166
+ },
167
+ {
168
+ "epoch": 0.96,
169
+ "learning_rate": 1.1153523292753974e-05,
170
+ "loss": 1.6996,
171
+ "step": 1350
172
+ },
173
+ {
174
+ "epoch": 0.99,
175
+ "learning_rate": 1.0580597621094166e-05,
176
+ "loss": 1.6959,
177
+ "step": 1400
178
+ },
179
+ {
180
+ "epoch": 1.03,
181
+ "learning_rate": 1.0005751725522469e-05,
182
+ "loss": 1.6188,
183
+ "step": 1450
184
+ },
185
+ {
186
+ "epoch": 1.06,
187
+ "learning_rate": 9.430886807136206e-06,
188
+ "loss": 1.6012,
189
+ "step": 1500
190
+ },
191
+ {
192
+ "epoch": 1.1,
193
+ "learning_rate": 8.857904129947295e-06,
194
+ "loss": 1.6043,
195
+ "step": 1550
196
+ },
197
+ {
198
+ "epoch": 1.14,
199
+ "learning_rate": 8.288698732787898e-06,
200
+ "loss": 1.602,
201
+ "step": 1600
202
+ },
203
+ {
204
+ "epoch": 1.17,
205
+ "learning_rate": 7.725153161804767e-06,
206
+ "loss": 1.604,
207
+ "step": 1650
208
+ },
209
+ {
210
+ "epoch": 1.21,
211
+ "learning_rate": 7.169131244270906e-06,
212
+ "loss": 1.6012,
213
+ "step": 1700
214
+ },
215
+ {
216
+ "epoch": 1.24,
217
+ "learning_rate": 6.6224719243066386e-06,
218
+ "loss": 1.5921,
219
+ "step": 1750
220
+ },
221
+ {
222
+ "epoch": 1.28,
223
+ "learning_rate": 6.086983180897257e-06,
224
+ "loss": 1.6,
225
+ "step": 1800
226
+ },
227
+ {
228
+ "epoch": 1.31,
229
+ "learning_rate": 5.564436048322415e-06,
230
+ "loss": 1.5887,
231
+ "step": 1850
232
+ },
233
+ {
234
+ "epoch": 1.35,
235
+ "learning_rate": 5.056558758773566e-06,
236
+ "loss": 1.59,
237
+ "step": 1900
238
+ },
239
+ {
240
+ "epoch": 1.38,
241
+ "learning_rate": 4.56503102653177e-06,
242
+ "loss": 1.5835,
243
+ "step": 1950
244
+ },
245
+ {
246
+ "epoch": 1.42,
247
+ "learning_rate": 4.091478492609857e-06,
248
+ "loss": 1.5847,
249
+ "step": 2000
250
+ },
251
+ {
252
+ "epoch": 1.46,
253
+ "learning_rate": 3.637467348232414e-06,
254
+ "loss": 1.5934,
255
+ "step": 2050
256
+ },
257
+ {
258
+ "epoch": 1.49,
259
+ "learning_rate": 3.2044991549353832e-06,
260
+ "loss": 1.588,
261
+ "step": 2100
262
+ },
263
+ {
264
+ "epoch": 1.53,
265
+ "learning_rate": 2.7940058784169543e-06,
266
+ "loss": 1.5882,
267
+ "step": 2150
268
+ },
269
+ {
270
+ "epoch": 1.56,
271
+ "learning_rate": 2.4073451525643566e-06,
272
+ "loss": 1.5791,
273
+ "step": 2200
274
+ },
275
+ {
276
+ "epoch": 1.6,
277
+ "learning_rate": 2.045795789319962e-06,
278
+ "loss": 1.5802,
279
+ "step": 2250
280
+ },
281
+ {
282
+ "epoch": 1.63,
283
+ "learning_rate": 1.710553549237043e-06,
284
+ "loss": 1.5806,
285
+ "step": 2300
286
+ },
287
+ {
288
+ "epoch": 1.67,
289
+ "learning_rate": 1.402727186713303e-06,
290
+ "loss": 1.5816,
291
+ "step": 2350
292
+ },
293
+ {
294
+ "epoch": 1.7,
295
+ "learning_rate": 1.1233347829818563e-06,
296
+ "loss": 1.5822,
297
+ "step": 2400
298
+ },
299
+ {
300
+ "epoch": 1.74,
301
+ "learning_rate": 8.733003789876726e-07,
302
+ "loss": 1.5751,
303
+ "step": 2450
304
+ },
305
+ {
306
+ "epoch": 1.77,
307
+ "learning_rate": 6.534509192856453e-07,
308
+ "loss": 1.5708,
309
+ "step": 2500
310
+ },
311
+ {
312
+ "epoch": 1.81,
313
+ "learning_rate": 4.645135170677595e-07,
314
+ "loss": 1.5777,
315
+ "step": 2550
316
+ },
317
+ {
318
+ "epoch": 1.85,
319
+ "learning_rate": 3.0711304936486664e-07,
320
+ "loss": 1.5673,
321
+ "step": 2600
322
+ },
323
+ {
324
+ "epoch": 1.88,
325
+ "learning_rate": 1.8177009037645633e-07,
326
+ "loss": 1.5766,
327
+ "step": 2650
328
+ },
329
+ {
330
+ "epoch": 1.92,
331
+ "learning_rate": 8.889918976358602e-08,
332
+ "loss": 1.5677,
333
+ "step": 2700
334
+ },
335
+ {
336
+ "epoch": 1.95,
337
+ "learning_rate": 2.8807501599226185e-08,
338
+ "loss": 1.5849,
339
+ "step": 2750
340
+ },
341
+ {
342
+ "epoch": 1.99,
343
+ "learning_rate": 1.6937685105011813e-09,
344
+ "loss": 1.5809,
345
+ "step": 2800
346
+ },
347
+ {
348
+ "epoch": 2.0,
349
+ "step": 2816,
350
+ "total_flos": 2.1906737398074573e+17,
351
+ "train_loss": 1.738261958414858,
352
+ "train_runtime": 14007.5678,
353
+ "train_samples_per_second": 16.084,
354
+ "train_steps_per_second": 0.201
355
+ }
356
+ ],
357
+ "logging_steps": 50,
358
+ "max_steps": 2816,
359
+ "num_input_tokens_seen": 0,
360
+ "num_train_epochs": 2,
361
+ "save_steps": 24000,
362
+ "total_flos": 2.1906737398074573e+17,
363
+ "train_batch_size": 8,
364
+ "trial_name": null,
365
+ "trial_params": null
366
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce6c87b0b3ba0cd7fb0d20932fa66e9ed29def2e45aa1e7458c852c456a54add
3
+ size 5947
vocab.json ADDED
The diff for this file is too large to render. See raw diff