Commit
•
47bca49
1
Parent(s):
2d3dc10
Upload tokenizer_config.json (#1)
Browse files- Upload tokenizer_config.json (1bf6d658d0cb80c64685eae646175e19068850cf)
Co-authored-by: Kirill <fedyanin@users.noreply.huggingface.co>
- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
@@ -16230,7 +16230,8 @@
|
|
16230 |
"chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'user' %}{{ '<|user|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}{% else %}{{ '<|assistant|>\n' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endfor %}",
|
16231 |
"clean_up_tokenization_spaces": true,
|
16232 |
"eos_token": "<|endoftext|>",
|
16233 |
-
"model_max_length":
|
16234 |
"pad_token": "<pad>",
|
16235 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
16236 |
}
|
|
|
|
16230 |
"chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'user' %}{{ '<|user|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}{% else %}{{ '<|assistant|>\n' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endfor %}",
|
16231 |
"clean_up_tokenization_spaces": true,
|
16232 |
"eos_token": "<|endoftext|>",
|
16233 |
+
"model_max_length": 32768,
|
16234 |
"pad_token": "<pad>",
|
16235 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
16236 |
}
|
16237 |
+
|