added tokenizer
Browse files- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
|
@@ -32861,9 +32861,10 @@
|
|
| 32861 |
}
|
| 32862 |
},
|
| 32863 |
"bos_token": "<s>",
|
| 32864 |
-
"chat_template": "{% if messages[0]['role'] == 'system' %}\n
|
| 32865 |
"clean_up_tokenization_spaces": false,
|
| 32866 |
"eos_token": "</s>",
|
|
|
|
| 32867 |
"legacy": false,
|
| 32868 |
"model_max_length": 1000000000000000019884624838656,
|
| 32869 |
"pad_token": "[PAD]",
|
|
|
|
| 32861 |
}
|
| 32862 |
},
|
| 32863 |
"bos_token": "<s>",
|
| 32864 |
+
"chat_template": "{% if messages[0]['role'] == 'system' %}\n {% set loop_messages = messages[1:] %}\n {% set system_message = messages[0]['content'] %}\n {% else %}\n {% set loop_messages = messages %}\n {% set system_message = false %}\n {% endif %}\n {% for message in loop_messages %}\n {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {% endif %}\n {% if loop.index0 == 0 and system_message != false %}\n {% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}\n {% else %}\n {% set content = message['content'] %}\n {% endif %}\n {% if message['role'] == 'user' %}\n {{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}\n {% elif message['role'] == 'assistant' %}\n {{ ' ' + content.strip() + ' ' + eos_token }}\n {% endif %}\n {% endfor %}",
|
| 32865 |
"clean_up_tokenization_spaces": false,
|
| 32866 |
"eos_token": "</s>",
|
| 32867 |
+
"extra_special_tokens": {},
|
| 32868 |
"legacy": false,
|
| 32869 |
"model_max_length": 1000000000000000019884624838656,
|
| 32870 |
"pad_token": "[PAD]",
|