Update tokenizer_config.json
Browse files- tokenizer_config.json +3 -1
tokenizer_config.json
CHANGED
@@ -35,7 +35,8 @@
|
|
35 |
"128004": {
|
36 |
"content": "<|reserved_special_token_2|>",
|
37 |
"lstrip": false,
|
38 |
-
"normalized": false
|
|
|
39 |
"rstrip": false,
|
40 |
"single_word": false,
|
41 |
"special": true
|
@@ -2052,6 +2053,7 @@
|
|
2052 |
"bos_token": "<|begin_of_text|>",
|
2053 |
"clean_up_tokenization_spaces": true,
|
2054 |
"eos_token": "<|end_of_text|>",
|
|
|
2055 |
"model_input_names": [
|
2056 |
"input_ids",
|
2057 |
"attention_mask"
|
|
|
35 |
"128004": {
|
36 |
"content": "<|reserved_special_token_2|>",
|
37 |
"lstrip": false,
|
38 |
+
"normalized": false
|
39 |
+
,
|
40 |
"rstrip": false,
|
41 |
"single_word": false,
|
42 |
"special": true
|
|
|
2053 |
"bos_token": "<|begin_of_text|>",
|
2054 |
"clean_up_tokenization_spaces": true,
|
2055 |
"eos_token": "<|end_of_text|>",
|
2056 |
+
"chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|endoftext|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
|
2057 |
"model_input_names": [
|
2058 |
"input_ids",
|
2059 |
"attention_mask"
|