EtashGuha commited on
Commit
da63c0d
1 Parent(s): da69102

Added chat templating to tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "tokenizer_class": "GPTNeoXTokenizer"}
 
1
+ {"chat_template": "Below is an instruction that describes a task.\n\nWrite a response that appropriately completes the request.\n\n### Instruction:\n{{if messages[-1].role == 'user'}}{{messages[-1].content}}{{else}}{{raise_exception('Last message must be from user')}}{{/if}}\n\n### Response:", "unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "tokenizer_class": "GPTNeoXTokenizer"}