tokenizer_config.json

{
 "add_prefix_space": false,
 "added_tokens_decoder": {
  "151643": {
   "content": "<|endoftext|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  },
  "151644": {
   "content": "<|im_start|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  },
  "151645": {
   "content": "<|im_end|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  },
  "151646": {
   "content": "<|jasper_img_start|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  },
  "151647": {
   "content": "<|jasper_img_token|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  },
  "151648": {
   "content": "<|jasper_img_end|>",
   "lstrip": false,
   "normalized": false,
   "rstrip": false,
   "single_word": false,
   "special": true
  }
 },
 "additional_special_tokens": [
  "<|im_start|>",
  "<|im_end|>"
 ],
 "auto_map": {
  "AutoTokenizer": [
   "tokenization_qwen.Qwen2Tokenizer",
   "tokenization_qwen.Qwen2TokenizerFast"
  ]
 },
 "bos_token": null,
 "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
 "clean_up_tokenization_spaces": false,
 "eos_token": "<|endoftext|>",
 "errors": "replace",
 "model_max_length": 32768,
 "pad_token": "<|endoftext|>",
 "split_special_tokens": false,
 "tokenizer_class": "Qwen2Tokenizer",
 "unk_token": null,
 "add_eos_token": true
}