bjoernp commited on
Commit
50acf79
·
verified ·
1 Parent(s): d448fa5

Update tokenizer_config.json

Browse files

Use correct bos token that the model was trained on

Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -2051,9 +2051,9 @@
2051
  "special": true
2052
  }
2053
  },
2054
- "bos_token": "<s>",
2055
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
2056
- "clean_up_tokenization_spaces": false,
2057
  "eos_token": "<|im_end|>",
2058
  "legacy": false,
2059
  "model_max_length": 1000000000000000019884624838656,
 
2051
  "special": true
2052
  }
2053
  },
2054
+ "bos_token": "<|im_start|>",
2055
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
2056
+ "clean_up_tokenization_spaces": true,
2057
  "eos_token": "<|im_end|>",
2058
  "legacy": false,
2059
  "model_max_length": 1000000000000000019884624838656,