THUdyh commited on
Commit
3724fda
1 Parent(s): d28df4f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -57,7 +57,7 @@ bibtext = """
57
  def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
58
  roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
59
 
60
- im_start, im_end = tokenizer.additional_special_tokens_ids
61
  nl_tokens = tokenizer("\n").input_ids
62
  _system = tokenizer("system").input_ids + nl_tokens
63
  _user = tokenizer("user").input_ids + nl_tokens
 
57
  def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
58
  roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
59
 
60
+ im_start, im_end = tokenizer.additional_special_tokens_ids[:2]
61
  nl_tokens = tokenizer("\n").input_ids
62
  _system = tokenizer("system").input_ids + nl_tokens
63
  _user = tokenizer("user").input_ids + nl_tokens