Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -57,7 +57,7 @@ bibtext = """
|
|
57 |
def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
|
58 |
roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
|
59 |
|
60 |
-
im_start, im_end = tokenizer.additional_special_tokens_ids
|
61 |
nl_tokens = tokenizer("\n").input_ids
|
62 |
_system = tokenizer("system").input_ids + nl_tokens
|
63 |
_user = tokenizer("user").input_ids + nl_tokens
|
|
|
57 |
def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
|
58 |
roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
|
59 |
|
60 |
+
im_start, im_end = tokenizer.additional_special_tokens_ids[:2]
|
61 |
nl_tokens = tokenizer("\n").input_ids
|
62 |
_system = tokenizer("system").input_ids + nl_tokens
|
63 |
_user = tokenizer("user").input_ids + nl_tokens
|