Spaces:
Running
on
Zero
Running
on
Zero
chenlin
commited on
Commit
•
079479b
1
Parent(s):
15f5821
remove eos token
Browse files
app.py
CHANGED
@@ -148,6 +148,7 @@ def model_gen(model, text, images, need_bos=True, hd_num=25, max_new_token=2048,
|
|
148 |
output_text = model.tokenizer.decode(
|
149 |
output_token, add_special_tokens=False)
|
150 |
output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
|
|
|
151 |
return output_text
|
152 |
|
153 |
|
|
|
148 |
output_text = model.tokenizer.decode(
|
149 |
output_token, add_special_tokens=False)
|
150 |
output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
|
151 |
+
output_text = output_text.split('<|im_end|>')[0].strip()
|
152 |
return output_text
|
153 |
|
154 |
|