chenlin commited on
Commit
079479b
1 Parent(s): 15f5821

remove eos token

Browse files
Files changed (1) hide show
  1. app.py +1 -0
app.py CHANGED
@@ -148,6 +148,7 @@ def model_gen(model, text, images, need_bos=True, hd_num=25, max_new_token=2048,
148
  output_text = model.tokenizer.decode(
149
  output_token, add_special_tokens=False)
150
  output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
 
151
  return output_text
152
 
153
 
 
148
  output_text = model.tokenizer.decode(
149
  output_token, add_special_tokens=False)
150
  output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
151
+ output_text = output_text.split('<|im_end|>')[0].strip()
152
  return output_text
153
 
154