ClefChen commited on
Commit
3441e15
·
verified ·
1 Parent(s): 286daf8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -0
app.py CHANGED
@@ -18,6 +18,11 @@ from transformers import AutoModel, AutoTokenizer
18
  # 导入预训练模型和分词器
19
  model_name = "microsoft/phi-2"
20
  tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 
 
 
 
21
  model = AutoModel.from_pretrained(model_name)
22
 
23
  # Import data
 
18
  # 导入预训练模型和分词器
19
  model_name = "microsoft/phi-2"
20
  tokenizer = AutoTokenizer.from_pretrained(model_name)
21
+
22
+ # 设置填充令牌,如果分词器没有默认的填充令牌
23
+ if tokenizer.pad_token is None:
24
+ tokenizer.pad_token = tokenizer.eos_token
25
+
26
  model = AutoModel.from_pretrained(model_name)
27
 
28
  # Import data