fix tokenizer initialization issue AttributeError: can't set attribute

#99
by katuni4ka - opened
Files changed (1) hide show
  1. tokenization_chatglm.py +3 -0
tokenization_chatglm.py CHANGED
@@ -70,6 +70,9 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
 
 
 
73
  self.special_tokens = {
74
  "<bos>": self.tokenizer.bos_id,
75
  "<eos>": self.tokenizer.eos_id,
 
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
73
+ kwargs.pop("eos_token", None)
74
+ kwargs.pop("pad_token", None)
75
+ kwargs.pop("unk_token", None)
76
  self.special_tokens = {
77
  "<bos>": self.tokenizer.bos_id,
78
  "<eos>": self.tokenizer.eos_id,