katuni4ka commited on
Commit
c2354b9
1 Parent(s): 7fabe56

fix tokenizer initialization issue AttributeError: can't set attribute

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -1
tokenization_chatglm.py CHANGED
@@ -70,12 +70,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
 
73
  self.special_tokens = {
74
  "<bos>": self.tokenizer.bos_id,
75
  "<eos>": self.tokenizer.eos_id,
76
  "<pad>": self.tokenizer.pad_id
77
  }
78
- super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
79
 
80
  def get_command(self, token):
81
  if token in self.special_tokens:
 
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
73
+ super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
74
  self.special_tokens = {
75
  "<bos>": self.tokenizer.bos_id,
76
  "<eos>": self.tokenizer.eos_id,
77
  "<pad>": self.tokenizer.pad_id
78
  }
 
79
 
80
  def get_command(self, token):
81
  if token in self.special_tokens: