BBuf commited on
Commit
06470d2
1 Parent(s): 4f76edf

Update tokenization_rwkv_world.py

Browse files
Files changed (1) hide show
  1. tokenization_rwkv_world.py +3 -3
tokenization_rwkv_world.py CHANGED
@@ -244,13 +244,13 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
244
  **kwargs
245
  ):
246
  self.add_bos_token = False
 
 
 
247
  super().__init__(
248
  errors=errors,
249
  **kwargs,
250
  )
251
-
252
- with open(vocab_file, encoding="utf-8") as vocab_handle:
253
- self.encoder = json.load(vocab_handle)
254
  self.decoder = {v: k for k, v in self.encoder.items()}
255
  self.trie = DATrie(self.all_special_ids)
256
  for k, v in self.encoder.items():
 
244
  **kwargs
245
  ):
246
  self.add_bos_token = False
247
+
248
+ with open(vocab_file, encoding="utf-8") as vocab_handle:
249
+ self.encoder = json.load(vocab_handle)
250
  super().__init__(
251
  errors=errors,
252
  **kwargs,
253
  )
 
 
 
254
  self.decoder = {v: k for k, v in self.encoder.items()}
255
  self.trie = DATrie(self.all_special_ids)
256
  for k, v in self.encoder.items():