Update tokenization_rwkv_world.py
Browse files
tokenization_rwkv_world.py
CHANGED
@@ -244,13 +244,13 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
|
|
244 |
**kwargs
|
245 |
):
|
246 |
self.add_bos_token = False
|
|
|
|
|
|
|
247 |
super().__init__(
|
248 |
errors=errors,
|
249 |
**kwargs,
|
250 |
)
|
251 |
-
|
252 |
-
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
253 |
-
self.encoder = json.load(vocab_handle)
|
254 |
self.decoder = {v: k for k, v in self.encoder.items()}
|
255 |
self.trie = DATrie(self.all_special_ids)
|
256 |
for k, v in self.encoder.items():
|
|
|
244 |
**kwargs
|
245 |
):
|
246 |
self.add_bos_token = False
|
247 |
+
|
248 |
+
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
249 |
+
self.encoder = json.load(vocab_handle)
|
250 |
super().__init__(
|
251 |
errors=errors,
|
252 |
**kwargs,
|
253 |
)
|
|
|
|
|
|
|
254 |
self.decoder = {v: k for k, v in self.encoder.items()}
|
255 |
self.trie = DATrie(self.all_special_ids)
|
256 |
for k, v in self.encoder.items():
|