Spaces:
Runtime error
Runtime error
Update tokenization_small100.py
Browse files- tokenization_small100.py +1 -1
tokenization_small100.py
CHANGED
@@ -180,7 +180,7 @@ class SMALL100Tokenizer(PreTrainedTokenizer):
|
|
180 |
|
181 |
@property
|
182 |
def vocab_size(self) -> int:
|
183 |
-
return len(self.
|
184 |
|
185 |
@property
|
186 |
def tgt_lang(self) -> str:
|
|
|
180 |
|
181 |
@property
|
182 |
def vocab_size(self) -> int:
|
183 |
+
return len(self.encode) + len(self.lang_token_to_id) + self.num_madeup_words
|
184 |
|
185 |
@property
|
186 |
def tgt_lang(self) -> str:
|