Commit
•
f9b0bf1
1
Parent(s):
7b2146d
Update envibert_tokenizer.py
Browse files- envibert_tokenizer.py +1 -1
envibert_tokenizer.py
CHANGED
|
@@ -3,7 +3,7 @@ import sentencepiece as spm
|
|
| 3 |
import os
|
| 4 |
from transformers import PreTrainedTokenizer
|
| 5 |
from collections import Counter
|
| 6 |
-
from typing import List, Optional
|
| 7 |
|
| 8 |
|
| 9 |
class RobertaTokenizer(PreTrainedTokenizer):
|
|
|
|
| 3 |
import os
|
| 4 |
from transformers import PreTrainedTokenizer
|
| 5 |
from collections import Counter
|
| 6 |
+
from typing import List, Optional, Tuple
|
| 7 |
|
| 8 |
|
| 9 |
class RobertaTokenizer(PreTrainedTokenizer):
|