Crystalcareai
commited on
Commit
•
c99aadf
1
Parent(s):
db7dc87
Update tokenization_gemmoe.py
Browse files- tokenization_gemmoe.py +3 -1
tokenization_gemmoe.py
CHANGED
@@ -14,14 +14,16 @@
|
|
14 |
# limitations under the License.
|
15 |
"""Tokenization classes for Gemmoe."""
|
16 |
import os
|
17 |
-
import logging
|
18 |
from shutil import copyfile
|
19 |
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
20 |
|
21 |
import sentencepiece as spm
|
22 |
|
|
|
|
|
23 |
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
|
24 |
|
|
|
25 |
if TYPE_CHECKING:
|
26 |
pass
|
27 |
|
|
|
14 |
# limitations under the License.
|
15 |
"""Tokenization classes for Gemmoe."""
|
16 |
import os
|
|
|
17 |
from shutil import copyfile
|
18 |
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
19 |
|
20 |
import sentencepiece as spm
|
21 |
|
22 |
+
from transformers.utils import logging
|
23 |
+
|
24 |
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
|
25 |
|
26 |
+
|
27 |
if TYPE_CHECKING:
|
28 |
pass
|
29 |
|