fix tokenization imports
Browse files
tokenization_character_bert.py
CHANGED
@@ -23,8 +23,8 @@ from typing import Dict, List, Optional, Tuple, Union
|
|
23 |
|
24 |
import numpy as np
|
25 |
|
26 |
-
from
|
27 |
-
from
|
28 |
BatchEncoding,
|
29 |
EncodedInput,
|
30 |
PaddingStrategy,
|
@@ -34,8 +34,8 @@ from ...tokenization_utils import (
|
|
34 |
_is_punctuation,
|
35 |
_is_whitespace,
|
36 |
)
|
37 |
-
from
|
38 |
-
from
|
39 |
|
40 |
|
41 |
logger = logging.get_logger(__name__)
|
|
|
23 |
|
24 |
import numpy as np
|
25 |
|
26 |
+
from transformers.file_utils import _is_tensorflow, _is_torch, is_tf_available, is_torch_available, to_py_obj
|
27 |
+
from transformers.tokenization_utils import (
|
28 |
BatchEncoding,
|
29 |
EncodedInput,
|
30 |
PaddingStrategy,
|
|
|
34 |
_is_punctuation,
|
35 |
_is_whitespace,
|
36 |
)
|
37 |
+
from transformers.tokenization_utils_base import ADDED_TOKENS_FILE
|
38 |
+
from transformers.utils import logging
|
39 |
|
40 |
|
41 |
logger = logging.get_logger(__name__)
|