Commit
•
3997c7f
1
Parent(s):
e56d474
Upload tokenization_cohere_fast.py with huggingface_hub
Browse files
tokenization_cohere_fast.py
CHANGED
|
@@ -751,4 +751,4 @@ class CohereTokenizerFast(PreTrainedTokenizerFast):
|
|
| 751 |
|
| 752 |
|
| 753 |
# register the tokenizer to AutoTokenizer
|
| 754 |
-
AutoTokenizer.register(CohereConfig, fast_tokenizer_class=CohereTokenizerFast)
|
|
|
|
| 751 |
|
| 752 |
|
| 753 |
# register the tokenizer to AutoTokenizer
|
| 754 |
+
AutoTokenizer.register(CohereConfig, fast_tokenizer_class=CohereTokenizerFast, exist_ok=True)
|