from transformers import LlamaTokenizerFast | |
from .tokenization_cognitivess import CognitivessTokenizer | |
class CognitivessTokenizerFast(LlamaTokenizerFast): | |
slow_tokenizer_class = CognitivessTokenizer | |
from transformers import LlamaTokenizerFast | |
from .tokenization_cognitivess import CognitivessTokenizer | |
class CognitivessTokenizerFast(LlamaTokenizerFast): | |
slow_tokenizer_class = CognitivessTokenizer | |