File size: 645 Bytes
5169b80 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 |
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from hf_olmo.configuration_olmo import OLMoConfig
class OLMoTokenizerFast(PreTrainedTokenizerFast):
# Note: OLMo's tokenizer is already a wrapper around huggingface. This is potentially unnecessary.
pass
# def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
# # This is required to make the implementation complete.
# pass
# Register the tokenizer class so that it is available for transformer pipelines, auto-loading etc.
AutoTokenizer.register(OLMoConfig, fast_tokenizer_class=OLMoTokenizerFast)
|