Spaces:
Build error
Build error
File size: 348 Bytes
590af54 |
1 2 3 4 5 6 7 8 9 |
from transformers import BertTokenizer
def bert_tokenizer(pretrained_model_name_or_path):
tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path=pretrained_model_name_or_path,
truncation_side='right')
tokenizer.add_special_tokens({"bos_token": "[DEC]"})
return tokenizer
|