Spaces:
Build error
Build error
from transformers import BertTokenizer | |
def bert_tokenizer(pretrained_model_name_or_path): | |
tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path=pretrained_model_name_or_path, | |
truncation_side='right') | |
tokenizer.add_special_tokens({"bos_token": "[DEC]"}) | |
return tokenizer | |