File size: 348 Bytes
590af54
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
from transformers import BertTokenizer


def bert_tokenizer(pretrained_model_name_or_path):
    tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path=pretrained_model_name_or_path,
                                              truncation_side='right')
    tokenizer.add_special_tokens({"bos_token": "[DEC]"})
    return tokenizer