814ee6b d27a756 814ee6b
1
2
3
4
5
6
7
8
9
10
11
12
USE_REMOTE = False # use remote tokenizer or local tokenizer # load_vocab_with_SPECIAL_TOKEN = True # 如果不包含会导致计算词典大小错误、overlap_token计算不一致。 # encoding config ADD_SPECIAL_TOKEN = False # LAZY_IMPORT = True # DEBUG: 设置环境变量 RUST_BACKTRACE=full #
