|
{ |
|
"auto_map": { |
|
"AutoTokenizer": [ |
|
"tokenization_chatglm.ChatGLMTokenizer", |
|
null |
|
] |
|
}, |
|
"bos_token": "<sop>", |
|
"clean_up_tokenization_spaces": true, |
|
"do_lower_case": false, |
|
"end_token": "</s>", |
|
"eos_token": "<eop>", |
|
"gmask_token": "[gMASK]", |
|
"mask_token": "[MASK]", |
|
"model_max_length": 1000000000000000019884624838656, |
|
"num_image_tokens": 0, |
|
"pad_token": "<pad>", |
|
"padding_side": "left", |
|
"remove_space": false, |
|
"special_tokens_map_file": "/remote-home/rikka/chat-law-key-word-extract/chatglm/model/chatglm/special_tokens_map.json", |
|
"tokenizer_class": "ChatGLMTokenizer", |
|
"unk_token": "<unk>" |
|
} |
|
|