| { | |
| "add_bos_token":true, | |
| "add_eos_token":false, | |
| "model_max_length":2048, | |
| "pad_token":null, | |
| "sp_model_kwargs":{ | |
| }, | |
| "tokenizer_class":"LlamaTokenizer", | |
| "clean_up_tokenization_spaces":false, | |
| "bos_token":{ | |
| "__type":"AddedToken", | |
| "content":"<s>", | |
| "lstrip":false, | |
| "normalized":true, | |
| "rstrip":false, | |
| "single_word":false | |
| }, | |
| "eos_token":{ | |
| "__type":"AddedToken", | |
| "content":"</s>", | |
| "lstrip":false, | |
| "normalized":true, | |
| "rstrip":false, | |
| "single_word":false | |
| }, | |
| "unk_token":{ | |
| "__type":"AddedToken", | |
| "content":"<unk>", | |
| "lstrip":false, | |
| "normalized":true, | |
| "rstrip":false, | |
| "single_word":false | |
| } | |
| } |