{ | |
"bos_token": "<|endoftext|>", | |
"eos_token": "AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=True)", | |
"pad_token": "AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=True)", | |
"unk_token": "<|endoftext|>" | |
} | |