gpt-neoxt-20b-v6 / tokenizer_config.json
manojpreveen's picture
Duplicate from iamplus/gpt-neoxt-20b-v6
a43a79f verified
raw
history blame contribute delete
414 Bytes
{
"add_prefix_space": false,
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"name_or_path": "EleutherAI/gpt-neox-20b",
"special_tokens_map_file": "/root/.cache/huggingface/transformers/d9026dc928c47ac2a72d46fea7db959acc4bacac2176bf32be5c331604b77d32.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22",
"tokenizer_class": "GPTNeoXTokenizer",
"unk_token": "<|endoftext|>"
}