jpacifico's picture
Upload tokenizer
19c88d9 verified
raw
history blame
940 Bytes
{
"<|/code|>": 32014,
"<|/data|>": 32033,
"<|/inst|>": 32037,
"<|/query|>": 32031,
"<|/sys|>": 32035,
"<|assistant_mask|>": 32017,
"<|assistant|>": 32001,
"<|calc|>": 32012,
"<|code|>": 32013,
"<|continue|>": 32009,
"<|data|>": 32032,
"<|diff_marker|>": 32025,
"<|disc_sep|>": 32029,
"<|disc_start|>": 32028,
"<|disc_thread|><|query|>": 32030,
"<|endoftext|>": 32000,
"<|end|>": 32007,
"<|fim_middle|>": 32021,
"<|fim_prefix|>": 32020,
"<|fim_suffix|>": 32022,
"<|function_call|>": 32005,
"<|function_list|>": 32011,
"<|function_output|>": 32003,
"<|ghissue|>": 32026,
"<|ghreview|>": 32027,
"<|inst|>": 32036,
"<|ipynb_marker|>": 32024,
"<|message|>": 32019,
"<|meta_start|>": 32023,
"<|raw|>": 32008,
"<|resource|>": 32016,
"<|start|>": 32018,
"<|step|>": 32002,
"<|summary|>": 32015,
"<|system|>": 32006,
"<|sys|>": 32034,
"<|tag|>": 32004,
"<|user|>": 32010
}