ec40-unigram / added_tokens.json
speedcell4's picture
Upload tokenizer
6b10695 verified
raw
history blame
720 Bytes
{
"<af>": 64002,
"<am>": 64003,
"<ar>": 64004,
"<ast>": 64005,
"<be>": 64006,
"<bg>": 64007,
"<bn>": 64008,
"<bs>": 64009,
"<ca>": 64010,
"<cs>": 64011,
"<da>": 64012,
"<de>": 64013,
"<en>": 64014,
"<eos>": 64001,
"<es>": 64015,
"<fr>": 64016,
"<gu>": 64017,
"<ha>": 64018,
"<he>": 64019,
"<hi>": 64020,
"<is>": 64021,
"<it>": 64022,
"<kab>": 64023,
"<kn>": 64024,
"<lb>": 64025,
"<mr>": 64026,
"<mt>": 64027,
"<ne>": 64028,
"<nl>": 64029,
"<no>": 64030,
"<oc>": 64031,
"<pl>": 64032,
"<pt>": 64033,
"<ro>": 64034,
"<ru>": 64035,
"<sd>": 64036,
"<so>": 64037,
"<sr>": 64038,
"<sv>": 64039,
"<ti>": 64040,
"<uk>": 64041,
"<ur>": 64042
}