Upload tokenizer
6b10695
verified
|
{ |
|
"<af>": 64002, |
|
"<am>": 64003, |
|
"<ar>": 64004, |
|
"<ast>": 64005, |
|
"<be>": 64006, |
|
"<bg>": 64007, |
|
"<bn>": 64008, |
|
"<bs>": 64009, |
|
"<ca>": 64010, |
|
"<cs>": 64011, |
|
"<da>": 64012, |
|
"<de>": 64013, |
|
"<en>": 64014, |
|
"<eos>": 64001, |
|
"<es>": 64015, |
|
"<fr>": 64016, |
|
"<gu>": 64017, |
|
"<ha>": 64018, |
|
"<he>": 64019, |
|
"<hi>": 64020, |
|
"<is>": 64021, |
|
"<it>": 64022, |
|
"<kab>": 64023, |
|
"<kn>": 64024, |
|
"<lb>": 64025, |
|
"<mr>": 64026, |
|
"<mt>": 64027, |
|
"<ne>": 64028, |
|
"<nl>": 64029, |
|
"<no>": 64030, |
|
"<oc>": 64031, |
|
"<pl>": 64032, |
|
"<pt>": 64033, |
|
"<ro>": 64034, |
|
"<ru>": 64035, |
|
"<sd>": 64036, |
|
"<so>": 64037, |
|
"<sr>": 64038, |
|
"<sv>": 64039, |
|
"<ti>": 64040, |
|
"<uk>": 64041, |
|
"<ur>": 64042 |
|
} |
|
|