Upload tokenizer
066931b
verified
|
{ |
|
"__af__": 256001, |
|
"__am__": 256002, |
|
"__ar__": 256003, |
|
"__as__": 256004, |
|
"__ast__": 256005, |
|
"__ay__": 256006, |
|
"__az__": 256007, |
|
"__ba__": 256008, |
|
"__be__": 256009, |
|
"__bg__": 256010, |
|
"__bn__": 256011, |
|
"__br__": 256012, |
|
"__bs__": 256013, |
|
"__ca__": 256014, |
|
"__ceb__": 256015, |
|
"__cjk__": 256016, |
|
"__cs__": 256017, |
|
"__cy__": 256018, |
|
"__da__": 256019, |
|
"__de__": 256020, |
|
"__dyu__": 256021, |
|
"__el__": 256022, |
|
"__en__": 256023, |
|
"__es__": 256024, |
|
"__et__": 256025, |
|
"__fa__": 256026, |
|
"__ff__": 256027, |
|
"__fi__": 256028, |
|
"__fr__": 256029, |
|
"__fy__": 256030, |
|
"__ga__": 256031, |
|
"__gd__": 256032, |
|
"__gl__": 256033, |
|
"__gu__": 256034, |
|
"__ha__": 256035, |
|
"__he__": 256036, |
|
"__hi__": 256037, |
|
"__hr__": 256038, |
|
"__ht__": 256039, |
|
"__hu__": 256040, |
|
"__hy__": 256041, |
|
"__id__": 256042, |
|
"__ig__": 256043, |
|
"__ilo__": 256044, |
|
"__is__": 256045, |
|
"__it__": 256046, |
|
"__ja__": 256047, |
|
"__jv__": 256048, |
|
"__ka__": 256049, |
|
"__kac__": 256050, |
|
"__kam__": 256051, |
|
"__kea__": 256052, |
|
"__kg__": 256053, |
|
"__kk__": 256054, |
|
"__km__": 256055, |
|
"__kmb__": 256056, |
|
"__kmr__": 256057, |
|
"__kn__": 256058, |
|
"__ko__": 256059, |
|
"__ku__": 256060, |
|
"__ky__": 256061, |
|
"__lb__": 256062, |
|
"__lg__": 256063, |
|
"__ln__": 256064, |
|
"__lo__": 256065, |
|
"__lt__": 256066, |
|
"__luo__": 256067, |
|
"__lv__": 256068, |
|
"__mg__": 256069, |
|
"__mi__": 256070, |
|
"__mk__": 256071, |
|
"__ml__": 256072, |
|
"__mn__": 256073, |
|
"__mr__": 256074, |
|
"__ms__": 256075, |
|
"__mt__": 256076, |
|
"__my__": 256077, |
|
"__ne__": 256078, |
|
"__nl__": 256079, |
|
"__no__": 256080, |
|
"__ns__": 256081, |
|
"__ny__": 256082, |
|
"__oc__": 256083, |
|
"__om__": 256084, |
|
"__or__": 256085, |
|
"__pa__": 256086, |
|
"__pl__": 256087, |
|
"__ps__": 256088, |
|
"__pt__": 256089, |
|
"__qu__": 256090, |
|
"__ro__": 256091, |
|
"__ru__": 256092, |
|
"__sd__": 256093, |
|
"__shn__": 256094, |
|
"__si__": 256095, |
|
"__sk__": 256096, |
|
"__sl__": 256097, |
|
"__sn__": 256098, |
|
"__so__": 256099, |
|
"__sq__": 256100, |
|
"__sr__": 256101, |
|
"__ss__": 256102, |
|
"__su__": 256103, |
|
"__sv__": 256104, |
|
"__sw__": 256105, |
|
"__ta__": 256106, |
|
"__te__": 256107, |
|
"__tg__": 256108, |
|
"__th__": 256109, |
|
"__ti__": 256110, |
|
"__tl__": 256111, |
|
"__tn__": 256112, |
|
"__tr__": 256113, |
|
"__uk__": 256114, |
|
"__umb__": 256115, |
|
"__ur__": 256116, |
|
"__uz__": 256117, |
|
"__vi__": 256118, |
|
"__wo__": 256119, |
|
"__xh__": 256120, |
|
"__yi__": 256121, |
|
"__yo__": 256122, |
|
"__zh__": 256123, |
|
"__zu__": 256124 |
|
} |
|
|