flores101_mm100_175M / added_tokens.json
VityaVitalich's picture
Upload tokenizer
066931b verified
{
"__af__": 256001,
"__am__": 256002,
"__ar__": 256003,
"__as__": 256004,
"__ast__": 256005,
"__ay__": 256006,
"__az__": 256007,
"__ba__": 256008,
"__be__": 256009,
"__bg__": 256010,
"__bn__": 256011,
"__br__": 256012,
"__bs__": 256013,
"__ca__": 256014,
"__ceb__": 256015,
"__cjk__": 256016,
"__cs__": 256017,
"__cy__": 256018,
"__da__": 256019,
"__de__": 256020,
"__dyu__": 256021,
"__el__": 256022,
"__en__": 256023,
"__es__": 256024,
"__et__": 256025,
"__fa__": 256026,
"__ff__": 256027,
"__fi__": 256028,
"__fr__": 256029,
"__fy__": 256030,
"__ga__": 256031,
"__gd__": 256032,
"__gl__": 256033,
"__gu__": 256034,
"__ha__": 256035,
"__he__": 256036,
"__hi__": 256037,
"__hr__": 256038,
"__ht__": 256039,
"__hu__": 256040,
"__hy__": 256041,
"__id__": 256042,
"__ig__": 256043,
"__ilo__": 256044,
"__is__": 256045,
"__it__": 256046,
"__ja__": 256047,
"__jv__": 256048,
"__ka__": 256049,
"__kac__": 256050,
"__kam__": 256051,
"__kea__": 256052,
"__kg__": 256053,
"__kk__": 256054,
"__km__": 256055,
"__kmb__": 256056,
"__kmr__": 256057,
"__kn__": 256058,
"__ko__": 256059,
"__ku__": 256060,
"__ky__": 256061,
"__lb__": 256062,
"__lg__": 256063,
"__ln__": 256064,
"__lo__": 256065,
"__lt__": 256066,
"__luo__": 256067,
"__lv__": 256068,
"__mg__": 256069,
"__mi__": 256070,
"__mk__": 256071,
"__ml__": 256072,
"__mn__": 256073,
"__mr__": 256074,
"__ms__": 256075,
"__mt__": 256076,
"__my__": 256077,
"__ne__": 256078,
"__nl__": 256079,
"__no__": 256080,
"__ns__": 256081,
"__ny__": 256082,
"__oc__": 256083,
"__om__": 256084,
"__or__": 256085,
"__pa__": 256086,
"__pl__": 256087,
"__ps__": 256088,
"__pt__": 256089,
"__qu__": 256090,
"__ro__": 256091,
"__ru__": 256092,
"__sd__": 256093,
"__shn__": 256094,
"__si__": 256095,
"__sk__": 256096,
"__sl__": 256097,
"__sn__": 256098,
"__so__": 256099,
"__sq__": 256100,
"__sr__": 256101,
"__ss__": 256102,
"__su__": 256103,
"__sv__": 256104,
"__sw__": 256105,
"__ta__": 256106,
"__te__": 256107,
"__tg__": 256108,
"__th__": 256109,
"__ti__": 256110,
"__tl__": 256111,
"__tn__": 256112,
"__tr__": 256113,
"__uk__": 256114,
"__umb__": 256115,
"__ur__": 256116,
"__uz__": 256117,
"__vi__": 256118,
"__wo__": 256119,
"__xh__": 256120,
"__yi__": 256121,
"__yo__": 256122,
"__zh__": 256123,
"__zu__": 256124
}