squareV2 / added_tokens.json
jla25's picture
Upload tokenizer
733355c verified
{
"__af__": 128004,
"__am__": 128005,
"__ar__": 128006,
"__ast__": 128007,
"__az__": 128008,
"__ba__": 128009,
"__be__": 128010,
"__bg__": 128011,
"__bn__": 128012,
"__br__": 128013,
"__bs__": 128014,
"__ca__": 128015,
"__ceb__": 128016,
"__cs__": 128017,
"__cy__": 128018,
"__da__": 128019,
"__de__": 128020,
"__el__": 128021,
"__en__": 128022,
"__es__": 128023,
"__et__": 128024,
"__fa__": 128025,
"__ff__": 128026,
"__fi__": 128027,
"__fr__": 128028,
"__fy__": 128029,
"__ga__": 128030,
"__gd__": 128031,
"__gl__": 128032,
"__gu__": 128033,
"__ha__": 128034,
"__he__": 128035,
"__hi__": 128036,
"__hr__": 128037,
"__ht__": 128038,
"__hu__": 128039,
"__hy__": 128040,
"__id__": 128041,
"__ig__": 128042,
"__ilo__": 128043,
"__is__": 128044,
"__it__": 128045,
"__ja__": 128046,
"__jv__": 128047,
"__ka__": 128048,
"__kk__": 128049,
"__km__": 128050,
"__kn__": 128051,
"__ko__": 128052,
"__lb__": 128053,
"__lg__": 128054,
"__ln__": 128055,
"__lo__": 128056,
"__lt__": 128057,
"__lv__": 128058,
"__mg__": 128059,
"__mk__": 128060,
"__ml__": 128061,
"__mn__": 128062,
"__mr__": 128063,
"__ms__": 128064,
"__my__": 128065,
"__ne__": 128066,
"__nl__": 128067,
"__no__": 128068,
"__ns__": 128069,
"__oc__": 128070,
"__or__": 128071,
"__pa__": 128072,
"__pl__": 128073,
"__ps__": 128074,
"__pt__": 128075,
"__ro__": 128076,
"__ru__": 128077,
"__sd__": 128078,
"__si__": 128079,
"__sk__": 128080,
"__sl__": 128081,
"__so__": 128082,
"__sq__": 128083,
"__sr__": 128084,
"__ss__": 128085,
"__su__": 128086,
"__sv__": 128087,
"__sw__": 128088,
"__ta__": 128089,
"__th__": 128090,
"__tl__": 128091,
"__tn__": 128092,
"__tr__": 128093,
"__uk__": 128094,
"__ur__": 128095,
"__uz__": 128096,
"__vi__": 128097,
"__wo__": 128098,
"__xh__": 128099,
"__yi__": 128100,
"__yo__": 128101,
"__zh__": 128102,
"__zu__": 128103
}