PereLluis13's picture
Upload tokenizer
c2c4184 verified
{
"--NME--": 128001,
"[E-0]": 128002,
"[E-10]": 128012,
"[E-11]": 128013,
"[E-12]": 128014,
"[E-13]": 128015,
"[E-14]": 128016,
"[E-15]": 128017,
"[E-16]": 128018,
"[E-17]": 128019,
"[E-18]": 128020,
"[E-19]": 128021,
"[E-1]": 128003,
"[E-20]": 128022,
"[E-21]": 128023,
"[E-22]": 128024,
"[E-23]": 128025,
"[E-24]": 128026,
"[E-25]": 128027,
"[E-26]": 128028,
"[E-27]": 128029,
"[E-28]": 128030,
"[E-29]": 128031,
"[E-2]": 128004,
"[E-30]": 128032,
"[E-31]": 128033,
"[E-32]": 128034,
"[E-33]": 128035,
"[E-34]": 128036,
"[E-35]": 128037,
"[E-36]": 128038,
"[E-37]": 128039,
"[E-38]": 128040,
"[E-39]": 128041,
"[E-3]": 128005,
"[E-40]": 128042,
"[E-41]": 128043,
"[E-42]": 128044,
"[E-43]": 128045,
"[E-44]": 128046,
"[E-45]": 128047,
"[E-46]": 128048,
"[E-47]": 128049,
"[E-48]": 128050,
"[E-49]": 128051,
"[E-4]": 128006,
"[E-50]": 128052,
"[E-51]": 128053,
"[E-52]": 128054,
"[E-53]": 128055,
"[E-54]": 128056,
"[E-55]": 128057,
"[E-56]": 128058,
"[E-57]": 128059,
"[E-58]": 128060,
"[E-59]": 128061,
"[E-5]": 128007,
"[E-60]": 128062,
"[E-61]": 128063,
"[E-62]": 128064,
"[E-63]": 128065,
"[E-64]": 128066,
"[E-65]": 128067,
"[E-66]": 128068,
"[E-67]": 128069,
"[E-68]": 128070,
"[E-69]": 128071,
"[E-6]": 128008,
"[E-70]": 128072,
"[E-71]": 128073,
"[E-72]": 128074,
"[E-73]": 128075,
"[E-74]": 128076,
"[E-75]": 128077,
"[E-76]": 128078,
"[E-77]": 128079,
"[E-78]": 128080,
"[E-79]": 128081,
"[E-7]": 128009,
"[E-80]": 128082,
"[E-81]": 128083,
"[E-82]": 128084,
"[E-83]": 128085,
"[E-84]": 128086,
"[E-85]": 128087,
"[E-86]": 128088,
"[E-87]": 128089,
"[E-88]": 128090,
"[E-89]": 128091,
"[E-8]": 128010,
"[E-90]": 128092,
"[E-91]": 128093,
"[E-92]": 128094,
"[E-93]": 128095,
"[E-94]": 128096,
"[E-95]": 128097,
"[E-96]": 128098,
"[E-97]": 128099,
"[E-98]": 128100,
"[E-99]": 128101,
"[E-9]": 128011,
"[MASK]": 128000
}