Upload tokenizer
c2c4184
verified
|
{ |
|
"--NME--": 128001, |
|
"[E-0]": 128002, |
|
"[E-10]": 128012, |
|
"[E-11]": 128013, |
|
"[E-12]": 128014, |
|
"[E-13]": 128015, |
|
"[E-14]": 128016, |
|
"[E-15]": 128017, |
|
"[E-16]": 128018, |
|
"[E-17]": 128019, |
|
"[E-18]": 128020, |
|
"[E-19]": 128021, |
|
"[E-1]": 128003, |
|
"[E-20]": 128022, |
|
"[E-21]": 128023, |
|
"[E-22]": 128024, |
|
"[E-23]": 128025, |
|
"[E-24]": 128026, |
|
"[E-25]": 128027, |
|
"[E-26]": 128028, |
|
"[E-27]": 128029, |
|
"[E-28]": 128030, |
|
"[E-29]": 128031, |
|
"[E-2]": 128004, |
|
"[E-30]": 128032, |
|
"[E-31]": 128033, |
|
"[E-32]": 128034, |
|
"[E-33]": 128035, |
|
"[E-34]": 128036, |
|
"[E-35]": 128037, |
|
"[E-36]": 128038, |
|
"[E-37]": 128039, |
|
"[E-38]": 128040, |
|
"[E-39]": 128041, |
|
"[E-3]": 128005, |
|
"[E-40]": 128042, |
|
"[E-41]": 128043, |
|
"[E-42]": 128044, |
|
"[E-43]": 128045, |
|
"[E-44]": 128046, |
|
"[E-45]": 128047, |
|
"[E-46]": 128048, |
|
"[E-47]": 128049, |
|
"[E-48]": 128050, |
|
"[E-49]": 128051, |
|
"[E-4]": 128006, |
|
"[E-50]": 128052, |
|
"[E-51]": 128053, |
|
"[E-52]": 128054, |
|
"[E-53]": 128055, |
|
"[E-54]": 128056, |
|
"[E-55]": 128057, |
|
"[E-56]": 128058, |
|
"[E-57]": 128059, |
|
"[E-58]": 128060, |
|
"[E-59]": 128061, |
|
"[E-5]": 128007, |
|
"[E-60]": 128062, |
|
"[E-61]": 128063, |
|
"[E-62]": 128064, |
|
"[E-63]": 128065, |
|
"[E-64]": 128066, |
|
"[E-65]": 128067, |
|
"[E-66]": 128068, |
|
"[E-67]": 128069, |
|
"[E-68]": 128070, |
|
"[E-69]": 128071, |
|
"[E-6]": 128008, |
|
"[E-70]": 128072, |
|
"[E-71]": 128073, |
|
"[E-72]": 128074, |
|
"[E-73]": 128075, |
|
"[E-74]": 128076, |
|
"[E-75]": 128077, |
|
"[E-76]": 128078, |
|
"[E-77]": 128079, |
|
"[E-78]": 128080, |
|
"[E-79]": 128081, |
|
"[E-7]": 128009, |
|
"[E-80]": 128082, |
|
"[E-81]": 128083, |
|
"[E-82]": 128084, |
|
"[E-83]": 128085, |
|
"[E-84]": 128086, |
|
"[E-85]": 128087, |
|
"[E-86]": 128088, |
|
"[E-87]": 128089, |
|
"[E-88]": 128090, |
|
"[E-89]": 128091, |
|
"[E-8]": 128010, |
|
"[E-90]": 128092, |
|
"[E-91]": 128093, |
|
"[E-92]": 128094, |
|
"[E-93]": 128095, |
|
"[E-94]": 128096, |
|
"[E-95]": 128097, |
|
"[E-96]": 128098, |
|
"[E-97]": 128099, |
|
"[E-98]": 128100, |
|
"[E-99]": 128101, |
|
"[E-9]": 128011, |
|
"[MASK]": 128000 |
|
} |
|
|