tpi_latn_5mb / added_tokens.json
goldfish-models's picture
Upload tpi_latn_5mb tokenizer.
9a3a666
{"[XXXXX86]": 40691, "[XXXXX7]": 40612, "[XXXXX44]": 40649, "[XXXXX265]": 40870, "[XXXXX84]": 40689, "[XXXXX247]": 40852, "[XXXXX233]": 40838, "[XXXXX21]": 40626, "[XXXXX286]": 40891, "[XXXXX139]": 40744, "[XXXXX159]": 40764, "[XXXXX333]": 40938, "[XXXXX97]": 40702, "[XXXXX45]": 40650, "[XXXXX1]": 40606, "[XXXXX306]": 40911, "[XXXXX352]": 40957, "[XXXXX239]": 40844, "[XXXXX269]": 40874, "[XXXXX254]": 40859, "[XXXXX263]": 40868, "[XXXXX2]": 40607, "[XXXXX27]": 40632, "[XXXXX88]": 40693, "[XXXXX143]": 40748, "[CLS]": 40601, "[XXXXX330]": 40935, "[XXXXX231]": 40836, "[XXXXX119]": 40724, "[XXXXX180]": 40785, "[XXXXX32]": 40637, "[XXXXX153]": 40758, "[XXXXX297]": 40902, "[XXXXX353]": 40958, "[XXXXX298]": 40903, "[XXXXX273]": 40878, "[XXXXX38]": 40643, "[XXXXX149]": 40754, "[XXXXX278]": 40883, "<pad>": 40603, "[XXXXX310]": 40915, "[XXXXX94]": 40699, "[XXXXX217]": 40822, "[XXXXX325]": 40930, "[XXXXX245]": 40850, "[XXXXX329]": 40934, "[XXXXX206]": 40811, "[XXXXX146]": 40751, "[XXXXX72]": 40677, "[XXXXX70]": 40675, "[XXXXX15]": 40620, "[XXXXX125]": 40730, "[XXXXX66]": 40671, "[XXXXX114]": 40719, "[XXXXX277]": 40882, "[XXXXX171]": 40776, "[XXXXX22]": 40627, "[XXXXX225]": 40830, "[XXXXX281]": 40886, "[XXXXX337]": 40942, "[XXXXX47]": 40652, "[XXXXX164]": 40769, "[XXXXX222]": 40827, "[XXXXX147]": 40752, "[XXXXX82]": 40687, "[XXXXX313]": 40918, "[XXXXX170]": 40775, "[XXXXX8]": 40613, "[XXXXX271]": 40876, "[XXXXX3]": 40608, "[XXXXX148]": 40753, "[XXXXX5]": 40610, "[XXXXX336]": 40941, "[XXXXX203]": 40808, "[XXXXX207]": 40812, "[XXXXX218]": 40823, "[XXXXX61]": 40666, "[XXXXX340]": 40945, "[XXXXX288]": 40893, "[XXXXX184]": 40789, "[XXXXX0]": 40605, "[XXXXX261]": 40866, "[XXXXX255]": 40860, "[XXXXX35]": 40640, "[XXXXX106]": 40711, "[XXXXX341]": 40946, "[XXXXX187]": 40792, "[XXXXX168]": 40773, "[XXXXX227]": 40832, "[XXXXX122]": 40727, "[XXXXX270]": 40875, "[XXXXX323]": 40928, "[XXXXX57]": 40662, "[XXXXX101]": 40706, "[XXXXX295]": 40900, "[XXXXX48]": 40653, "[XXXXX178]": 40783, "[XXXXX136]": 40741, "[XXXXX25]": 40630, "[XXXXX60]": 40665, "[XXXXX129]": 40734, "[XXXXX17]": 40622, "[XXXXX18]": 40623, "[XXXXX166]": 40771, "[XXXXX287]": 40892, "[XXXXX118]": 40723, "[XXXXX324]": 40929, "[XXXXX320]": 40925, "[XXXXX177]": 40782, "[XXXXX68]": 40673, "[XXXXX219]": 40824, "[XXXXX326]": 40931, "[XXXXX59]": 40664, "[XXXXX79]": 40684, "[XXXXX55]": 40660, "[XXXXX93]": 40698, "[XXXXX144]": 40749, "[XXXXX317]": 40922, "[XXXXX63]": 40668, "[XXXXX283]": 40888, "[XXXXX30]": 40635, "[XXXXX235]": 40840, "[XXXXX158]": 40763, "[XXXXX220]": 40825, "[XXXXX221]": 40826, "[XXXXX321]": 40926, "[XXXXX284]": 40889, "[XXXXX127]": 40732, "[XXXXX54]": 40659, "[XXXXX304]": 40909, "[XXXXX115]": 40720, "[XXXXX175]": 40780, "[XXXXX208]": 40813, "[XXXXX98]": 40703, "[XXXXX240]": 40845, "[XXXXX96]": 40701, "[XXXXX290]": 40895, "[XXXXX309]": 40914, "[XXXXX267]": 40872, "[XXXXX338]": 40943, "[MASK]": 40604, "[XXXXX350]": 40955, "[XXXXX351]": 40956, "[XXXXX42]": 40647, "[XXXXX138]": 40743, "[XXXXX280]": 40885, "[XXXXX300]": 40905, "[XXXXX215]": 40820, "[XXXXX19]": 40624, "[XXXXX161]": 40766, "[XXXXX89]": 40694, "[XXXXX76]": 40681, "[XXXXX162]": 40767, "[XXXXX132]": 40737, "[XXXXX37]": 40642, "[XXXXX251]": 40856, "[XXXXX141]": 40746, "[XXXXX28]": 40633, "[XXXXX103]": 40708, "[XXXXX49]": 40654, "[XXXXX318]": 40923, "[XXXXX56]": 40661, "[XXXXX250]": 40855, "[XXXXX123]": 40728, "[XXXXX305]": 40910, "[XXXXX163]": 40768, "[XXXXX91]": 40696, "[XXXXX110]": 40715, "[XXXXX296]": 40901, "[XXXXX124]": 40729, "[XXXXX193]": 40798, "[XXXXX335]": 40940, "[XXXXX216]": 40821, "[XXXXX64]": 40669, "[XXXXX160]": 40765, "[XXXXX169]": 40774, "[XXXXX259]": 40864, "[XXXXX289]": 40894, "[XXXXX249]": 40854, "[XXXXX348]": 40953, "[XXXXX43]": 40648, "[XXXXX189]": 40794, "[XXXXX211]": 40816, "[XXXXX322]": 40927, "[XXXXX62]": 40667, "[XXXXX10]": 40615, "[XXXXX67]": 40672, "[XXXXX312]": 40917, "[XXXXX228]": 40833, "[XXXXX111]": 40716, "[XXXXX12]": 40617, "[XXXXX230]": 40835, "[XXXXX186]": 40791, "[XXXXX116]": 40721, "[XXXXX343]": 40948, "[XXXXX134]": 40739, "[XXXXX346]": 40951, "[XXXXX308]": 40913, "[XXXXX182]": 40787, "[XXXXX223]": 40828, "[XXXXX205]": 40810, "[XXXXX268]": 40873, "[XXXXX107]": 40712, "[XXXXX301]": 40906, "[XXXXX24]": 40629, "[XXXXX185]": 40790, "[XXXXX274]": 40879, "[XXXXX199]": 40804, "[XXXXX191]": 40796, "[XXXXX87]": 40692, "[XXXXX73]": 40678, "[XXXXX142]": 40747, "[XXXXX179]": 40784, "[XXXXX26]": 40631, "[XXXXX264]": 40869, "[XXXXX266]": 40871, "[XXXXX197]": 40802, "[XXXXX156]": 40761, "[XXXXX34]": 40639, "[XXXXX39]": 40644, "[XXXXX327]": 40932, "[XXXXX334]": 40939, "[XXXXX224]": 40829, "[XXXXX234]": 40839, "[XXXXX99]": 40704, "[XXXXX113]": 40718, "[XXXXX14]": 40619, "[XXXXX302]": 40907, "[XXXXX16]": 40621, "[XXXXX85]": 40690, "[XXXXX257]": 40862, "[XXXXX354]": 40959, "[XXXXX238]": 40843, "[XXXXX150]": 40755, "[XXXXX112]": 40717, "[XXXXX244]": 40849, "[XXXXX276]": 40881, "[XXXXX74]": 40679, "[XXXXX314]": 40919, "[XXXXX316]": 40921, "[XXXXX293]": 40898, "[XXXXX121]": 40726, "[XXXXX155]": 40760, "[XXXXX4]": 40609, "[XXXXX133]": 40738, "[XXXXX131]": 40736, "[XXXXX31]": 40636, "[XXXXX252]": 40857, "[XXXXX165]": 40770, "[XXXXX51]": 40656, "[XXXXX109]": 40714, "[XXXXX20]": 40625, "[XXXXX272]": 40877, "[XXXXX344]": 40949, "[XXXXX92]": 40697, "[XXXXX291]": 40896, "[XXXXX345]": 40950, "[XXXXX174]": 40779, "[XXXXX151]": 40756, "[XXXXX90]": 40695, "[XXXXX33]": 40638, "[XXXXX258]": 40863, "[XXXXX140]": 40745, "[XXXXX183]": 40788, "[XXXXX210]": 40815, "[XXXXX236]": 40841, "[XXXXX120]": 40725, "[XXXXX53]": 40658, "[XXXXX226]": 40831, "[XXXXX292]": 40897, "[XXXXX77]": 40682, "[XXXXX190]": 40795, "[XXXXX81]": 40686, "[XXXXX181]": 40786, "[XXXXX196]": 40801, "[SEP]": 40602, "[XXXXX260]": 40865, "[XXXXX11]": 40616, "[XXXXX117]": 40722, "[XXXXX202]": 40807, "[XXXXX71]": 40676, "[XXXXX95]": 40700, "[XXXXX192]": 40797, "[XXXXX209]": 40814, "[XXXXX36]": 40641, "[XXXXX242]": 40847, "[XXXXX137]": 40742, "[XXXXX58]": 40663, "[XXXXX256]": 40861, "[XXXXX294]": 40899, "[XXXXX83]": 40688, "[XXXXX157]": 40762, "[XXXXX331]": 40936, "[XXXXX349]": 40954, "[XXXXX145]": 40750, "[XXXXX46]": 40651, "[XXXXX246]": 40851, "[XXXXX41]": 40646, "[XXXXX232]": 40837, "[XXXXX248]": 40853, "[XXXXX105]": 40710, "[XXXXX69]": 40674, "[XXXXX195]": 40800, "[XXXXX285]": 40890, "[XXXXX75]": 40680, "[XXXXX78]": 40683, "[XXXXX328]": 40933, "[XXXXX342]": 40947, "[XXXXX200]": 40805, "[XXXXX332]": 40937, "[XXXXX201]": 40806, "[XXXXX108]": 40713, "[XXXXX104]": 40709, "[XXXXX172]": 40777, "[XXXXX65]": 40670, "[XXXXX23]": 40628, "[XXXXX275]": 40880, "[XXXXX213]": 40818, "[XXXXX135]": 40740, "[XXXXX253]": 40858, "[XXXXX173]": 40778, "[XXXXX154]": 40759, "[XXXXX315]": 40920, "[XXXXX198]": 40803, "[XXXXX303]": 40908, "[XXXXX52]": 40657, "[XXXXX347]": 40952, "[XXXXX100]": 40705, "[XXXXX299]": 40904, "[XXXXX319]": 40924, "[XXXXX80]": 40685, "[XXXXX229]": 40834, "[XXXXX311]": 40916, "[XXXXX188]": 40793, "[XXXXX152]": 40757, "[XXXXX9]": 40614, "[XXXXX204]": 40809, "[XXXXX279]": 40884, "[XXXXX40]": 40645, "[XXXXX126]": 40731, "[XXXXX339]": 40944, "[XXXXX243]": 40848, "[XXXXX50]": 40655, "[XXXXX128]": 40733, "[XXXXX176]": 40781, "[XXXXX29]": 40634, "[XXXXX282]": 40887, "[XXXXX212]": 40817, "[XXXXX262]": 40867, "[XXXXX194]": 40799, "[XXXXX214]": 40819, "[XXXXX13]": 40618, "[XXXXX102]": 40707, "[XXXXX241]": 40846, "[XXXXX167]": 40772, "[XXXXX6]": 40611, "[XXXXX237]": 40842, "[XXXXX130]": 40735, "[XXXXX307]": 40912}