tavbert-he / tokenizer.json
omrikeren's picture
Added files
41265b0
raw
history blame
No virus
4.3 kB
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"String":""},"behavior":"Isolated","invert":false},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[0],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[CLS]":0,"[PAD]":1,"[SEP]":2,"[UNK]":3,"unused0":4,"unused1":5,"unused2":6,"unused3":7,"unused4":8,"unused5":9,"unused6":10,"unused7":11,"unused8":12,"unused9":13,"unused10":14,"unused11":15,"unused12":16,"unused13":17,"unused14":18,"unused15":19,"unused16":20,"unused17":21,"unused18":22,"unused19":23,"unused20":24,"unused21":25,"unused22":26,"unused23":27,"unused24":28,"unused25":29,"unused26":30,"unused27":31,"unused28":32,"unused29":33,"unused30":34,"unused31":35,"unused32":36,"unused33":37,"unused34":38,"unused35":39,"unused36":40,"unused37":41,"unused38":42,"unused39":43,"unused40":44,"unused41":45,"unused42":46,"unused43":47,"unused44":48,"unused45":49,"unused46":50,"unused47":51,"unused48":52,"unused49":53,"unused50":54,"unused51":55,"unused52":56,"unused53":57,"unused54":58,"unused55":59,"unused56":60,"unused57":61,"unused58":62,"unused59":63,"unused60":64,"unused61":65,"unused62":66,"unused63":67,"unused64":68,"unused65":69,"unused66":70,"unused67":71,"unused68":72,"unused69":73,"unused70":74,"unused71":75,"unused72":76,"unused73":77,"unused74":78,"unused75":79,"unused76":80,"unused77":81,"unused78":82,"unused79":83,"unused80":84,"unused81":85,"unused82":86,"unused83":87,"unused84":88,"unused85":89,"unused86":90,"unused87":91,"unused88":92,"unused89":93,"unused90":94,"unused91":95,"unused92":96,"unused93":97,"unused94":98,"unused95":99,"unused96":100,"unused97":101,"unused98":102,"unused99":103," ":104,"!":105,"\"":106,"#":107,"$":108,"%":109,"&":110,"'":111,"(":112,")":113,"*":114,"+":115,",":116,"-":117,".":118,"/":119,"0":120,"1":121,"2":122,"3":123,"4":124,"5":125,"6":126,"7":127,"8":128,"9":129,":":130,";":131,"<":132,"=":133,">":134,"?":135,"@":136,"A":137,"B":138,"C":139,"D":140,"E":141,"F":142,"G":143,"H":144,"I":145,"J":146,"K":147,"L":148,"M":149,"N":150,"O":151,"P":152,"Q":153,"R":154,"S":155,"T":156,"U":157,"V":158,"W":159,"X":160,"Y":161,"Z":162,"[":163,"\\":164,"]":165,"^":166,"_":167,"`":168,"a":169,"b":170,"c":171,"d":172,"e":173,"f":174,"g":175,"h":176,"i":177,"j":178,"k":179,"l":180,"m":181,"n":182,"o":183,"p":184,"q":185,"r":186,"s":187,"t":188,"u":189,"v":190,"w":191,"x":192,"y":193,"z":194,"{":195,"|":196,"}":197,"¢":198,"¨":199,"©":200,"°":201,"³":202,"´":203,"·":204,"»":205,"×":206,"é":207,"А":208,"Б":209,"В":210,"Г":211,"Д":212,"И":213,"К":214,"М":215,"Н":216,"О":217,"П":218,"Р":219,"С":220,"Т":221,"Х":222,"Ш":223,"Э":224,"Я":225,"а":226,"б":227,"в":228,"г":229,"д":230,"е":231,"ж":232,"з":233,"и":234,"й":235,"к":236,"л":237,"м":238,"н":239,"о":240,"п":241,"р":242,"с":243,"т":244,"у":245,"ф":246,"х":247,"ц":248,"ч":249,"ш":250,"щ":251,"ы":252,"ь":253,"э":254,"ю":255,"я":256,"֑":257,"֔":258,"֖":259,"֗":260,"֙":261,"֛":262,"֣":263,"֤":264,"֥":265,"֨":266,"ְ":267,"ֱ":268,"ֲ":269,"ֳ":270,"ִ":271,"ֵ":272,"ֶ":273,"ַ":274,"ָ":275,"ֹ":276,"ֻ":277,"ּ":278,"ֽ":279,"־":280,"ׁ":281,"ׂ":282,"׃":283,"א":284,"ב":285,"ג":286,"ד":287,"ה":288,"ו":289,"ז":290,"ח":291,"ט":292,"י":293,"ך":294,"כ":295,"ל":296,"ם":297,"מ":298,"ן":299,"נ":300,"ס":301,"ע":302,"ף":303,"פ":304,"ץ":305,"צ":306,"ק":307,"ר":308,"ש":309,"ת":310,"׳":311,"״":312,"ا":313,"ب":314,"ة":315,"ت":316,"ح":317,"د":318,"ر":319,"س":320,"ع":321,"ف":322,"ق":323,"ك":324,"ل":325,"م":326,"ن":327,"و":328,"ي":329,"–":330,"—":331,"‘":332,"’":333,"“":334,"”":335,"•":336,"…":337,"₪":338,"€":339,"™":340,"←":341,"■":342,"�":343,"[MASK]":344},"unk_token":"[UNK]"}}