imgremlin's picture
Upload tokenizer
35137d9
raw
history blame
21.3 kB
{
"<bos>": 0,
"<pad>": 1,
"<eos>": 2,
"<unk>": 3,
".0": 4,
".1": 5,
".2": 6,
".3": 7,
".4": 8,
".5": 9,
".6": 10,
".7": 11,
".8": 12,
".9": 13,
"100": 14,
"1000": 15,
"1001": 16,
"1002": 17,
"1003": 18,
"1004": 19,
"1005": 20,
"1006": 21,
"1007": 22,
"1008": 23,
"1009": 24,
"101": 25,
"1011": 26,
"1012": 27,
"1013": 28,
"1014": 29,
"1015": 30,
"1016": 31,
"1017": 32,
"1018": 33,
"1019": 34,
"102": 35,
"1020": 36,
"1021": 37,
"1022": 38,
"1023": 39,
"1024": 40,
"1026": 41,
"1027": 42,
"1028": 43,
"1029": 44,
"103": 45,
"1030": 46,
"1031": 47,
"1032": 48,
"1033": 49,
"1034": 50,
"1035": 51,
"1036": 52,
"1037": 53,
"1038": 54,
"1039": 55,
"104": 56,
"1040": 57,
"1041": 58,
"1043": 59,
"1044": 60,
"1045": 61,
"1046": 62,
"1047": 63,
"1048": 64,
"1049": 65,
"105": 66,
"1050": 67,
"1051": 68,
"1052": 69,
"1053": 70,
"1054": 71,
"1055": 72,
"1057": 73,
"1058": 74,
"1059": 75,
"106": 76,
"1060": 77,
"1061": 78,
"1062": 79,
"1063": 80,
"1064": 81,
"1065": 82,
"1067": 83,
"1068": 84,
"1069": 85,
"107": 86,
"1070": 87,
"1071": 88,
"1072": 89,
"1073": 90,
"1074": 91,
"1075": 92,
"1076": 93,
"1077": 94,
"1078": 95,
"1079": 96,
"108": 97,
"1080": 98,
"1083": 99,
"1084": 100,
"1085": 101,
"1086": 102,
"1087": 103,
"1088": 104,
"1089": 105,
"109": 106,
"1090": 107,
"1091": 108,
"1092": 109,
"1093": 110,
"1094": 111,
"1095": 112,
"1096": 113,
"1098": 114,
"1099": 115,
"110": 116,
"1100": 117,
"1101": 118,
"1102": 119,
"1103": 120,
"1104": 121,
"1105": 122,
"1106": 123,
"1107": 124,
"1109": 125,
"111": 126,
"1111": 127,
"1112": 128,
"1113": 129,
"1114": 130,
"1115": 131,
"1117": 132,
"1118": 133,
"1119": 134,
"112": 135,
"1120": 136,
"1121": 137,
"1122": 138,
"1123": 139,
"1124": 140,
"1125": 141,
"1126": 142,
"1127": 143,
"1128": 144,
"113": 145,
"1130": 146,
"1131": 147,
"1134": 148,
"1135": 149,
"1136": 150,
"1137": 151,
"1139": 152,
"114": 153,
"1140": 154,
"1141": 155,
"1142": 156,
"1143": 157,
"1144": 158,
"1145": 159,
"1147": 160,
"1149": 161,
"115": 162,
"1150": 163,
"1151": 164,
"1152": 165,
"1153": 166,
"1154": 167,
"1155": 168,
"1156": 169,
"1159": 170,
"116": 171,
"1161": 172,
"1162": 173,
"1163": 174,
"1164": 175,
"1165": 176,
"1166": 177,
"1167": 178,
"117": 179,
"1173": 180,
"1175": 181,
"1176": 182,
"1177": 183,
"1178": 184,
"1179": 185,
"118": 186,
"1180": 187,
"1181": 188,
"1182": 189,
"1183": 190,
"1184": 191,
"1185": 192,
"1186": 193,
"1189": 194,
"119": 195,
"1191": 196,
"1192": 197,
"1193": 198,
"1198": 199,
"120": 200,
"1201": 201,
"1203": 202,
"1205": 203,
"1206": 204,
"1207": 205,
"1208": 206,
"1209": 207,
"121": 208,
"1210": 209,
"1211": 210,
"1212": 211,
"1215": 212,
"1216": 213,
"1217": 214,
"1218": 215,
"1219": 216,
"122": 217,
"1220": 218,
"1221": 219,
"1222": 220,
"1223": 221,
"1224": 222,
"1225": 223,
"1226": 224,
"1227": 225,
"1228": 226,
"123": 227,
"1230": 228,
"1231": 229,
"1232": 230,
"1233": 231,
"1234": 232,
"1236": 233,
"1237": 234,
"1238": 235,
"1239": 236,
"124": 237,
"1240": 238,
"1241": 239,
"1242": 240,
"1243": 241,
"1245": 242,
"1246": 243,
"1249": 244,
"125": 245,
"1250": 246,
"1251": 247,
"1252": 248,
"1255": 249,
"1257": 250,
"1258": 251,
"1259": 252,
"126": 253,
"1260": 254,
"1263": 255,
"1265": 256,
"1266": 257,
"1267": 258,
"1268": 259,
"1269": 260,
"127": 261,
"1270": 262,
"1271": 263,
"1275": 264,
"1276": 265,
"1277": 266,
"1278": 267,
"1279": 268,
"128": 269,
"1280": 270,
"1281": 271,
"1282": 272,
"1283": 273,
"1287": 274,
"1288": 275,
"1289": 276,
"129": 277,
"1290": 278,
"1293": 279,
"1294": 280,
"1296": 281,
"1297": 282,
"1298": 283,
"1299": 284,
"130": 285,
"1303": 286,
"1305": 287,
"1306": 288,
"1307": 289,
"1308": 290,
"1309": 291,
"131": 292,
"1310": 293,
"1311": 294,
"1314": 295,
"1315": 296,
"1317": 297,
"1318": 298,
"132": 299,
"1321": 300,
"1323": 301,
"1324": 302,
"1325": 303,
"1327": 304,
"133": 305,
"1332": 306,
"1333": 307,
"1339": 308,
"134": 309,
"1340": 310,
"1341": 311,
"1345": 312,
"1347": 313,
"135": 314,
"1353": 315,
"1356": 316,
"1357": 317,
"1358": 318,
"1359": 319,
"136": 320,
"1360": 321,
"1362": 322,
"1363": 323,
"1365": 324,
"1366": 325,
"1367": 326,
"1369": 327,
"137": 328,
"1370": 329,
"1371": 330,
"1372": 331,
"1377": 332,
"1378": 333,
"138": 334,
"1381": 335,
"1382": 336,
"1384": 337,
"1385": 338,
"1389": 339,
"139": 340,
"1390": 341,
"1391": 342,
"1393": 343,
"1394": 344,
"1395": 345,
"1396": 346,
"1399": 347,
"140": 348,
"1403": 349,
"1404": 350,
"1407": 351,
"1408": 352,
"1409": 353,
"141": 354,
"1411": 355,
"1412": 356,
"1413": 357,
"1419": 358,
"142": 359,
"1421": 360,
"1422": 361,
"1425": 362,
"1426": 363,
"1427": 364,
"1429": 365,
"143": 366,
"1430": 367,
"1431": 368,
"1432": 369,
"1437": 370,
"1438": 371,
"1439": 372,
"144": 373,
"1441": 374,
"1443": 375,
"1444": 376,
"1449": 377,
"145": 378,
"1450": 379,
"1455": 380,
"146": 381,
"1461": 382,
"1466": 383,
"1467": 384,
"147": 385,
"1471": 386,
"1472": 387,
"1479": 388,
"148": 389,
"1480": 390,
"1489": 391,
"149": 392,
"1490": 393,
"1491": 394,
"1492": 395,
"1496": 396,
"1497": 397,
"1498": 398,
"150": 399,
"151": 400,
"152": 401,
"1527": 402,
"1528": 403,
"153": 404,
"154": 405,
"1543": 406,
"1546": 407,
"155": 408,
"1553": 409,
"1559": 410,
"156": 411,
"1561": 412,
"1569": 413,
"157": 414,
"1574": 415,
"158": 416,
"159": 417,
"160": 418,
"161": 419,
"1611": 420,
"162": 421,
"1620": 422,
"1621": 423,
"163": 424,
"1633": 425,
"164": 426,
"165": 427,
"1653": 428,
"1659": 429,
"166": 430,
"167": 431,
"168": 432,
"169": 433,
"170": 434,
"171": 435,
"172": 436,
"173": 437,
"174": 438,
"175": 439,
"176": 440,
"177": 441,
"178": 442,
"179": 443,
"180": 444,
"181": 445,
"182": 446,
"183": 447,
"184": 448,
"185": 449,
"186": 450,
"187": 451,
"188": 452,
"189": 453,
"190": 454,
"191": 455,
"192": 456,
"193": 457,
"194": 458,
"195": 459,
"196": 460,
"197": 461,
"198": 462,
"199": 463,
"200": 464,
"201": 465,
"202": 466,
"203": 467,
"204": 468,
"205": 469,
"206": 470,
"207": 471,
"208": 472,
"209": 473,
"210": 474,
"211": 475,
"212": 476,
"213": 477,
"214": 478,
"215": 479,
"216": 480,
"217": 481,
"218": 482,
"219": 483,
"220": 484,
"221": 485,
"222": 486,
"223": 487,
"224": 488,
"225": 489,
"226": 490,
"227": 491,
"228": 492,
"229": 493,
"230": 494,
"231": 495,
"232": 496,
"233": 497,
"234": 498,
"235": 499,
"236": 500,
"237": 501,
"238": 502,
"239": 503,
"240": 504,
"241": 505,
"242": 506,
"243": 507,
"244": 508,
"245": 509,
"246": 510,
"247": 511,
"248": 512,
"249": 513,
"250": 514,
"251": 515,
"252": 516,
"253": 517,
"254": 518,
"255": 519,
"256": 520,
"257": 521,
"258": 522,
"259": 523,
"260": 524,
"261": 525,
"262": 526,
"263": 527,
"264": 528,
"265": 529,
"266": 530,
"267": 531,
"268": 532,
"269": 533,
"270": 534,
"271": 535,
"272": 536,
"273": 537,
"274": 538,
"275": 539,
"276": 540,
"277": 541,
"278": 542,
"279": 543,
"280": 544,
"281": 545,
"282": 546,
"283": 547,
"284": 548,
"285": 549,
"286": 550,
"287": 551,
"288": 552,
"289": 553,
"290": 554,
"291": 555,
"292": 556,
"293": 557,
"294": 558,
"295": 559,
"296": 560,
"297": 561,
"298": 562,
"299": 563,
"300": 564,
"301": 565,
"302": 566,
"303": 567,
"304": 568,
"305": 569,
"306": 570,
"307": 571,
"308": 572,
"309": 573,
"310": 574,
"311": 575,
"312": 576,
"313": 577,
"314": 578,
"315": 579,
"316": 580,
"317": 581,
"318": 582,
"319": 583,
"320": 584,
"321": 585,
"322": 586,
"323": 587,
"324": 588,
"325": 589,
"326": 590,
"327": 591,
"328": 592,
"329": 593,
"330": 594,
"331": 595,
"332": 596,
"333": 597,
"334": 598,
"335": 599,
"336": 600,
"337": 601,
"338": 602,
"339": 603,
"340": 604,
"341": 605,
"342": 606,
"343": 607,
"344": 608,
"345": 609,
"346": 610,
"347": 611,
"348": 612,
"349": 613,
"350": 614,
"351": 615,
"352": 616,
"353": 617,
"354": 618,
"355": 619,
"356": 620,
"357": 621,
"358": 622,
"359": 623,
"360": 624,
"361": 625,
"362": 626,
"363": 627,
"364": 628,
"365": 629,
"366": 630,
"367": 631,
"368": 632,
"369": 633,
"370": 634,
"371": 635,
"372": 636,
"373": 637,
"374": 638,
"375": 639,
"376": 640,
"377": 641,
"378": 642,
"379": 643,
"380": 644,
"381": 645,
"382": 646,
"383": 647,
"384": 648,
"385": 649,
"386": 650,
"387": 651,
"388": 652,
"389": 653,
"390": 654,
"391": 655,
"392": 656,
"393": 657,
"394": 658,
"395": 659,
"396": 660,
"397": 661,
"398": 662,
"399": 663,
"400": 664,
"401": 665,
"402": 666,
"403": 667,
"404": 668,
"405": 669,
"406": 670,
"407": 671,
"408": 672,
"409": 673,
"410": 674,
"411": 675,
"412": 676,
"413": 677,
"414": 678,
"415": 679,
"416": 680,
"417": 681,
"418": 682,
"419": 683,
"420": 684,
"421": 685,
"422": 686,
"423": 687,
"424": 688,
"425": 689,
"426": 690,
"427": 691,
"428": 692,
"429": 693,
"430": 694,
"431": 695,
"432": 696,
"433": 697,
"434": 698,
"435": 699,
"436": 700,
"437": 701,
"438": 702,
"439": 703,
"440": 704,
"441": 705,
"442": 706,
"443": 707,
"444": 708,
"445": 709,
"446": 710,
"447": 711,
"448": 712,
"449": 713,
"450": 714,
"451": 715,
"452": 716,
"453": 717,
"454": 718,
"455": 719,
"456": 720,
"457": 721,
"458": 722,
"459": 723,
"460": 724,
"461": 725,
"462": 726,
"463": 727,
"464": 728,
"465": 729,
"466": 730,
"467": 731,
"468": 732,
"469": 733,
"470": 734,
"471": 735,
"472": 736,
"473": 737,
"474": 738,
"475": 739,
"476": 740,
"477": 741,
"478": 742,
"479": 743,
"480": 744,
"481": 745,
"482": 746,
"483": 747,
"484": 748,
"485": 749,
"486": 750,
"487": 751,
"488": 752,
"489": 753,
"490": 754,
"491": 755,
"492": 756,
"493": 757,
"494": 758,
"495": 759,
"496": 760,
"497": 761,
"498": 762,
"499": 763,
"50": 764,
"500": 765,
"501": 766,
"502": 767,
"503": 768,
"504": 769,
"505": 770,
"506": 771,
"507": 772,
"508": 773,
"509": 774,
"51": 775,
"510": 776,
"511": 777,
"512": 778,
"513": 779,
"514": 780,
"515": 781,
"516": 782,
"517": 783,
"518": 784,
"519": 785,
"52": 786,
"520": 787,
"521": 788,
"522": 789,
"523": 790,
"524": 791,
"525": 792,
"526": 793,
"527": 794,
"528": 795,
"529": 796,
"53": 797,
"530": 798,
"531": 799,
"532": 800,
"533": 801,
"534": 802,
"535": 803,
"536": 804,
"537": 805,
"538": 806,
"539": 807,
"54": 808,
"540": 809,
"541": 810,
"542": 811,
"543": 812,
"544": 813,
"545": 814,
"546": 815,
"547": 816,
"548": 817,
"549": 818,
"55": 819,
"550": 820,
"551": 821,
"552": 822,
"553": 823,
"554": 824,
"555": 825,
"556": 826,
"557": 827,
"558": 828,
"559": 829,
"56": 830,
"560": 831,
"561": 832,
"562": 833,
"563": 834,
"564": 835,
"565": 836,
"566": 837,
"567": 838,
"568": 839,
"569": 840,
"57": 841,
"570": 842,
"571": 843,
"572": 844,
"573": 845,
"574": 846,
"575": 847,
"576": 848,
"577": 849,
"578": 850,
"579": 851,
"58": 852,
"580": 853,
"581": 854,
"582": 855,
"583": 856,
"584": 857,
"585": 858,
"586": 859,
"587": 860,
"588": 861,
"589": 862,
"59": 863,
"590": 864,
"591": 865,
"592": 866,
"593": 867,
"594": 868,
"595": 869,
"596": 870,
"597": 871,
"598": 872,
"599": 873,
"60": 874,
"600": 875,
"601": 876,
"602": 877,
"603": 878,
"604": 879,
"605": 880,
"606": 881,
"607": 882,
"608": 883,
"609": 884,
"61": 885,
"610": 886,
"611": 887,
"612": 888,
"613": 889,
"614": 890,
"615": 891,
"616": 892,
"617": 893,
"618": 894,
"619": 895,
"62": 896,
"620": 897,
"621": 898,
"622": 899,
"623": 900,
"624": 901,
"625": 902,
"626": 903,
"627": 904,
"628": 905,
"629": 906,
"63": 907,
"630": 908,
"631": 909,
"632": 910,
"633": 911,
"634": 912,
"635": 913,
"636": 914,
"637": 915,
"638": 916,
"639": 917,
"64": 918,
"640": 919,
"641": 920,
"642": 921,
"643": 922,
"644": 923,
"645": 924,
"646": 925,
"647": 926,
"648": 927,
"649": 928,
"65": 929,
"650": 930,
"651": 931,
"652": 932,
"653": 933,
"654": 934,
"655": 935,
"656": 936,
"657": 937,
"658": 938,
"659": 939,
"66": 940,
"660": 941,
"661": 942,
"662": 943,
"663": 944,
"664": 945,
"665": 946,
"666": 947,
"667": 948,
"668": 949,
"669": 950,
"67": 951,
"670": 952,
"671": 953,
"673": 954,
"674": 955,
"675": 956,
"676": 957,
"677": 958,
"678": 959,
"679": 960,
"68": 961,
"680": 962,
"681": 963,
"682": 964,
"683": 965,
"684": 966,
"685": 967,
"686": 968,
"687": 969,
"688": 970,
"689": 971,
"69": 972,
"690": 973,
"691": 974,
"692": 975,
"693": 976,
"694": 977,
"695": 978,
"696": 979,
"697": 980,
"698": 981,
"699": 982,
"70": 983,
"700": 984,
"701": 985,
"702": 986,
"703": 987,
"704": 988,
"705": 989,
"706": 990,
"707": 991,
"708": 992,
"709": 993,
"71": 994,
"710": 995,
"711": 996,
"712": 997,
"713": 998,
"714": 999,
"715": 1000,
"716": 1001,
"717": 1002,
"718": 1003,
"719": 1004,
"72": 1005,
"720": 1006,
"721": 1007,
"722": 1008,
"723": 1009,
"724": 1010,
"725": 1011,
"726": 1012,
"727": 1013,
"728": 1014,
"729": 1015,
"73": 1016,
"730": 1017,
"731": 1018,
"732": 1019,
"733": 1020,
"734": 1021,
"735": 1022,
"736": 1023,
"737": 1024,
"738": 1025,
"739": 1026,
"74": 1027,
"740": 1028,
"741": 1029,
"742": 1030,
"743": 1031,
"744": 1032,
"745": 1033,
"746": 1034,
"747": 1035,
"748": 1036,
"749": 1037,
"75": 1038,
"750": 1039,
"751": 1040,
"752": 1041,
"753": 1042,
"754": 1043,
"755": 1044,
"756": 1045,
"757": 1046,
"758": 1047,
"759": 1048,
"76": 1049,
"760": 1050,
"761": 1051,
"762": 1052,
"763": 1053,
"764": 1054,
"765": 1055,
"766": 1056,
"767": 1057,
"768": 1058,
"769": 1059,
"77": 1060,
"770": 1061,
"771": 1062,
"772": 1063,
"773": 1064,
"774": 1065,
"775": 1066,
"776": 1067,
"777": 1068,
"778": 1069,
"779": 1070,
"78": 1071,
"780": 1072,
"781": 1073,
"782": 1074,
"783": 1075,
"784": 1076,
"785": 1077,
"786": 1078,
"787": 1079,
"788": 1080,
"789": 1081,
"79": 1082,
"790": 1083,
"791": 1084,
"792": 1085,
"793": 1086,
"794": 1087,
"795": 1088,
"796": 1089,
"797": 1090,
"798": 1091,
"799": 1092,
"80": 1093,
"800": 1094,
"801": 1095,
"803": 1096,
"804": 1097,
"805": 1098,
"806": 1099,
"807": 1100,
"808": 1101,
"809": 1102,
"81": 1103,
"810": 1104,
"811": 1105,
"812": 1106,
"813": 1107,
"814": 1108,
"815": 1109,
"816": 1110,
"817": 1111,
"818": 1112,
"819": 1113,
"82": 1114,
"820": 1115,
"821": 1116,
"822": 1117,
"823": 1118,
"824": 1119,
"825": 1120,
"826": 1121,
"827": 1122,
"828": 1123,
"829": 1124,
"83": 1125,
"830": 1126,
"831": 1127,
"832": 1128,
"833": 1129,
"834": 1130,
"835": 1131,
"836": 1132,
"837": 1133,
"838": 1134,
"839": 1135,
"84": 1136,
"840": 1137,
"841": 1138,
"842": 1139,
"843": 1140,
"844": 1141,
"845": 1142,
"846": 1143,
"847": 1144,
"848": 1145,
"849": 1146,
"85": 1147,
"850": 1148,
"851": 1149,
"852": 1150,
"853": 1151,
"854": 1152,
"855": 1153,
"856": 1154,
"857": 1155,
"858": 1156,
"859": 1157,
"86": 1158,
"860": 1159,
"861": 1160,
"862": 1161,
"863": 1162,
"864": 1163,
"865": 1164,
"866": 1165,
"867": 1166,
"868": 1167,
"869": 1168,
"87": 1169,
"870": 1170,
"871": 1171,
"872": 1172,
"873": 1173,
"875": 1174,
"876": 1175,
"877": 1176,
"878": 1177,
"879": 1178,
"88": 1179,
"880": 1180,
"881": 1181,
"882": 1182,
"883": 1183,
"884": 1184,
"885": 1185,
"886": 1186,
"887": 1187,
"888": 1188,
"889": 1189,
"89": 1190,
"890": 1191,
"891": 1192,
"892": 1193,
"893": 1194,
"894": 1195,
"895": 1196,
"896": 1197,
"897": 1198,
"898": 1199,
"899": 1200,
"90": 1201,
"900": 1202,
"901": 1203,
"902": 1204,
"903": 1205,
"905": 1206,
"906": 1207,
"907": 1208,
"908": 1209,
"909": 1210,
"91": 1211,
"910": 1212,
"911": 1213,
"912": 1214,
"913": 1215,
"914": 1216,
"917": 1217,
"918": 1218,
"919": 1219,
"92": 1220,
"921": 1221,
"923": 1222,
"924": 1223,
"925": 1224,
"926": 1225,
"927": 1226,
"928": 1227,
"929": 1228,
"93": 1229,
"930": 1230,
"931": 1231,
"932": 1232,
"935": 1233,
"937": 1234,
"938": 1235,
"939": 1236,
"94": 1237,
"940": 1238,
"941": 1239,
"942": 1240,
"943": 1241,
"944": 1242,
"945": 1243,
"946": 1244,
"947": 1245,
"948": 1246,
"949": 1247,
"95": 1248,
"950": 1249,
"951": 1250,
"952": 1251,
"953": 1252,
"954": 1253,
"955": 1254,
"956": 1255,
"957": 1256,
"958": 1257,
"959": 1258,
"96": 1259,
"960": 1260,
"961": 1261,
"962": 1262,
"963": 1263,
"965": 1264,
"967": 1265,
"968": 1266,
"969": 1267,
"97": 1268,
"970": 1269,
"971": 1270,
"972": 1271,
"973": 1272,
"974": 1273,
"975": 1274,
"977": 1275,
"978": 1276,
"979": 1277,
"98": 1278,
"980": 1279,
"981": 1280,
"982": 1281,
"983": 1282,
"985": 1283,
"986": 1284,
"987": 1285,
"988": 1286,
"989": 1287,
"99": 1288,
"990": 1289,
"991": 1290,
"992": 1291,
"993": 1292,
"994": 1293,
"995": 1294,
"996": 1295,
"997": 1296,
"998": 1297,
"999": 1298
}