add tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +6 -0
tokenizer_config.json +11 -0
vocab.json +532 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,11 @@

+{
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
+}

vocab.json ADDED Viewed

	@@ -0,0 +1,532 @@

+{
+  "0": 59,
+  "1": 299,
+  "10": 20,
+  "100": 315,
+  "101": 321,
+  "102": 46,
+  "103": 332,
+  "104": 335,
+  "105": 467,
+  "106": 495,
+  "107": 30,
+  "108": 145,
+  "109": 202,
+  "11": 35,
+  "110": 385,
+  "111": 207,
+  "112": 366,
+  "113": 121,
+  "114": 109,
+  "115": 16,
+  "116": 57,
+  "117": 451,
+  "118": 224,
+  "119": 282,
+  "12": 496,
+  "120": 501,
+  "121": 408,
+  "122": 484,
+  "123": 235,
+  "124": 485,
+  "125": 173,
+  "126": 176,
+  "127": 137,
+  "128": 374,
+  "129": 236,
+  "13": 340,
+  "130": 521,
+  "131": 110,
+  "132": 240,
+  "133": 290,
+  "134": 488,
+  "135": 393,
+  "136": 302,
+  "137": 10,
+  "138": 378,
+  "139": 489,
+  "14": 428,
+  "140": 79,
+  "141": 349,
+  "142": 8,
+  "143": 61,
+  "144": 247,
+  "145": 163,
+  "146": 268,
+  "147": 55,
+  "148": 347,
+  "149": 76,
+  "15": 443,
+  "150": 499,
+  "151": 72,
+  "152": 439,
+  "153": 251,
+  "154": 361,
+  "155": 257,
+  "156": 64,
+  "157": 169,
+  "158": 265,
+  "159": 250,
+  "16": 308,
+  "160": 218,
+  "161": 450,
+  "162": 96,
+  "163": 314,
+  "164": 508,
+  "165": 516,
+  "166": 62,
+  "167": 363,
+  "168": 338,
+  "169": 90,
+  "17": 396,
+  "170": 131,
+  "171": 294,
+  "172": 350,
+  "173": 352,
+  "174": 254,
+  "175": 199,
+  "176": 179,
+  "177": 226,
+  "178": 3,
+  "179": 134,
+  "18": 239,
+  "180": 243,
+  "181": 124,
+  "182": 65,
+  "183": 284,
+  "184": 360,
+  "185": 220,
+  "186": 47,
+  "187": 432,
+  "188": 305,
+  "189": 138,
+  "19": 461,
+  "190": 200,
+  "191": 263,
+  "192": 185,
+  "193": 73,
+  "194": 22,
+  "195": 468,
+  "196": 445,
+  "197": 466,
+  "198": 66,
+  "199": 502,
+  "2": 83,
+  "20": 407,
+  "200": 319,
+  "201": 464,
+  "202": 397,
+  "203": 195,
+  "204": 452,
+  "205": 204,
+  "206": 435,
+  "207": 293,
+  "208": 470,
+  "209": 400,
+  "21": 358,
+  "210": 222,
+  "211": 513,
+  "212": 157,
+  "213": 147,
+  "214": 89,
+  "215": 525,
+  "216": 165,
+  "217": 212,
+  "218": 469,
+  "219": 43,
+  "22": 426,
+  "220": 152,
+  "221": 129,
+  "222": 295,
+  "223": 328,
+  "224": 274,
+  "225": 313,
+  "226": 529,
+  "227": 404,
+  "228": 50,
+  "229": 132,
+  "23": 209,
+  "230": 210,
+  "231": 71,
+  "232": 371,
+  "233": 171,
+  "234": 474,
+  "235": 421,
+  "236": 359,
+  "237": 326,
+  "238": 162,
+  "239": 416,
+  "24": 97,
+  "240": 515,
+  "241": 56,
+  "242": 480,
+  "243": 187,
+  "244": 411,
+  "245": 422,
+  "246": 323,
+  "247": 182,
+  "248": 98,
+  "249": 327,
+  "25": 465,
+  "250": 100,
+  "251": 409,
+  "252": 229,
+  "253": 434,
+  "254": 103,
+  "255": 130,
+  "256": 151,
+  "257": 479,
+  "258": 438,
+  "259": 276,
+  "26": 104,
+  "260": 201,
+  "261": 244,
+  "262": 387,
+  "263": 324,
+  "264": 351,
+  "265": 166,
+  "266": 92,
+  "267": 381,
+  "268": 380,
+  "269": 386,
+  "27": 447,
+  "270": 69,
+  "271": 401,
+  "272": 213,
+  "273": 148,
+  "274": 526,
+  "275": 38,
+  "276": 170,
+  "277": 376,
+  "278": 227,
+  "279": 261,
+  "28": 91,
+  "280": 287,
+  "281": 519,
+  "282": 106,
+  "283": 9,
+  "284": 427,
+  "285": 172,
+  "286": 477,
+  "287": 198,
+  "288": 430,
+  "289": 481,
+  "29": 118,
+  "290": 458,
+  "291": 80,
+  "292": 491,
+  "293": 492,
+  "294": 17,
+  "295": 419,
+  "296": 269,
+  "297": 362,
+  "298": 412,
+  "299": 343,
+  "3": 390,
+  "30": 471,
+  "300": 462,
+  "301": 153,
+  "302": 344,
+  "303": 292,
+  "304": 105,
+  "305": 158,
+  "306": 423,
+  "307": 167,
+  "308": 127,
+  "309": 140,
+  "31": 486,
+  "310": 115,
+  "311": 365,
+  "312": 456,
+  "313": 188,
+  "314": 512,
+  "315": 379,
+  "316": 177,
+  "317": 339,
+  "318": 415,
+  "319": 325,
+  "32": 123,
+  "320": 524,
+  "321": 28,
+  "322": 483,
+  "323": 37,
+  "324": 364,
+  "325": 249,
+  "326": 181,
+  "327": 219,
+  "328": 60,
+  "329": 286,
+  "33": 514,
+  "330": 33,
+  "331": 402,
+  "332": 231,
+  "333": 116,
+  "334": 316,
+  "335": 39,
+  "336": 436,
+  "337": 155,
+  "338": 460,
+  "339": 414,
+  "34": 431,
+  "340": 440,
+  "341": 406,
+  "342": 527,
+  "343": 270,
+  "344": 193,
+  "345": 211,
+  "346": 355,
+  "347": 297,
+  "348": 357,
+  "349": 77,
+  "35": 160,
+  "350": 85,
+  "351": 45,
+  "352": 417,
+  "353": 11,
+  "354": 31,
+  "355": 178,
+  "356": 478,
+  "357": 21,
+  "358": 384,
+  "359": 189,
+  "36": 36,
+  "360": 74,
+  "361": 262,
+  "362": 206,
+  "363": 99,
+  "364": 382,
+  "365": 146,
+  "366": 394,
+  "367": 518,
+  "368": 320,
+  "369": 283,
+  "37": 398,
+  "370": 154,
+  "371": 215,
+  "372": 107,
+  "373": 383,
+  "374": 78,
+  "375": 348,
+  "376": 214,
+  "377": 246,
+  "378": 53,
+  "379": 0,
+  "38": 441,
+  "380": 234,
+  "381": 40,
+  "382": 128,
+  "383": 255,
+  "384": 288,
+  "385": 111,
+  "386": 190,
+  "387": 12,
+  "388": 277,
+  "389": 4,
+  "39": 273,
+  "390": 472,
+  "391": 18,
+  "392": 453,
+  "393": 44,
+  "394": 70,
+  "395": 342,
+  "396": 117,
+  "397": 446,
+  "398": 318,
+  "399": 183,
+  "4": 32,
+  "40": 29,
+  "400": 139,
+  "401": 6,
+  "402": 498,
+  "403": 233,
+  "404": 14,
+  "405": 370,
+  "406": 196,
+  "407": 520,
+  "408": 493,
+  "409": 507,
+  "41": 95,
+  "410": 221,
+  "411": 275,
+  "412": 143,
+  "413": 25,
+  "414": 429,
+  "415": 373,
+  "416": 223,
+  "417": 87,
+  "418": 303,
+  "419": 307,
+  "42": 175,
+  "420": 252,
+  "421": 482,
+  "422": 372,
+  "423": 266,
+  "424": 285,
+  "425": 125,
+  "426": 159,
+  "427": 329,
+  "428": 180,
+  "429": 88,
+  "43": 81,
+  "430": 442,
+  "431": 304,
+  "432": 336,
+  "433": 317,
+  "434": 528,
+  "435": 126,
+  "436": 7,
+  "437": 399,
+  "438": 279,
+  "439": 5,
+  "44": 392,
+  "440": 504,
+  "441": 237,
+  "442": 500,
+  "443": 122,
+  "444": 205,
+  "445": 291,
+  "446": 264,
+  "447": 391,
+  "448": 454,
+  "449": 75,
+  "45": 473,
+  "450": 497,
+  "451": 144,
+  "452": 67,
+  "453": 345,
+  "454": 522,
+  "455": 19,
+  "456": 267,
+  "457": 101,
+  "458": 356,
+  "459": 509,
+  "46": 63,
+  "460": 322,
+  "461": 413,
+  "462": 94,
+  "463": 24,
+  "464": 194,
+  "465": 260,
+  "466": 510,
+  "467": 490,
+  "468": 192,
+  "469": 86,
+  "47": 186,
+  "470": 271,
+  "471": 225,
+  "472": 330,
+  "473": 433,
+  "474": 337,
+  "475": 511,
+  "476": 42,
+  "477": 494,
+  "478": 311,
+  "479": 476,
+  "48": 114,
+  "480": 13,
+  "481": 253,
+  "482": 15,
+  "483": 241,
+  "484": 141,
+  "485": 331,
+  "486": 306,
+  "487": 341,
+  "488": 403,
+  "489": 353,
+  "49": 455,
+  "490": 280,
+  "491": 281,
+  "492": 149,
+  "493": 208,
+  "494": 52,
+  "495": 296,
+  "496": 34,
+  "497": 108,
+  "498": 410,
+  "499": 48,
+  "5": 405,
+  "50": 230,
+  "500": 216,
+  "51": 161,
+  "52": 333,
+  "53": 346,
+  "54": 93,
+  "55": 142,
+  "56": 54,
+  "57": 354,
+  "58": 112,
+  "59": 102,
+  "6": 156,
+  "60": 133,
+  "61": 289,
+  "62": 238,
+  "63": 334,
+  "64": 135,
+  "65": 232,
+  "66": 448,
+  "67": 164,
+  "68": 395,
+  "69": 84,
+  "7": 174,
+  "70": 475,
+  "71": 49,
+  "72": 51,
+  "73": 505,
+  "74": 377,
+  "75": 389,
+  "76": 203,
+  "77": 463,
+  "78": 503,
+  "79": 248,
+  "8": 367,
+  "80": 278,
+  "81": 256,
+  "82": 310,
+  "83": 437,
+  "84": 2,
+  "85": 517,
+  "86": 168,
+  "87": 217,
+  "88": 424,
+  "89": 444,
+  "9": 523,
+  "90": 113,
+  "91": 259,
+  "92": 418,
+  "93": 136,
+  "94": 242,
+  "95": 258,
+  "96": 301,
+  "97": 300,
+  "98": 27,
+  "99": 245,
+  "[PAD]": 420,
+  "[UNK]": 1,
+  "a": 459,
+  "b": 368,
+  "c": 457,
+  "d": 68,
+  "e": 309,
+  "f": 58,
+  "g": 298,
+  "h": 191,
+  "i": 41,
+  "j": 369,
+  "k": 272,
+  "l": 150,
+  "m": 449,
+  "n": 506,
+  "o": 82,
+  "p": 197,
+  "q": 26,
+  "r": 425,
+  "s": 119,
+  "t": 228,
+  "u": 375,
+  "v": 120,
+  "w": 184,
+  "x": 487,
+  "y": 388,
+  "z": 312,
+  "|": 23
+}