Upload tokenizer

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 5,
+      "id": 3,
       "content": "UTT_BOUNDARY",
       "single_word": false,
       "lstrip": false,
@@ -34,13 +34,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "\n"
-        },
-        "content": " UTT_BOUNDARY"
-      },
       {
         "type": "Strip",
         "strip_left": true,
@@ -51,207 +44,258 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
   "decoder": null,
   "model": {
     "type": "WordLevel",
     "vocab": {
       "UNK": 0,
       "PAD": 1,
-      "BOS": 2,
-      "EOS": 3,
-      "WORD_BOUNDARY": 4,
-      "UTT_BOUNDARY": 5,
-      "s": 6,
-      "iː": 7,
-      "ð": 8,
-      "ɛ": 9,
-      "ɹ": 10,
-      "z": 11,
-      "ʌ": 12,
-      "f": 13,
-      "eɪ": 14,
-      "w": 15,
-      "ɪ": 16,
-      "ɡ": 17,
-      "l": 18,
-      "æ": 19,
-      "ɑ": 20,
-      "h": 21,
-      "ə": 22,
-      "ʊ": 23,
-      "k": 24,
-      "p": 25,
-      "uː": 26,
-      "b": 27,
-      "i": 28,
-      "t": 29,
-      "aɪ": 30,
-      "θ": 31,
-      "ŋ": 32,
-      "j": 33,
-      "ɔ": 34,
-      "m": 35,
-      "ɔɪ": 36,
-      "n": 37,
-      "d": 38,
-      "oʊ": 39,
-      "aʊ": 40,
-      "v": 41,
-      "ɜː": 42,
-      "t̠ʃ": 43,
-      "d̠ʒ": 44,
-      "ʃ": 45,
-      "iə": 46,
-      "ʒ": 47,
-      "ɑ̃": 48,
-      "r": 49,
-      "x": 50,
-      "nʲ": 51,
-      "ɒ": 52,
-      "a": 53,
-      "ɑː": 54,
-      "ɔː": 55,
-      "əʊ": 56,
-      "ɐ": 57,
-      "eə": 58,
-      "ʊə": 59,
-      "n̩": 60,
-      "aː": 61,
-      "y": 62,
-      "ɛ̃": 63,
-      "ʁ": 64,
-      "e": 65,
-      "ɔ̃": 66,
-      "u": 67,
-      "o": 68,
-      "œ̃": 69,
-      "ø": 70,
-      "œ": 71,
-      "oː": 72,
-      "yː": 73,
-      "ɲ": 74,
-      "ts": 75,
-      "eː": 76,
-      "ʀ": 77,
-      "ç": 78,
-      "ɛɪ": 79,
-      "ʏ": 80,
-      "ɛː": 81,
-      "pf": 82,
-      "øː": 83,
-      "ã": 84,
-      "ɾ": 85,
-      "β": 86,
-      "ʎ": 87,
-      "ɣ": 88,
-      "ʝ": 89,
-      "oɪ": 90,
-      "eʊ": 91,
-      "pː": 92,
-      "ɟ": 93,
-      "ʋ": 94,
-      "ɪː": 95,
-      "ɵ": 96,
-      "œy": 97,
-      "tʲ": 98,
-      "au̯": 99,
-      "ʂ": 100,
-      "ɤ": 101,
-      "kʰ": 102,
-      "ʈʂʰ": 103,
-      "ɕ": 104,
-      "pʰ": 105,
-      "tɕ": 106,
-      "ʈʂ": 107,
-      "ɹ̩": 108,
-      "tɕʰ": 109,
-      "tʰ": 110,
-      "ɻ": 111,
-      "ɥ": 112,
-      "tsʰ": 113,
-      "ei̯": 114,
-      "ou̯": 115,
-      "ɻ̩": 116,
-      "ai̯": 117,
-      "ɯː": 118,
-      "ɯ": 119,
-      "pʲ": 120,
-      "ɸ": 121,
-      "rʲ": 122,
-      "kʲ": 123,
-      "bʲ": 124,
-      "mʲ": 125,
-      "kː": 126,
-      "æi": 127,
-      "yi": 128,
-      "ɵː": 129,
-      "tː": 130,
-      "æː": 131,
-      "dʑ": 132,
-      "l̩": 133,
-      "œː": 134,
-      "ʌː": 135,
-      "ɜ": 136,
-      "ʔ": 137,
-      "s̺": 138,
-      "ts̻": 139,
-      "s̻": 140,
-      "c": 141,
-      "ts̺": 142,
-      "tsː": 143,
-      "ɟː": 144,
-      "t̠ʃː": 145,
-      "ɡː": 146,
-      "dː": 147,
-      "cː": 148,
-      "bː": 149,
-      "dzː": 150,
-      "ɫ": 151,
-      "ʊː": 152,
-      "q": 153,
-      "øy": 154,
-      "χ": 155,
-      "i̯": 156,
-      "t̪": 157,
-      "d̪": 158,
-      "lʲ": 159,
-      "ɡʲ": 160,
-      "hʲ": 161,
-      "dʲ": 162,
-      "çʲ": 163,
-      "uə": 164,
-      "ŭ": 165,
-      "fʲ": 166,
-      "aɨ": 167,
-      "ɨ": 168,
-      "uɨ": 169,
-      "əɪ": 170,
-      "ɨː": 171,
-      "ɬ": 172,
-      "əɨ": 173,
-      "ɪu": 174,
-      "ʉ": 175,
-      "ʉː": 176,
-      "ɑɪ": 177,
-      "ʑ": 178,
-      "dz": 179,
-      "d̠ʒː": 180,
-      "ɐ̃": 181,
-      "ɛʊ": 182,
-      "ũ": 183,
-      "iʊ": 184,
-      "õ": 185,
-      "uɪ": 186,
-      "sʲ": 187,
-      "t̠ʃʲ": 188,
-      "ɔa": 189,
-      "ea": 190,
-      "iɪ": 191,
-      "tsʲ": 192,
-      "eo": 193,
-      "d̠ʒʲ": 194,
-      "ɾʲ": 195
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "s": 4,
+      "iː": 5,
+      "ð": 6,
+      "ɛ": 7,
+      "ɹ": 8,
+      "z": 9,
+      "ʌ": 10,
+      "f": 11,
+      "eɪ": 12,
+      "w": 13,
+      "ɪ": 14,
+      "ɡ": 15,
+      "l": 16,
+      "æ": 17,
+      "ɑ": 18,
+      "h": 19,
+      "ə": 20,
+      "ʊ": 21,
+      "k": 22,
+      "p": 23,
+      "uː": 24,
+      "b": 25,
+      "i": 26,
+      "t": 27,
+      "aɪ": 28,
+      "θ": 29,
+      "ŋ": 30,
+      "j": 31,
+      "ɔ": 32,
+      "m": 33,
+      "ɔɪ": 34,
+      "n": 35,
+      "d": 36,
+      "oʊ": 37,
+      "aʊ": 38,
+      "v": 39,
+      "ɜː": 40,
+      "t̠ʃ": 41,
+      "d̠ʒ": 42,
+      "ʃ": 43,
+      "iə": 44,
+      "ʒ": 45,
+      "ɑ̃": 46,
+      "r": 47,
+      "x": 48,
+      "nʲ": 49,
+      "ɒ": 50,
+      "a": 51,
+      "ɑː": 52,
+      "ɔː": 53,
+      "əʊ": 54,
+      "ɐ": 55,
+      "eə": 56,
+      "ʊə": 57,
+      "n̩": 58,
+      "aː": 59,
+      "y": 60,
+      "ɛ̃": 61,
+      "ʁ": 62,
+      "e": 63,
+      "ɔ̃": 64,
+      "u": 65,
+      "o": 66,
+      "œ̃": 67,
+      "ø": 68,
+      "œ": 69,
+      "oː": 70,
+      "yː": 71,
+      "ɲ": 72,
+      "ts": 73,
+      "eː": 74,
+      "ʀ": 75,
+      "ç": 76,
+      "ɛɪ": 77,
+      "ʏ": 78,
+      "ɛː": 79,
+      "pf": 80,
+      "øː": 81,
+      "ã": 82,
+      "ɾ": 83,
+      "β": 84,
+      "ʎ": 85,
+      "ɣ": 86,
+      "ʝ": 87,
+      "oɪ": 88,
+      "eʊ": 89,
+      "pː": 90,
+      "ɟ": 91,
+      "ʋ": 92,
+      "ɪː": 93,
+      "ɵ": 94,
+      "œy": 95,
+      "tʲ": 96,
+      "au̯": 97,
+      "ʂ": 98,
+      "ɤ": 99,
+      "kʰ": 100,
+      "ʈʂʰ": 101,
+      "ɕ": 102,
+      "pʰ": 103,
+      "tɕ": 104,
+      "ʈʂ": 105,
+      "ɹ̩": 106,
+      "tɕʰ": 107,
+      "tʰ": 108,
+      "ɻ": 109,
+      "ɥ": 110,
+      "tsʰ": 111,
+      "ei̯": 112,
+      "ou̯": 113,
+      "ɻ̩": 114,
+      "ai̯": 115,
+      "ɯː": 116,
+      "ɯ": 117,
+      "pʲ": 118,
+      "ɸ": 119,
+      "rʲ": 120,
+      "kʲ": 121,
+      "bʲ": 122,
+      "mʲ": 123,
+      "kː": 124,
+      "æi": 125,
+      "yi": 126,
+      "ɵː": 127,
+      "tː": 128,
+      "æː": 129,
+      "dʑ": 130,
+      "l̩": 131,
+      "œː": 132,
+      "ʌː": 133,
+      "ɜ": 134,
+      "ʔ": 135,
+      "s̺": 136,
+      "ts̻": 137,
+      "s̻": 138,
+      "c": 139,
+      "ts̺": 140,
+      "tsː": 141,
+      "ɟː": 142,
+      "t̠ʃː": 143,
+      "ɡː": 144,
+      "dː": 145,
+      "cː": 146,
+      "bː": 147,
+      "dzː": 148,
+      "ɫ": 149,
+      "ʊː": 150,
+      "q": 151,
+      "øy": 152,
+      "χ": 153,
+      "i̯": 154,
+      "t̪": 155,
+      "d̪": 156,
+      "lʲ": 157,
+      "ɡʲ": 158,
+      "hʲ": 159,
+      "dʲ": 160,
+      "çʲ": 161,
+      "uə": 162,
+      "ŭ": 163,
+      "fʲ": 164,
+      "aɨ": 165,
+      "ɨ": 166,
+      "uɨ": 167,
+      "əɪ": 168,
+      "ɨː": 169,
+      "ɬ": 170,
+      "əɨ": 171,
+      "ɪu": 172,
+      "ʉ": 173,
+      "ʉː": 174,
+      "ɑɪ": 175,
+      "ʑ": 176,
+      "dz": 177,
+      "d̠ʒː": 178,
+      "ɐ̃": 179,
+      "ɛʊ": 180,
+      "ũ": 181,
+      "iʊ": 182,
+      "õ": 183,
+      "uɪ": 184,
+      "sʲ": 185,
+      "t̠ʃʲ": 186,
+      "ɔa": 187,
+      "ea": 188,
+      "iɪ": 189,
+      "tsʲ": 190,
+      "eo": 191,
+      "d̠ʒʲ": 192,
+      "ɾʲ": 193
     },
     "unk_token": "UNK"
   }

tokenizer_config.json

@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "3": {
       "content": "UTT_BOUNDARY",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"s":6,"iː":7,"ð":8,"ɛ":9,"ɹ":10,"z":11,"ʌ":12,"f":13,"eɪ":14,"w":15,"ɪ":16,"ɡ":17,"l":18,"æ":19,"ɑ":20,"h":21,"ə":22,"ʊ":23,"k":24,"p":25,"uː":26,"b":27,"i":28,"t":29,"aɪ":30,"θ":31,"ŋ":32,"j":33,"ɔ":34,"m":35,"ɔɪ":36,"n":37,"d":38,"oʊ":39,"aʊ":40,"v":41,"ɜː":42,"t̠ʃ":43,"d̠ʒ":44,"ʃ":45,"iə":46,"ʒ":47,"ɑ̃":48,"r":49,"x":50,"nʲ":51,"ɒ":52,"a":53,"ɑː":54,"ɔː":55,"əʊ":56,"ɐ":57,"eə":58,"ʊə":59,"n̩":60,"aː":61,"y":62,"ɛ̃":63,"ʁ":64,"e":65,"ɔ̃":66,"u":67,"o":68,"œ̃":69,"ø":70,"œ":71,"oː":72,"yː":73,"ɲ":74,"ts":75,"eː":76,"ʀ":77,"ç":78,"ɛɪ":79,"ʏ":80,"ɛː":81,"pf":82,"øː":83,"ã":84,"ɾ":85,"β":86,"ʎ":87,"ɣ":88,"ʝ":89,"oɪ":90,"eʊ":91,"pː":92,"ɟ":93,"ʋ":94,"ɪː":95,"ɵ":96,"œy":97,"tʲ":98,"au̯":99,"ʂ":100,"ɤ":101,"kʰ":102,"ʈʂʰ":103,"ɕ":104,"pʰ":105,"tɕ":106,"ʈʂ":107,"ɹ̩":108,"tɕʰ":109,"tʰ":110,"ɻ":111,"ɥ":112,"tsʰ":113,"ei̯":114,"ou̯":115,"ɻ̩":116,"ai̯":117,"ɯː":118,"ɯ":119,"pʲ":120,"ɸ":121,"rʲ":122,"kʲ":123,"bʲ":124,"mʲ":125,"kː":126,"æi":127,"yi":128,"ɵː":129,"tː":130,"æː":131,"dʑ":132,"l̩":133,"œː":134,"ʌː":135,"ɜ":136,"ʔ":137,"s̺":138,"ts̻":139,"s̻":140,"c":141,"ts̺":142,"tsː":143,"ɟː":144,"t̠ʃː":145,"ɡː":146,"dː":147,"cː":148,"bː":149,"dzː":150,"ɫ":151,"ʊː":152,"q":153,"øy":154,"χ":155,"i̯":156,"t̪":157,"d̪":158,"lʲ":159,"ɡʲ":160,"hʲ":161,"dʲ":162,"çʲ":163,"uə":164,"ŭ":165,"fʲ":166,"aɨ":167,"ɨ":168,"uɨ":169,"əɪ":170,"ɨː":171,"ɬ":172,"əɨ":173,"ɪu":174,"ʉ":175,"ʉː":176,"ɑɪ":177,"ʑ":178,"dz":179,"d̠ʒː":180,"ɐ̃":181,"ɛʊ":182,"ũ":183,"iʊ":184,"õ":185,"uɪ":186,"sʲ":187,"t̠ʃʲ":188,"ɔa":189,"ea":190,"iɪ":191,"tsʲ":192,"eo":193,"d̠ʒʲ":194,"ɾʲ":195}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"s":4,"iː":5,"ð":6,"ɛ":7,"ɹ":8,"z":9,"ʌ":10,"f":11,"eɪ":12,"w":13,"ɪ":14,"ɡ":15,"l":16,"æ":17,"ɑ":18,"h":19,"ə":20,"ʊ":21,"k":22,"p":23,"uː":24,"b":25,"i":26,"t":27,"aɪ":28,"θ":29,"ŋ":30,"j":31,"ɔ":32,"m":33,"ɔɪ":34,"n":35,"d":36,"oʊ":37,"aʊ":38,"v":39,"ɜː":40,"t̠ʃ":41,"d̠ʒ":42,"ʃ":43,"iə":44,"ʒ":45,"ɑ̃":46,"r":47,"x":48,"nʲ":49,"ɒ":50,"a":51,"ɑː":52,"ɔː":53,"əʊ":54,"ɐ":55,"eə":56,"ʊə":57,"n̩":58,"aː":59,"y":60,"ɛ̃":61,"ʁ":62,"e":63,"ɔ̃":64,"u":65,"o":66,"œ̃":67,"ø":68,"œ":69,"oː":70,"yː":71,"ɲ":72,"ts":73,"eː":74,"ʀ":75,"ç":76,"ɛɪ":77,"ʏ":78,"ɛː":79,"pf":80,"øː":81,"ã":82,"ɾ":83,"β":84,"ʎ":85,"ɣ":86,"ʝ":87,"oɪ":88,"eʊ":89,"pː":90,"ɟ":91,"ʋ":92,"ɪː":93,"ɵ":94,"œy":95,"tʲ":96,"au̯":97,"ʂ":98,"ɤ":99,"kʰ":100,"ʈʂʰ":101,"ɕ":102,"pʰ":103,"tɕ":104,"ʈʂ":105,"ɹ̩":106,"tɕʰ":107,"tʰ":108,"ɻ":109,"ɥ":110,"tsʰ":111,"ei̯":112,"ou̯":113,"ɻ̩":114,"ai̯":115,"ɯː":116,"ɯ":117,"pʲ":118,"ɸ":119,"rʲ":120,"kʲ":121,"bʲ":122,"mʲ":123,"kː":124,"æi":125,"yi":126,"ɵː":127,"tː":128,"æː":129,"dʑ":130,"l̩":131,"œː":132,"ʌː":133,"ɜ":134,"ʔ":135,"s̺":136,"ts̻":137,"s̻":138,"c":139,"ts̺":140,"tsː":141,"ɟː":142,"t̠ʃː":143,"ɡː":144,"dː":145,"cː":146,"bː":147,"dzː":148,"ɫ":149,"ʊː":150,"q":151,"øy":152,"χ":153,"i̯":154,"t̪":155,"d̪":156,"lʲ":157,"ɡʲ":158,"hʲ":159,"dʲ":160,"çʲ":161,"uə":162,"ŭ":163,"fʲ":164,"aɨ":165,"ɨ":166,"uɨ":167,"əɪ":168,"ɨː":169,"ɬ":170,"əɨ":171,"ɪu":172,"ʉ":173,"ʉː":174,"ɑɪ":175,"ʑ":176,"dz":177,"d̠ʒː":178,"ɐ̃":179,"ɛʊ":180,"ũ":181,"iʊ":182,"õ":183,"uɪ":184,"sʲ":185,"t̠ʃʲ":186,"ɔa":187,"ea":188,"iɪ":189,"tsʲ":190,"eo":191,"d̠ʒʲ":192,"ɾʲ":193}