Upload tokenizer

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 5,
+      "id": 3,
       "content": "UTT_BOUNDARY",
       "single_word": false,
       "lstrip": false,
@@ -34,13 +34,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "\n"
-        },
-        "content": " UTT_BOUNDARY"
-      },
       {
         "type": "Strip",
         "strip_left": true,
@@ -51,60 +44,111 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
   "decoder": null,
   "model": {
     "type": "WordLevel",
     "vocab": {
       "UNK": 0,
       "PAD": 1,
-      "BOS": 2,
-      "EOS": 3,
-      "WORD_BOUNDARY": 4,
-      "UTT_BOUNDARY": 5,
-      "s": 6,
-      "æ": 7,
-      "n": 8,
-      "o": 9,
-      "j": 10,
-      "ʊ": 11,
-      "ɔ": 12,
-      "a": 13,
-      "r": 14,
-      "m": 15,
-      "ɯ": 16,
-      "k": 17,
-      "ɪ": 18,
-      "l": 19,
-      "i": 20,
-      "ɛ": 21,
-      "v": 22,
-      "d": 23,
-      "d̠ʒ": 24,
-      "y": 25,
-      "t": 26,
-      "b": 27,
-      "u": 28,
-      "z": 29,
-      "ʃ": 30,
-      "ɟ": 31,
-      "e": 32,
-      "p": 33,
-      "ɡ": 34,
-      "ɫ": 35,
-      "h": 36,
-      "t̠ʃ": 37,
-      "ɾ": 38,
-      "f": 39,
-      "ø": 40,
-      "œ": 41,
-      "aː": 42,
-      "c": 43,
-      "ʊː": 44,
-      "tː": 45,
-      "oː": 46,
-      "œː": 47,
-      "ʒ": 48
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "s": 4,
+      "æ": 5,
+      "n": 6,
+      "o": 7,
+      "j": 8,
+      "ʊ": 9,
+      "ɔ": 10,
+      "a": 11,
+      "r": 12,
+      "m": 13,
+      "ɯ": 14,
+      "k": 15,
+      "ɪ": 16,
+      "l": 17,
+      "i": 18,
+      "ɛ": 19,
+      "v": 20,
+      "d": 21,
+      "d̠ʒ": 22,
+      "y": 23,
+      "t": 24,
+      "b": 25,
+      "u": 26,
+      "z": 27,
+      "ʃ": 28,
+      "ɟ": 29,
+      "e": 30,
+      "p": 31,
+      "ɡ": 32,
+      "ɫ": 33,
+      "h": 34,
+      "t̠ʃ": 35,
+      "ɾ": 36,
+      "f": 37,
+      "ø": 38,
+      "œ": 39,
+      "aː": 40,
+      "c": 41,
+      "ʊː": 42,
+      "tː": 43,
+      "oː": 44,
+      "œː": 45,
+      "ʒ": 46
     },
     "unk_token": "UNK"
   }

tokenizer_config.json

@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "3": {
       "content": "UTT_BOUNDARY",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"s":6,"æ":7,"n":8,"o":9,"j":10,"ʊ":11,"ɔ":12,"a":13,"r":14,"m":15,"ɯ":16,"k":17,"ɪ":18,"l":19,"i":20,"ɛ":21,"v":22,"d":23,"d̠ʒ":24,"y":25,"t":26,"b":27,"u":28,"z":29,"ʃ":30,"ɟ":31,"e":32,"p":33,"ɡ":34,"ɫ":35,"h":36,"t̠ʃ":37,"ɾ":38,"f":39,"ø":40,"œ":41,"aː":42,"c":43,"ʊː":44,"tː":45,"oː":46,"œː":47,"ʒ":48}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"s":4,"æ":5,"n":6,"o":7,"j":8,"ʊ":9,"ɔ":10,"a":11,"r":12,"m":13,"ɯ":14,"k":15,"ɪ":16,"l":17,"i":18,"ɛ":19,"v":20,"d":21,"d̠ʒ":22,"y":23,"t":24,"b":25,"u":26,"z":27,"ʃ":28,"ɟ":29,"e":30,"p":31,"ɡ":32,"ɫ":33,"h":34,"t̠ʃ":35,"ɾ":36,"f":37,"ø":38,"œ":39,"aː":40,"c":41,"ʊː":42,"tː":43,"oː":44,"œː":45,"ʒ":46}