add tokenizer

vocab.json

@@ -1 +1 @@
-{"m": 0, "r": 1, "w": 2, "”": 3, "a": 4, "l": 5, "u": 6, "“": 7, "x": 8, "ʻ": 9, "t": 10, "k": 11, "ƙ": 12, "ɗ": 13, "i": 14, "’": 15, "e": 16, "o": 17, "í": 18, "j": 19, "'": 20, "h": 21, "z": 22, "p": 23, "d": 24, "n": 25, "y": 26, "b": 27, "g": 28, "—": 29, "ƴ": 30, "s": 31, "f": 32, "q": 34, "v": 35, "ɓ": 36, "c": 37, "|": 33, "[UNK]": 38, "[PAD]": 39}
+{"s": 0, "m": 1, "e": 2, "g": 3, "f": 4, "í": 5, "c": 6, "'": 7, "a": 8, "v": 9, "i": 10, "q": 11, "j": 12, "r": 13, "o": 14, "ɓ": 15, "z": 16, "p": 17, "ƙ": 19, "x": 20, "b": 21, "k": 22, "y": 23, "l": 24, "d": 25, "ɗ": 26, "w": 27, "h": 28, "ƴ": 29, "u": 30, "n": 31, "t": 32, "|": 18, "[UNK]": 33, "[PAD]": 34}