add tokenizer

vocab.json

@@ -1 +1 @@
-{"ƴ": 0, "s": 1, "k": 2, "ɓ": 3, "“": 4, "d": 5, "y": 6, "x": 7, "z": 8, "j": 9, "u": 10, "t": 11, "o": 12, "c": 13, "n": 14, "i": 15, "h": 16, "b": 17, "e": 18, "ɗ": 19, "w": 20, "g": 21, "’": 22, "ƙ": 23, "ʻ": 24, "”": 26, "m": 27, "v": 28, "a": 29, "—": 30, "í": 31, "l": 32, "f": 33, "p": 34, "q": 35, "'": 36, "r": 37, "|": 25, "[UNK]": 38, "[PAD]": 39}
+{"q": 0, "x": 1, "ƙ": 2, "“": 3, "y": 4, "f": 5, "ɓ": 6, "’": 7, "h": 8, "d": 9, "t": 10, "u": 11, "n": 12, "r": 13, "a": 14, "j": 15, "ɗ": 16, "p": 18, "”": 19, "g": 20, "m": 21, "v": 22, "w": 23, "l": 24, "í": 25, "k": 26, "s": 27, "z": 28, "ʻ": 29, "i": 30, "o": 31, "'": 32, "—": 33, "ƴ": 34, "b": 35, "e": 36, "c": 37, "|": 17, "[UNK]": 38, "[PAD]": 39}