markjosims's picture
Upload tokenizer
dc23353
raw
history blame contribute delete
577 Bytes
{
"\n": 0,
"\r": 1,
"[PAD]": 49,
"[UNK]": 48,
"a": 3,
"b": 4,
"c": 5,
"d": 6,
"e": 7,
"g": 8,
"h": 9,
"i": 10,
"j": 11,
"k": 12,
"l": 13,
"m": 14,
"n": 15,
"o": 16,
"p": 17,
"r": 18,
"s": 19,
"t": 20,
"u": 21,
"v": 22,
"w": 23,
"|": 2,
"´": 24,
"ð": 25,
"ŋ": 26,
"ɔ": 27,
"ə": 28,
"ɛ": 29,
"ɜ": 30,
"ɟ": 31,
"ɲ": 32,
"ɽ": 33,
"ɾ": 34,
"ʃ": 35,
"̀": 36,
"́": 37,
"̂": 38,
"̄": 39,
"̊": 40,
"̋": 41,
"̌": 42,
"̏": 43,
"̪": 44,
"̯": 45,
"̺": 46,
"͡": 47
}