yaswanth's picture
add tokenizer
1f646b6
raw
history blame
985 Bytes
{"ॅ": 0, "र": 1, "अ": 2, "z": 3, "ौ": 4, "आ": 5, "य": 6, "m": 7, "क": 8, "ण": 9, "्": 10, "c": 11, "ञ": 12, "ज": 13, "द": 14, "व": 15, "ै": 16, "ँ": 17, "ं": 18, "v": 19, "श": 20, "ऐ": 21, "प": 22, "ई": 23, "ओ": 24, "त": 25, "w": 26, "स": 27, "y": 28, "d": 29, "ल": 30, "k": 31, "े": 32, "s": 33, "थ": 34, "ड": 35, "फ": 36, "ु": 37, "i": 38, "ा": 40, "ः": 41, "ऊ": 42, "u": 43, "ए": 44, "औ": 45, "e": 46, "j": 47, "म": 48, "t": 49, "ढ": 50, "ड़": 51, "ी": 52, "ट": 53, "न": 54, "उ": 55, "ध": 56, "ह": 57, "़": 58, "ढ़": 59, "इ": 60, "ू": 61, "भ": 62, "झ": 63, "ृ": 64, "f": 65, "x": 66, "ऑ": 67, "ब": 68, "ष": 69, "ॉ": 70, "l": 71, "छ": 72, "a": 73, "च": 74, "h": 75, "r": 76, "ऋ": 77, "ज़": 78, "ग़": 79, "ग": 80, "ि": 81, "ख": 82, "क़": 83, "'": 84, "n": 85, "o": 86, "b": 87, "g": 88, "p": 89, "ठ": 90, "घ": 91, "ो": 92, "|": 39, "[UNK]": 93, "[PAD]": 94}