add tokenizer

runs/Nov07_08-21-50_c07766eb6e6a/events.out.tfevents.1636273349.c07766eb6e6a.77.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:836e7dbb92927d3a9fd0e530813e8dad226b62521b97b3919c766965b6e61b08
-size 4919
+oid sha256:2355ff8bcb91fb0777a22bd6a8c6c2952ef0aa699ed970791105e28e8be6c834
+size 5273

vocab.json

@@ -1 +1 @@
-{"ँ": 0, "ु": 1, "।": 2, "त": 3, "ओ": 4, "े": 5, "ं": 6, "l": 7, "ज": 8, "अ": 9, "आ": 10, "ू": 11, "ध": 12, "्": 13, "f": 14, "थ": 15, "द": 16, "w": 17, "प": 18, "ग": 19, "ह": 20, "ख": 21, "r": 22, "u": 23, "श": 24, "ौ": 25, "ण": 26, "ॉ": 27, "i": 28, "ब": 29, "इ": 30, "फ": 31, "़": 32, "उ": 33, "र": 34, "e": 35, "ढ": 36, "ि": 37, "च": 38, "ठ": 39, "ई": 40, "ो": 41, "p": 42, "घ": 43, "औ": 44, "ै": 45, "क": 46, "m": 47, "ऐ": 48, "ऑ": 49, "'": 50, "ऊ": 51, "ए": 53, "म": 54, "a": 55, "य": 56, "ा": 57, "ड़": 58, "व": 59, "ृ": 60, "ड": 61, "ट": 62, "झ": 63, "ष": 64, "छ": 65, "भ": 66, "ः": 67, "ी": 68, "स": 69, "न": 70, "ल": 71, "|": 52, "[UNK]": 72, "[PAD]": 73}
+{"क": 0, "य": 1, "फ": 2, "़": 3, "ख": 4, "झ": 5, "ऑ": 6, "आ": 7, "ी": 8, "ँ": 9, "i": 10, "प": 11, "e": 12, "ग": 13, "ऊ": 14, "व": 15, "थ": 16, "u": 17, "श": 18, "w": 19, "अ": 20, "ौ": 21, "ज": 22, "ओ": 23, "इ": 24, "ि": 25, "स": 26, "च": 27, "f": 28, "ं": 29, "े": 30, "ठ": 31, "ब": 32, "r": 33, "ण": 34, "द": 35, "उ": 36, "l": 37, "ड़": 38, "ह": 39, "्": 40, "ट": 41, "म": 42, "त": 43, "p": 44, "a": 45, "ॉ": 46, "ृ": 47, "ः": 48, "ल": 49, "छ": 50, "ु": 51, "।": 52, "औ": 53, "ा": 54, "ढ": 55, "'": 56, "ई": 57, "ध": 58, "ष": 59, "र": 60, "भ": 61, "ू": 62, "न": 63, "ड": 65, "ए": 66, "ो": 67, "ऐ": 68, "ै": 69, "घ": 70, "m": 71, "|": 64, "[UNK]": 72, "[PAD]": 73}