Update app.py

app.py

@@ -90,12 +90,16 @@ title = "Ghalib doing tiktok"
 description = "A simple Gradio interface to infer urdu tokenizer"
 
 tokenizer = Tokenizer()
+temp = Tokenizer()
 with open('merges.pkl', 'rb') as files:
     tokenizer.vocab = pickle.load(files)
 with open('vocab.pkl', 'rb') as files:
     tokenizer.merges = pickle.load(files)
 
 def inference(text):
+    print(len(tokenizer.merges))
+    print(len(tokenizer.vocab))
+    print(len(temp.encode(text)) / len(tokenizer.encode(text)))
     return tokenizer.encode(text)
 
 iface = gr.Interface(