Update app.py

app.py

@@ -2,13 +2,15 @@ from transformers import AutoTokenizer
 import gradio as gr
 
 
-gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
-gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
-llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
-falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
-phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
-t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
-mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
+def load_tokenizers()
+    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+    gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+    llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
+    falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
+    phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
+    t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
+    mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
+
 
 def tokenize(input_text):
     gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
@@ -21,5 +23,9 @@ def tokenize(input_text):
 
     return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}\nMistral: {len(mistral_tokens)}"
 
-iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
-iface.launch()
+
+if __name__ == "__main__":
+    load_tokenizers()
+    
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
+    iface.launch()