Update app.py

app.py

@@ -1,29 +1,17 @@
 from transformers import AutoTokenizer
 import gradio as gr
 
-def tokenize(input_text):
-    llama_tokens = len(llama_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    mistral_tokens = len(mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    gpt_neox_tokens = len(gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    falcon_tokens = len(falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-
-    token_lengths = {
-        "LLaMa": llama_tokens,
-        "Mistral": mistral_tokens,
-        "GPT-2/GPT-J": gpt2_tokens,
-        "GPT-NeoX": gpt_neox_tokens,
-        "Falcon": falcon_tokens,
-        "Phi-2": phi2_tokens,
-        "T5": t5_tokens
-    }
 
-    sorted_tokens = sorted(token_lengths.items(), key=lambda x: x[1], reverse=True)
-    result = "\n".join([f"{name}: {length}" for name, length in sorted_tokens])
+def tokenize(input_text):
+    llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
 
-    return result
+    return f"LLaMa: {len(llama_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}"
 
 
 if __name__ == "__main__":