Update app.py

app.py

@@ -55,20 +55,40 @@ def generate_colored_html(tokens, decoded_tokens):
     return html_output
 
 def tokenize_text(text, tokenizer_name):
-    if tokenizer_name.split("/")[0]=="Xenova":
+    if tokenizer_name.split("/")[0] == "Xenova":
         tokenizer = GPT2TokenizerFast.from_pretrained(tokenizer_name)
-    else :
+    else:
         tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+    
     tokens = tokenizer.encode(text, add_special_tokens=True)
     decoded_tokens = [tokenizer.decode(token) for token in tokens]
     html_output = generate_colored_html(tokens, decoded_tokens)
-    return html_output
+    
+    # Calculate tokenizer metrics
+    total_tokens = len(tokens)
+    unique_tokens = len(set(tokens))
+    vocab_size = len(tokenizer.get_vocab())
+    avg_token_length = sum(len(t) for t in decoded_tokens) / total_tokens if total_tokens > 0 else 0
+    
+    metrics = f"""
+    <h4>Tokenizer Metrics:</h4>
+    <ul>
+        <li>Total Tokens: {total_tokens}</li>
+        <li>Unique Tokens: {unique_tokens}</li>
+        <li>Vocabulary Size: {vocab_size}</li>
+        <li>Average Token Length: {avg_token_length:.2f} characters</li>
+    </ul>
+    """
+    
+    return html_output, metrics
 
 def compare_tokenizers(text, selected_tokenizers):
     results = ""
     for tokenizer_name in selected_tokenizers:
         results += f"<h3>{tokenizer_name}</h3>"
-        results += tokenize_text(text, tokenizer_name)
+        html_output, metrics = tokenize_text(text, tokenizer_name)
+        results += html_output
+        results += metrics
         results += "<hr>"
     return results
 
@@ -81,7 +101,7 @@ iface = gr.Interface(
     ],
     outputs=gr.HTML(label="Tokenization Results"),
     title="Tokenizer Comparison",
-    description="Compare tokenization results from different tokenizers.",
+    description="Compare tokenization results and metrics from different tokenizers.",
     examples=[
         ["Allergies: \nNo Known Allergies / Adverse Drug Reactions\n\nChief Complaint: \nRight hand erythema/edema\n\nHistory of Present Illness: \nThe patient is a ___ y/o M with PMHx significant", ["allenai/longformer-base-4096", "gpt2"]],
         ["In the ED, initial vitals: 98.3 80 144/90 18 96% ra  \nLabs notable for: Leukocytosis to 12.5.  The patient endorsed \nsevere R hand tenderness but otherwise denied any fevers, \nchills, nausea, vomiting, numbness, tingling or weakness.  Hand \nsurgery was consulted and recommended admission to medicine for \nIV antibiotics, possible ultrasound to evaluate for \nthrombophlebitis and pain control.  In the ED, patient received \nZosyn and IV dilaudid 0.5mg x1.\n\nUpon arrival to the floor, patient endorses R hand pain- stable \nfrom prior.  \n\nROS:  Per HPI \n\n\nPast Medical History: \nCrohn's disease\nAnal fissure\nh/o DVT on coumadin\nRotator cuff repair in ___\n\n\nFamily History: \nNo family h/o IBD\n\n\nMedications on Admission:  EXAMINATION:  UNILAT UP EXT VEINS US RIGHT\n\nINDICATION:  ___ year old man with concern for R hand cellulitis vs.\nthrombophlebitis ", ["roberta-base", "allenai/longformer-base-4096"]],