Spaces:
Runtime error
Runtime error
HeshamHaroon
commited on
Commit
•
b613c61
1
Parent(s):
00fb4c8
Update app.py
Browse files
app.py
CHANGED
@@ -38,7 +38,7 @@ tokenizer_options = [
|
|
38 |
"inception-mbzuai/jais-13b",
|
39 |
"aubmindlab/bert-base-arabertv2",
|
40 |
"CohereForAI/c4ai-command-r-v01",
|
41 |
-
"CohereForAI/c4ai-command-r-plus"
|
42 |
]
|
43 |
|
44 |
if meta_llama_tokenizer:
|
@@ -73,13 +73,13 @@ def compare_tokenizers(tokenizer_name, text):
|
|
73 |
tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
|
74 |
|
75 |
# Prepare the results to be displayed in HTML format
|
76 |
-
tokens_html = "".join([f"<span style='background-color:#
|
77 |
-
encoded_html = "".join([f"<span style='background-color:#
|
78 |
-
decoded_html = f"<div style='background-color:#
|
79 |
|
80 |
results_html = f"""
|
81 |
<div style='font-family: Arial, sans-serif;'>
|
82 |
-
<h3>Tokenizer: {tokenizer_name}</h3>
|
83 |
<p><strong>Tokens:</strong> {tokens_html}</p>
|
84 |
<p><strong>Encoded:</strong> {encoded_html}</p>
|
85 |
<p><strong>Decoded:</strong> {decoded_html}</p>
|
@@ -89,7 +89,7 @@ def compare_tokenizers(tokenizer_name, text):
|
|
89 |
|
90 |
# Define the Gradio interface components with a dropdown for model selection
|
91 |
inputs_component = [
|
92 |
-
gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
|
93 |
gr.Textbox(lines=2, placeholder="اكتب النص هنا...", label="Input Text")
|
94 |
]
|
95 |
|
|
|
38 |
"inception-mbzuai/jais-13b",
|
39 |
"aubmindlab/bert-base-arabertv2",
|
40 |
"CohereForAI/c4ai-command-r-v01",
|
41 |
+
"CohereForAI/c4ai-command-r-plus"
|
42 |
]
|
43 |
|
44 |
if meta_llama_tokenizer:
|
|
|
73 |
tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
|
74 |
|
75 |
# Prepare the results to be displayed in HTML format
|
76 |
+
tokens_html = "".join([f"<span style='background-color:#f0f0f0; padding:4px; margin:2px; border-radius:3px; border:1px solid #ccc;'>{token}</span>" for token in tokens_display])
|
77 |
+
encoded_html = "".join([f"<span style='background-color:#e0f7fa; padding:4px; margin:2px; border-radius:3px; border:1px solid #00acc1;'>{token}</span>" for token in encoded_output])
|
78 |
+
decoded_html = f"<div style='background-color:#e8f5e9; padding:10px; border-radius:3px; border:1px solid #4caf50;'>{decoded_text}</div>"
|
79 |
|
80 |
results_html = f"""
|
81 |
<div style='font-family: Arial, sans-serif;'>
|
82 |
+
<h3 style='color: #00796b;'>Tokenizer: {tokenizer_name}</h3>
|
83 |
<p><strong>Tokens:</strong> {tokens_html}</p>
|
84 |
<p><strong>Encoded:</strong> {encoded_html}</p>
|
85 |
<p><strong>Decoded:</strong> {decoded_html}</p>
|
|
|
89 |
|
90 |
# Define the Gradio interface components with a dropdown for model selection
|
91 |
inputs_component = [
|
92 |
+
gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer", type="index"),
|
93 |
gr.Textbox(lines=2, placeholder="اكتب النص هنا...", label="Input Text")
|
94 |
]
|
95 |
|