Update app.py

app.py

@@ -69,8 +69,11 @@ def compare_tokenizers(tokenizer_name, text):
     encoded_output = tokenizer.encode(text, add_special_tokens=True)
     decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
 
+    # Ensure the tokens are properly decoded
+    tokens_display = [token for token in tokens]
+
     # Prepare the results to be displayed in HTML format
-    tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens])
+    tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens_display])
     encoded_html = "".join([f"<span style='background-color:#ADD8E6; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in encoded_output])
     decoded_html = f"<div style='background-color:#90EE90; padding:10px; border-radius:5px;'>{decoded_text}</div>"