Update app.py

app.py

@@ -70,7 +70,7 @@ def compare_tokenizers(tokenizer_name, text):
     decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
 
     # Ensure the tokens are properly decoded
-    tokens_display = [token for token in tokens]
+    tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
 
     # Prepare the results to be displayed in HTML format
     tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens_display])