Update app.py

app.py

@@ -2,14 +2,29 @@ from gradio import Interface
 import gradio as gr
 import aranizer
 from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, HfFolder, logging
+import os
+
+# Retrieve your Hugging Face token from the environment variable
+HF_TOKEN = os.getenv('HF_TOKEN')
+
+if HF_TOKEN:
+    HfFolder.save_token(HF_TOKEN)
+    os.environ["HUGGINGFACEHUB_API_TOKEN"] = HF_TOKEN
 
 # Load additional tokenizers from transformers
 gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
 gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
 jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
 arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
-meta_llama_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
+
+# Try to load the gated tokenizer
+try:
+    meta_llama_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
+except Exception as e:
+    meta_llama_tokenizer = None
+    logging.warning(f"Could not load meta-llama/Meta-Llama-3-8B tokenizer: {e}")
+
 cohere_command_r_v01_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")
 cohere_command_r_plus_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-plus")
 
@@ -21,11 +36,13 @@ tokenizer_options = [
     "FreedomIntelligence/AceGPT-7B",   
     "inception-mbzuai/jais-13b",
     "aubmindlab/bert-base-arabertv2",
-    "meta-llama/Meta-Llama-3-8B",
     "CohereForAI/c4ai-command-r-v01",
     "CohereForAI/c4ai-command-r-plus"   
 ]
 
+if meta_llama_tokenizer:
+    tokenizer_options.append("meta-llama/Meta-Llama-3-8B")
+
 tokenizers = {
     "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
     "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
@@ -38,11 +55,13 @@ tokenizers = {
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
     "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
     "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer,
-    "meta-llama/Meta-Llama-3-8B": lambda: meta_llama_tokenizer,
     "CohereForAI/c4ai-command-r-v01": lambda: cohere_command_r_v01_tokenizer,
     "CohereForAI/c4ai-command-r-plus": lambda: cohere_command_r_plus_tokenizer
 }
 
+if meta_llama_tokenizer:
+    tokenizers["meta-llama/Meta-Llama-3-8B"] = lambda: meta_llama_tokenizer
+
 def compare_tokenizers(tokenizer_name, text):
     # Handle the transformer tokenizers separately due to API differences
     if tokenizer_name in [