.

utils.py

@@ -1,6 +1,6 @@
 from transformers import AutoTokenizer
 import logging
-from transformers import GemmaTokenizer  # Import GemmaTokenizer
+import os
 
 # Configure logging
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -8,10 +8,7 @@ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %
 def get_tokenizer(id):
     logging.debug(f"Loading tokenizer: {id}")
     try:
-        if "gemma" in id.lower():
-            tokenizer = GemmaTokenizer.from_pretrained(id)
-        else:
-            tokenizer = AutoTokenizer.from_pretrained(id, trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(id, trust_remote_code=True, token=os.getenv("HF_TOKEN"))
         logging.debug(f"Tokenizer loaded: {tokenizer}")
         return tokenizer
     except Exception as e: