Update app.py

app.py

@@ -2,14 +2,14 @@ import os
 import time
 import spaces
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM, TextIteratorStreamer
 import gradio as gr
 from threading import Thread
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-MODEL = "LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct"
+MODEL = "NousResearch/Hermes-3-Llama-3.2-3B"
 
-TITLE = "<h1><center>EXAONE-3.5-7.8B-Instruct</center></h1>"
+TITLE = "<h1><center>Hermes-3-Llama-3.2-3B</center></h1>"
 
 PLACEHOLDER = """
 <center>
@@ -35,9 +35,11 @@ device = "cuda" # for GPU usage or "cpu" for CPU usage
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL,
-    torch_dtype=torch.bfloat16,
+    torch_dtype=torch.float16,
     device_map="auto",
-    trust_remote_code=True)
+    load_in_8bit=False,
+    load_in_4bit=True,
+    use_flash_attention_2=True)
 
 @spaces.GPU()
 def stream_chat(
@@ -74,7 +76,7 @@ def stream_chat(
         do_sample = False,
         top_p = top_p,
         top_k = top_k,
-        eos_token_id=tokenizer.eos_token_id,
+        eos_token_id = 128039,
         temperature = temperature,
         repetition_penalty=penalty,
         streamer=streamer,