Update app.py

app.py

@@ -2,22 +2,23 @@ import torch
 from PIL import Image
 import gradio as gr
 import spaces
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer,BitsAndBytesConfig
 import os
 from threading import Thread
 
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-MODEL_ID = "Qwen/Qwen2-7B-Instruct"
+MODEL_ID = "google/gemma-2-27b-it"
 MODELS = os.environ.get("MODELS")
 MODEL_NAME = MODELS.split("/")[-1]
+MAX_INPUT_TOKEN_LENGTH = int(os.environ.get("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
 TITLE = "<h1><center>Qwen2-Chatbox</center></h1>"
 
 DESCRIPTION = f"""
 <h3>MODEL: <a href="https://hf.co/{MODELS}">{MODEL_NAME}</a></h3>
 <center>
-<p>Qwen is the large language model built by Alibaba Cloud.
+<p>Gemma is the large language model built by Google.
 <br>
 Feel free to test without log.
 </p>
@@ -37,13 +38,15 @@ h3 {
 """
 
 model = AutoModelForCausalLM.from_pretrained(
-          MODELS,
-          torch_dtype=torch.float16,
-          device_map="auto",
+    MODELS,
+    device_map="auto",
+    quantization_config=BitsAndBytesConfig(load_in_4bit=True)
         )
-tokenizer = AutoTokenizer.from_pretrained(MODELS)
+tokenizer = GemmaTokenizerFast.from_pretrained(MODELS)
+model.config.sliding_window = 4096
+model.eval()
 
-@spaces.GPU
+@spaces.GPU(duration=90)
 def stream_chat(message: str, history: list, temperature: float, max_new_tokens: int, top_p: float, top_k: int, penalty: float):
     print(f'message is - {message}')
     print(f'history is - {history}')
@@ -54,13 +57,16 @@ def stream_chat(message: str, history: list, temperature: float, max_new_tokens:
 
     print(f"Conversation is -\n{conversation}")
     
-    input_ids = tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
-    inputs = tokenizer(input_ids, return_tensors="pt").to(0)
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(0)
     
-    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
 
     generate_kwargs = dict(
-        inputs, 
+        {"input_ids": input_ids},
         streamer=streamer,
         top_k=top_k,
         top_p=top_p,
@@ -68,7 +74,8 @@ def stream_chat(message: str, history: list, temperature: float, max_new_tokens:
         max_new_tokens=max_new_tokens, 
         do_sample=True, 
         temperature=temperature,
-        eos_token_id = [151645, 151643],
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
     )
     
     thread = Thread(target=model.generate, kwargs=generate_kwargs)
@@ -81,9 +88,9 @@ def stream_chat(message: str, history: list, temperature: float, max_new_tokens:
 
 
 
-chatbot = gr.Chatbot(height=450)
+chatbot = gr.Chatbot(height=600)
 
-with gr.Blocks(css=CSS) as demo:
+with gr.Blocks(css=CSS, theme="soft") as demo:
     gr.HTML(TITLE)
     gr.HTML(DESCRIPTION)
     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
@@ -103,7 +110,7 @@ with gr.Blocks(css=CSS) as demo:
             ),
             gr.Slider(
                 minimum=128,
-                maximum=4096,
+                maximum=2048,
                 step=1,
                 value=1024,
                 label="Max new tokens",