code

app.py

@@ -11,7 +11,7 @@ os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
 
 model = AutoModelForCausalLM.from_pretrained(
     "NyxKrage/Microsoft_Phi-4", 
-    device_map="auto", 
+    device_map="cuda", 
     torch_dtype="auto", 
     trust_remote_code=True, 
 )
@@ -49,9 +49,9 @@ def respond(
         max_new_tokens=max_tokens,
         temperature=temperature,
         streamer=streamer,
-        top_p=top_p,
+        # top_p=top_p,
         return_full_text=False,
-        do_sample=True,
+        do_sample=False,
     )
     
     response = ""
@@ -60,7 +60,7 @@ def respond(
     thread.start()
     # Print the generated text in real-time
     for new_text in streamer:
-        yield new_text
+        yield tokenizer.decode(new_text, skip_special_tokens=True)
     
 with gr.Blocks() as demo:
     with gr.Row():
@@ -79,7 +79,7 @@ with gr.Blocks() as demo:
             system_message = gr.Textbox(value="You are a friendly Chatbot.", label="System message")
             max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
             temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
+            # top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
             seed = gr.Slider(minimum=0, maximum=20091114, value=42, step=1, label="Seed")
         
         with gr.Column():