model select

app_local.py

@@ -12,16 +12,27 @@ import gradio as gr
 llm = None
 llm_model = None
 
+# Dropdown for Model Selection
+model_dropdown = gr.Dropdown(
+    [
+        'qwen2-0_5b-instruct-q4_k_m.gguf',
+        'qwen2_500m.gguf',
+        'mistrallite.Q4_K_M.gguf',
+    ],
+    value="qwen2-0_5b-instruct-q4_k_m.gguf",
+    label="Model"
+)
+
 def respond(
     message,
     history: list[tuple[str, str]],
-    selected_model,  # This is now a parameter received from the interface
-    system_message,
+        system_message,
     max_tokens,
     temperature,
     top_p,
     top_k,
     repeat_penalty,
+	selected_model,  # This is now a parameter received from the interface
 ):
     chat_template = MessagesFormatterType.GEMMA_2
 
@@ -86,56 +97,48 @@ def respond(
 description = """<p align="center">Defaults to Qwen 500M</p>
 """
 
-# Dropdown for Model Selection
-model_dropdown = gr.Dropdown(
-    [
-        'qwen2-0_5b-instruct-q4_k_m.gguf',
-        'qwen2_500m.gguf',
-        'mistrallite.Q4_K_M.gguf',
-    ],
-    value="qwen2-0_5b-instruct-q4_k_m.gguf",
-    label="Model"
-)
+# Create the Gradio interface
+with gr.Blocks() as demo:  # Create a Gradio Blocks context
+    
+    # Model selection dropdown above the chat
+    model_dropdown.render()
 
-# Create the main chat interface
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        model_dropdown,  # Pass the selected model from Dropdown
-        gr.Textbox(value="You are a helpful assistant.", label="System message"),
-        gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p",
-        ),
-        gr.Slider(
-            minimum=0,
-            maximum=100,
-            value=40,
-            step=1,
-            label="Top-k",
-        ),
-        gr.Slider(
-            minimum=0.0,
-            maximum=2.0,
-            value=1.1,
-            step=0.1,
-            label="Repetition penalty",
-        ),
-    ],
-    retry_btn="Retry",
-    undo_btn="Undo",
-    clear_btn="Clear",
-    submit_btn="Send",
-    title="Chat with Qwen 2 and friends using llama.cpp",
-    description=description,
-    chatbot=gr.Chatbot(
-        scale=1,
-        likeable=False,
-        show_copy_button=True
+    # Main chat interface
+    chat_interface = gr.ChatInterface(
+        respond,
+        additional_inputs=[
+            gr.Textbox(value="You are a helpful assistant.", label="System message"),
+            gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
+            gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+            gr.Slider(
+                minimum=0.1,
+                maximum=1.0,
+                value=0.95,
+                step=0.05,
+                label="Top-p",
+            ),
+            gr.Slider(
+                minimum=0,
+                maximum=100,
+                value=40,
+                step=1,
+                label="Top-k",
+            ),
+            gr.Slider(
+                minimum=0.0,
+                maximum=2.0,
+                value=1.1,
+                step=0.1,
+                label="Repetition penalty",
+            ),
+            model_dropdown  # Pass the dropdown directly
+        ],
+        retry_btn="Retry",
+        undo_btn="Undo",
+        clear_btn="Clear",
+        submit_btn="Send",
+        title="Chat with Qwen 2 and friends using llama.cpp",
+        description=description,
     )
-).launch()
+
+demo.queue().launch()