Update app.py

app.py

@@ -1,39 +1,42 @@
 import gradio as gr
-from transformers import pipeline, set_seed
+from transformers import pipeline, AutoTokenizer
 import torch
 
-# Function to generate responses using the entire conversation history
-def generate_response(messages, model_name, sampling_temperature, max_tokens, top_p):
-    generator = pipeline('text-generation', model=model_name, torch_dtype=torch.float16)
-    set_seed(42)  # You can set a different seed for reproducibility
-    
-    # Combine entire conversation history
-    conversation = ""
-    for message in messages:
-        role = message['role']
-        content = message['content']
-        conversation += f"<|im_start|>{role}\n{content}<|im_end|>\n"
-    
-    # Generate response
-    response = generator(conversation, max_length=2048, temperature=sampling_temperature, max_tokens=max_tokens, top_p=top_p, repetition_penalty=1.1, top_k=12)
-    
-    return [{'content': response[0]['generated_text'], 'role': 'assistant'}]
+def load_model(model_name):
+    return pipeline("text-generation", model=model_name, device="cuda", torch_dtype=torch.float16)
 
-# Gradio chatbot interface with conversation history
-iface = gr.Interface(
-    fn=generate_response,
+def generate(
+    model_name,
+    user_input,
+    temperature=0.4,
+    top_p=0.95,
+    top_k=50,
+    max_new_tokens=256,
+):
+    pipe = load_model(model_name)
+
+    # Set tokenize correctly. Otherwise ticking the box breaks it.
+    prompt = f"<|im_start|>user\n{user_input}<|im_end|>\n"
+    outputs = pipe(prompt, max_new_tokens=max_new_tokens, do_sample=True,
+                   temperature=temperature, top_k=top_k, top_p=top_p, repetition_penalty=1.10)
+    return outputs[0]["generated_text"]
+
+model_choices = ["Locutusque/UltraQwen-7B", "Locutusque/UltraQwen-1_8B", "Locutusque/TinyMistral-248M-v2.5-Instruct", "M4-ai/TinyMistral-6x248M-Instruct"]
+# What at the best options? 
+g = gr.Interface(
+    fn=generate,
     inputs=[
-        gr.Chat("You", "Chatbot"),
-        gr.Dropdown("Select Model", ["Locutusque/TinyMistral-248M-v2.5-Instruct", "Locutusque/Hercules-1.0-Mistral-7B", "Locutusque/UltraQwen-1_8B"]),
-        gr.Slider("Sampling Temperature", 0.1, 2.0, 1.0, 0.1),
-        gr.Slider("Max Tokens", 5, 200, 50, 5),
-        gr.Slider("Top P", 0.1, 0.5, 0.75, 0.1)
+        gr.components.Dropdown(choices=model_choices, label="Model", value=model_choices[0], interactive=True),
+        gr.components.Textbox(lines=2, label="Prompt", value="How many planets are in our solar system?"),
+        gr.components.Slider(minimum=0, maximum=1, value=0.4, label="Temperature"),
+        gr.components.Slider(minimum=0, maximum=1, value=0.95, label="Top p"),
+        gr.components.Slider(minimum=0, maximum=100, step=1, value=50, label="Top k"),
+        gr.components.Slider(minimum=1, maximum=1024, step=1, value=256, label="Max tokens"),  
     ],
-    outputs=gr.Chat(role="Chatbot"),
-    live=True,
-    capture_session=True
+    outputs=[gr.Textbox(lines=10, label="Output")],
+    title="Hugging Face Transformers Model",
+    description="A simple interface for generating text with a Hugging Face Transformers model.",
+    concurrency_limit=1
 )
 
-# Launch Gradio chatbot interface
-iface.launch()
-
+g.launch(max_threads=2)