import gradio as gr import os from pathlib import Path import argparse from huggingface_hub import snapshot_download from llama_cpp import Llama repo_name = 'mradermacher/Turkcell-LLM-7b-v1-GGUF' model_file = "Turkcell-LLM-7b-v1.Q5_K_M.gguf" snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_file) DEFAULT_MODEL_PATH = model_file llm = Llama(model_path=model_file, model_type="mistral") def predict(input, chatbot, max_length, top_p, temperature, history): chatbot.append((input, "")) response = "" history.append(input) for output in llm(input, stream=True, temperature=temperature, top_p=top_p, max_tokens=max_length, ): piece = output['choices'][0]['text'] response += piece chatbot[-1] = (chatbot[-1][0], response) yield chatbot, history history.append(response) yield chatbot, history def reset_user_input(): return gr.update(value="") def reset_state(): return [], [] with gr.Blocks() as demo: gr.HTML("""

TurkcellLLM Chatbot Demo

This is unofficial demo of `mradermacher/Turkcell-LLM-7b-v1-GGUF/Q5_K_M.gguf` model based on Mistral architecture.

Hit the like button if you liked! 🤗

""") chatbot = gr.Chatbot() with gr.Row(): with gr.Column(scale=4): user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=8, elem_id="user_input") submitBtn = gr.Button("Submit", variant="primary", elem_id="submit_btn") with gr.Column(scale=1): max_length = gr.Slider(0, 2048, value=1024, step=2.0, label="Maximum Length", interactive=True) top_p = gr.Slider(0, 1, value=0.7, step=0.01, label="Top P", interactive=True) temperature = gr.Slider(0, 1.0, value=0.7, step=0.1, label="Temperature", interactive=True) emptyBtn = gr.Button("Clear History") history = gr.State([]) submitBtn.click( predict, [user_input, chatbot, max_length, top_p, temperature, history], [chatbot, history], show_progress=True ) submitBtn.click(reset_user_input, [], [user_input]) emptyBtn.click(reset_state, outputs=[chatbot, history], show_progress=True) demo.queue().launch(share=False, inbrowser=True)