File size: 4,580 Bytes
4a27403 346d734 4a27403 346d734 4a27403 d9f8d6b c2d4494 4a27403 c2d4494 4a27403 c2d4494 4a27403 c2d4494 346d734 c2d4494 4a27403 c2d4494 346d734 4a27403 346d734 f1beb5e 946ce87 f1beb5e d56eca7 4a27403 fdb0542 4a27403 1affa63 4a27403 1affa63 4a27403 fdb0542 4a27403 f1beb5e 4a27403 d9f8d6b 4a27403 ddb02b4 4a27403 94867a8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
import os
import gradio as gr
from openai import OpenAI
SYSTEM_PROMPT = "Ты — Сайга, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им."
BASE_URL = os.getenv("BASE_URL")
API_KEY = os.getenv("API_KEY")
MODEL_NAME = "IlyaGusev/saiga_nemo_12b_gptq_8bit"
CLIENT = OpenAI(base_url=BASE_URL, api_key=API_KEY)
def user(message, history):
new_history = history + [[message, None]]
return "", new_history
def bot(
history,
system_prompt,
top_p,
temp
):
messages = [{"role": "system", "content": SYSTEM_PROMPT}]
for user_message, bot_message in history[:-1]:
messages.append({"role": "user", "content": user_message})
if bot_message:
messages.append({"role": "assistant", "content": bot_message})
last_user_message = history[-1][0]
messages.append({"role": "user", "content": last_user_message})
response = CLIENT.chat.completions.create(
model=MODEL_NAME,
messages=messages,
temperature=temp,
top_p=top_p,
stream=True,
)
partial_text = ""
for chunk in response:
content = chunk.choices[0].delta.content
partial_text += content
history[-1][1] = partial_text
yield history
with gr.Blocks(
theme=gr.themes.Soft()
) as demo:
favicon = '<img src="https://cdn.midjourney.com/b88e5beb-6324-4820-8504-a1a37a9ba36d/0_1.png" width="48px" style="display: inline">'
gr.Markdown(
f"""<h1><center>{favicon}Saiga Nemo 12B GPTQ 8 bit</center></h1>
This is a demo of a **Russian**-speaking Mistral Nemo based model.
Это демонстрационная версия [Сайги Немо с 12 миллиардами параметров](https://huggingface.co/IlyaGusev/saiga_nemo_12b).
"""
)
with gr.Row():
with gr.Column(scale=5):
system_prompt = gr.Textbox(label="Системный промпт", placeholder="", value=SYSTEM_PROMPT, interactive=False)
chatbot = gr.Chatbot(label="Диалог")
with gr.Column(min_width=80, scale=1):
with gr.Tab(label="Параметры генерации"):
top_p = gr.Slider(
minimum=0.0,
maximum=1.0,
value=0.9,
step=0.05,
interactive=True,
label="Top-p",
)
temp = gr.Slider(
minimum=0.0,
maximum=2.0,
value=0.01,
step=0.01,
interactive=True,
label="Температура"
)
with gr.Row():
with gr.Column():
msg = gr.Textbox(
label="Отправить сообщение",
placeholder="Отправить сообщение",
show_label=False,
)
with gr.Column():
with gr.Row():
submit = gr.Button("Отправить")
stop = gr.Button("Остановить")
clear = gr.Button("Очистить")
with gr.Row():
gr.Markdown(
"""ПРЕДУПРЕЖДЕНИЕ: Модель может генерировать фактически или этически некорректные тексты. Мы не несём за это ответственность."""
)
# Pressing Enter
submit_event = msg.submit(
fn=user,
inputs=[msg, chatbot],
outputs=[msg, chatbot],
queue=False,
).success(
fn=bot,
inputs=[
chatbot,
system_prompt,
top_p,
temp
],
outputs=chatbot,
queue=True,
)
# Pressing the button
submit_click_event = submit.click(
fn=user,
inputs=[msg, chatbot],
outputs=[msg, chatbot],
queue=False,
).success(
fn=bot,
inputs=[
chatbot,
system_prompt,
top_p,
temp
],
outputs=chatbot,
queue=True,
)
# Stop generation
stop.click(
fn=None,
inputs=None,
outputs=None,
cancels=[submit_event, submit_click_event],
queue=False,
)
# Clear history
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue(max_size=128)
demo.launch(show_error=True)
|