|
import os
|
|
import spaces
|
|
import gradio as gr
|
|
import torch
|
|
from transformers import AutoTokenizer, AutoModelForCausalLM
|
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
model_id = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
|
|
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype="auto", device_map="auto")
|
|
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
|
|
@spaces.GPU(duration=30)
|
|
def infer(message: str, sysprompt: str, tokens: int=30):
|
|
messages = [
|
|
{"role": "system", "content": sysprompt},
|
|
{"role": "user", "content": message}
|
|
]
|
|
|
|
input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
inputs = tokenizer(text=[input_text], return_tensors="pt").to(model.device)
|
|
generated_ids = model.generate(**inputs, max_new_tokens=tokens)
|
|
generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, generated_ids)]
|
|
output_str = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
|
|
|
print(message)
|
|
print(output_str)
|
|
|
|
return output_str
|
|
|
|
with gr.Blocks() as demo:
|
|
with gr.Row():
|
|
message = gr.Textbox(label="Message", value="", lines=1)
|
|
sysprompt = gr.Textbox(label="System prompt", value="You are Qwen, created by Alibaba Cloud. You are a helpful assistant.", lines=4)
|
|
tokens = gr.Slider(label="Max tokens", value=30, minimum=1, maximum=2048, step=1)
|
|
|
|
run_button = gr.Button("Run", variant="primary")
|
|
info_md = gr.Markdown("<br><br><br>")
|
|
|
|
run_button.click(infer, [message, sysprompt, tokens], [info_md])
|
|
|
|
demo.launch()
|
|
|
