File size: 1,174 Bytes
9e2d04e 3f45785 5d295fa cf610a7 a268dd1 3f45785 9e2d04e 3f45785 9e2d04e 3f45785 9e2d04e 3f45785 aa251d2 3f45785 9e2d04e 3f45785 9e2d04e 3f45785 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
from ctransformers import AutoModelForCausalLM
import gradio as gr
def generate_prompt(history):
prompt = " "
for chain in history[-2:-1]:
prompt += f"<human>: {chain[0]}\n<bot>: {chain[1]}{end_token}\n"
prompt += f"<human>: {history[-1][0]}\n<bot>:"
return prompt
def generate(history):
prompt = generate_prompt(history)
streamer = llm(prompt, stream=True, temperature=0, repetition_penalty=1.2)
return streamer
llm = AutoModelForCausalLM.from_pretrained("theodotus/llama-uk", model_file="model.bin", model_type='llama')
end_token = "</s>"
with gr.Blocks() as demo:
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.Button("Clear")
def user(user_message, history):
return "", history + [[user_message, ""]]
def bot(history):
streamer = generate(history)
for token in streamer:
history[-1][1] += token
yield history
msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
bot, chatbot, chatbot
)
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue()
if __name__ == "__main__":
demo.launch() |