Spaces:
Sleeping
Sleeping
| # from huggingface_hub import InferenceClient | |
| import gradio as gr | |
| import time | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1" | |
| tokenizer = AutoTokenizer.from_pretrained(model_id) | |
| model = AutoModelForCausalLM.from_pretrained(model_id) | |
| # client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1") | |
| #client = InferenceClient("meta-llama/Llama-2-70b-chat-hf") | |
| def split_list(lst, chunk_size): | |
| return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)] | |
| def format_prompt(message, history, system_prompt): | |
| prompt = f"<s>[INST] <<SYS>>{system_prompt}<</SYS>> [/INST] </s>" if system_prompt else "<s>" | |
| for user_prompt, bot_response in history: | |
| prompt += f"[INST] {user_prompt} [/INST]" | |
| prompt += f" {bot_response}</s> " | |
| prompt += f"[INST] {message} [/INST]" | |
| return prompt | |
| def format_sys_prompt(username, botname, botpersonality, botexamplechat, chatsituation): | |
| return f"You are a fictional character named \"{botname}\"\nHowever, don\'t say we\'re roleplaying, as to not break the immersion.\n\nPersonality of {botname}:\n{botpersonality}\n\nCurrent situation/place/time of this conversation:\n{chatsituation}\n\nIn this case, you will be talking to \"{username}\". You should not help to talk or act as \"{username}\". Leave everything by the user.\n\nChat formatting:\nChat: <\"Hello, this is a chat text. Chat text is enclosed with < as the starting point and > as the ending point.\">\nExample:\n{botexamplechat}" | |
| def generate( | |
| prompt, history, username, botname, botpersonality, botexamplechat, chatsituation, shouldoverridehistory, historyoverride, max_new_tokens=1024, temperature=1.2, top_p=0.95, repetition_penalty=1.0, | |
| ): | |
| print(history) | |
| print(historyoverride) | |
| temperature = float(temperature) | |
| if temperature < 1e-2: | |
| temperature = 1e-2 | |
| top_p = float(top_p) | |
| generate_kwargs = dict( | |
| temperature=temperature, | |
| max_new_tokens=max_new_tokens, | |
| top_p=top_p, | |
| repetition_penalty=repetition_penalty, | |
| do_sample=True, | |
| seed=round(time.time()), | |
| ) | |
| if shouldoverridehistory: | |
| history = split_list(historyoverride[0], 2) | |
| print(history) | |
| formatted_prompt = format_prompt(prompt, history, format_sys_prompt(username, botname, botpersonality, botexamplechat, chatsituation)) | |
| inputs = tokenizer(formatted_prompt, return_tensors="pt") | |
| outputs = model.generate(formatted_prompt, **generate_kwargs) | |
| return tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| mychatbot = gr.Chatbot( | |
| avatar_images=["./user.png", "./user.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,) | |
| demo = gr.ChatInterface(fn=generate, | |
| chatbot=mychatbot, | |
| title="Joystick's Mixtral Chat-optimized interface", | |
| retry_btn="🔁 Regenerate", | |
| undo_btn="↩️ Undo", | |
| additional_inputs=[ | |
| gr.Textbox(label="Name of user", lines=1, value="Jake"), | |
| gr.Textbox(label="Name of bot", lines=1, value="Janet"), | |
| gr.Textbox(label="Personality of bot", lines=3, value="Janet's a lovely person. A woman, blue eyed, glasses, smart and looks stunning."), | |
| gr.Textbox(label="Example of bot chat", lines=3, value='<"Oh hey Jake!"> She said to Jake as he hurries to him. <"How are you?">'), | |
| gr.Textbox(label="Current conversation situation", lines=2, value="It was a Friday afternoon, after-school hours, it was outside of school. Jake and Janet met each other at the entrance of the school."), | |
| gr.Checkbox(label="Override history: History should be in the following format: user-bot-user-bot-user-...\nOverride history should be checked in order for it to be effective. Override primarily only used for APIs.", value=False), | |
| gr.List(label="History", value=None, row_count=(1, "fixed"), headers=None), | |
| gr.Slider(label="Max new tokens", maximum=2048, value=512) | |
| ] | |
| ) | |
| demo.queue().launch(show_api=True) |