import gradio as gr import json import requests import os from text_generation import Client, InferenceAPIClient # Load pre-trained model and tokenizer - for THUDM model from transformers import AutoModel, AutoTokenizer tokenizer_glm = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) model_glm = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model_glm = model_glm.eval() # Load pre-trained model and tokenizer for Chinese to English translator from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer model_chtoen = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") tokenizer_chtoen = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M") #Streaming endpoint for OPENAI ChatGPT API_URL = "https://api.openai.com/v1/chat/completions" #Streaming endpoint for OPENCHATKIT API_URL_TGTHR = os.getenv('API_URL_TGTHR') openchat_preprompt = ( "\n: Hi!\n: My name is Bot, model version is 0.15, part of an open-source kit for " "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source " "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, " "but I am programmed to be helpful, polite, honest, and friendly.\n") #Predict function for CHATGPT def predict_chatgpt(inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt=[], history=[]): #Define payload and header for chatgpt API payload = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": f"{inputs}"}], "temperature" : 1.0, "top_p":1.0, "n" : 1, "stream": True, "presence_penalty":0, "frequency_penalty":0, } headers = { "Content-Type": "application/json", "Authorization": f"Bearer {openai_api_key}" } #debug #print(f"chat_counter_chatgpt - {chat_counter_chatgpt}") #Handling the different roles for ChatGPT if chat_counter_chatgpt != 0 : messages=[] for data in chatbot_chatgpt: temp1 = {} temp1["role"] = "user" temp1["content"] = data[0] temp2 = {} temp2["role"] = "assistant" temp2["content"] = data[1] messages.append(temp1) messages.append(temp2) temp3 = {} temp3["role"] = "user" temp3["content"] = inputs messages.append(temp3) payload = { "model": "gpt-3.5-turbo", "messages": messages, #[{"role": "user", "content": f"{inputs}"}], "temperature" : temperature_chatgpt, #1.0, "top_p": top_p_chatgpt, #1.0, "n" : 1, "stream": True, "presence_penalty":0, "frequency_penalty":0, } chat_counter_chatgpt+=1 history.append(inputs) # make a POST request to the API endpoint using the requests.post method, passing in stream=True response = requests.post(API_URL, headers=headers, json=payload, stream=True) token_counter = 0 partial_words = "" counter=0 for chunk in response.iter_lines(): #Skipping the first chunk if counter == 0: counter+=1 continue # check whether each line is non-empty if chunk.decode() : chunk = chunk.decode() # decode each line as response data is in bytes if len(chunk) > 13 and "content" in json.loads(chunk[6:])['choices'][0]["delta"]: partial_words = partial_words + json.loads(chunk[6:])['choices'][0]["delta"]["content"] if token_counter == 0: history.append(" " + partial_words) else: history[-1] = partial_words chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2) ] # convert to tuples of list token_counter+=1 yield chat, history, chat_counter_chatgpt # this resembles {chatbot: chat, state: history} #Predict function for OPENCHATKIT def predict_together(model: str, inputs: str, top_p: float, temperature: float, top_k: int, repetition_penalty: float, watermark: bool, chatbot, history,): client = Client(os.getenv("API_URL_TGTHR")) #get_client(model) # debug #print(f"^^client is - {client}") user_name, assistant_name = ": ", ": " preprompt = openchat_preprompt sep = '\n' history.append(inputs) past = [] for data in chatbot: user_data, model_data = data if not user_data.startswith(user_name): user_data = user_name + user_data if not model_data.startswith("\n" + assistant_name): model_data = "\n" + assistant_name + model_data past.append(user_data + model_data.rstrip() + "\n") if not inputs.startswith(user_name): inputs = user_name + inputs total_inputs = preprompt + "".join(past) + inputs + "\n" + assistant_name.rstrip() # truncate total_inputs #total_inputs = total_inputs[-1000:] partial_words = "" for i, response in enumerate(client.generate_stream( total_inputs, top_p=top_p, top_k=top_k, repetition_penalty=repetition_penalty, watermark=watermark, temperature=temperature, max_new_tokens=500, stop_sequences=[user_name.rstrip(), assistant_name.rstrip()], )): if response.token.special: continue partial_words = partial_words + response.token.text if partial_words.endswith(user_name.rstrip()): partial_words = partial_words.rstrip(user_name.rstrip()) if partial_words.endswith(assistant_name.rstrip()): partial_words = partial_words.rstrip(assistant_name.rstrip()) if i == 0: history.append(" " + partial_words) else: history[-1] = partial_words chat = [ (history[i].strip(), history[i + 1].strip()) for i in range(0, len(history) - 1, 2) ] yield chat, history # Define function to generate model predictions and update the history def predict_glm(input, history=[]): response, history = model_glm.chat(tokenizer_glm, input, history) # translate Chinese to English history = [(query, translate_Chinese_English(response)) for query, response in history] return history, history #[history] + updates def translate_Chinese_English(chinese_text): # translate Chinese to English tokenizer_chtoen.src_lang = "zh" encoded_zh = tokenizer_chtoen(chinese_text, return_tensors="pt") generated_tokens = model_chtoen.generate(**encoded_zh, forced_bos_token_id=tokenizer_chtoen.get_lang_id("en")) trans_eng_text = tokenizer_chtoen.batch_decode(generated_tokens, skip_special_tokens=True) return trans_eng_text[0] """ with gr.Blocks() as demo: chatbot = gr.Chatbot() state = gr.State([]) with gr.Row(): txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False) txt.submit(predict, [txt, state], [chatbot, state]) demo.launch(debug=True) """ def reset_textbox(): return gr.update(value="") def reset_chat(chatbot, state): # debug #print(f"^^chatbot value is - {chatbot}") #print(f"^^state value is - {state}") return None, [] #title = """

🔥🔥Comparison: ChatGPT & OpenChatKit


🚀A Gradio Streaming Demo


Official Demo: OpenChatKit feedback app""" title = """

🔥🔥Comparison: ChatGPT & Open Sourced CHatGLM-6B


🚀A Gradio Chatbot Demo

""" description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form: ``` User: Assistant: User: Assistant: ... ``` In this app, you can explore the outputs of multiple LLMs when prompted in similar ways. """ with gr.Blocks(css="""#col_container {width: 1000px; margin-left: auto; margin-right: auto;} #chatgpt {height: 520px; overflow: auto;} #chatglm {height: 520px; overflow: auto;} """ ) as demo: #chattogether {height: 520px; overflow: auto;} """ ) as demo: #clear {width: 100px; height:50px; font-size:12px}""") as demo: gr.HTML(title) with gr.Row(): with gr.Column(scale=14): with gr.Box(): with gr.Row(): with gr.Column(scale=13): openai_api_key = gr.Textbox(type='password', label="Enter your OpenAI API key here for ChatGPT") inputs = gr.Textbox(placeholder="Hi there!", label="Type an input and press Enter ⤵️ " ) with gr.Column(scale=1): b1 = gr.Button('🏃Run', elem_id = 'run').style(full_width=True) b2 = gr.Button('🔄Clear up Chatbots!', elem_id = 'clear').style(full_width=True) state_chatgpt = gr.State([]) #state_together = gr.State([]) state_glm = gr.State([]) with gr.Box(): with gr.Row(): chatbot_chatgpt = gr.Chatbot(elem_id="chatgpt", label='ChatGPT API - OPENAI') #chatbot_together = gr.Chatbot(elem_id="chattogether", label='OpenChatKit - Text Generation') chatbot_glm = gr.Chatbot(elem_id="chatglm", label='THUDM-ChatGLM6B') with gr.Column(scale=2, elem_id='parameters'): with gr.Box(): gr.HTML("Parameters for #OpenCHAtKit", visible=False) top_p = gr.Slider(minimum=-0, maximum=1.0,value=0.25, step=0.05,interactive=True, label="Top-p", visible=False) temperature = gr.Slider(minimum=-0, maximum=5.0, value=0.6, step=0.1, interactive=True, label="Temperature", visible=False) top_k = gr.Slider( minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k", visible=False) repetition_penalty = gr.Slider( minimum=0.1, maximum=3.0, value=1.01, step=0.01, interactive=True, label="Repetition Penalty", visible=False) watermark = gr.Checkbox(value=True, label="Text watermarking", visible=False) model = gr.CheckboxGroup(value="Rallio67/joi2_20B_instruct_alpha", choices=["togethercomputer/GPT-NeoXT-Chat-Base-20B", "Rallio67/joi2_20B_instruct_alpha", "google/flan-t5-xxl", "google/flan-ul2", "bigscience/bloomz", "EleutherAI/gpt-neox-20b",], label="Model",visible=False,) temp_textbox_together = gr.Textbox(value=model.choices[0], visible=False) with gr.Box(): gr.HTML("Parameters for OpenAI's ChatGPT") top_p_chatgpt = gr.Slider( minimum=-0, maximum=1.0, value=1.0, step=0.05, interactive=True, label="Top-p",) temperature_chatgpt = gr.Slider( minimum=-0, maximum=5.0, value=1.0, step=0.1, interactive=True, label="Temperature",) chat_counter_chatgpt = gr.Number(value=0, visible=False, precision=0) inputs.submit(reset_textbox, [], [inputs]) inputs.submit( predict_chatgpt, [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) #inputs.submit( predict_together, # [temp_textbox_together, inputs, top_p, temperature, top_k, repetition_penalty, watermark, chatbot_together, state_together, ], # [chatbot_together, state_together],) inputs.submit( predict_glm, [inputs, state_glm, ], [chatbot_glm, state_glm],) b1.click( predict_chatgpt, [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) #b1.click( predict_together, # [temp_textbox_together, inputs, top_p, temperature, top_k, repetition_penalty, watermark, chatbot_together, state_together, ], # [chatbot_together, state_together],) b1.click( predict_glm, [inputs, state_glm, ], [chatbot_glm, state_glm],) b2.click(reset_chat, [chatbot_chatgpt, state_chatgpt], [chatbot_chatgpt, state_chatgpt]) #b2.click(reset_chat, [chatbot_together, state_together], [chatbot_together, state_together]) b2.click(reset_chat, [chatbot_glm, state_glm], [chatbot_glm, state_glm]) gr.HTML('''
Duplicate SpaceDuplicate the Space and run securely with your OpenAI API Key
''') gr.Markdown(description) demo.queue(concurrency_count=16).launch(height= 2500, debug=True)