Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import json | |
| import requests | |
| import os | |
| from text_generation import Client, InferenceAPIClient | |
| # Load pre-trained model and tokenizer - for THUDM model | |
| from transformers import AutoModel, AutoTokenizer | |
| tokenizer_glm = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) | |
| model_glm = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() | |
| model_glm = model_glm.eval() | |
| # Load pre-trained model and tokenizer for Chinese to English translator | |
| from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer | |
| model_chtoen = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") | |
| tokenizer_chtoen = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M") | |
| #Predict function for CHATGPT | |
| def predict_chatgpt(inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt=[], history=[]): | |
| #Define payload and header for chatgpt API | |
| payload = { | |
| "model": "gpt-3.5-turbo", | |
| "messages": [{"role": "user", "content": f"{inputs}"}], | |
| "temperature" : 1.0, | |
| "top_p":1.0, | |
| "n" : 1, | |
| "stream": True, | |
| "presence_penalty":0, | |
| "frequency_penalty":0, | |
| } | |
| headers = { | |
| "Content-Type": "application/json", | |
| "Authorization": f"Bearer {openai_api_key}" | |
| } | |
| #debug | |
| #print(f"chat_counter_chatgpt - {chat_counter_chatgpt}") | |
| #Handling the different roles for ChatGPT | |
| if chat_counter_chatgpt != 0 : | |
| messages=[] | |
| for data in chatbot_chatgpt: | |
| temp1 = {} | |
| temp1["role"] = "user" | |
| temp1["content"] = data[0] | |
| temp2 = {} | |
| temp2["role"] = "assistant" | |
| temp2["content"] = data[1] | |
| messages.append(temp1) | |
| messages.append(temp2) | |
| temp3 = {} | |
| temp3["role"] = "user" | |
| temp3["content"] = inputs | |
| messages.append(temp3) | |
| payload = { | |
| "model": "gpt-3.5-turbo", | |
| "messages": messages, #[{"role": "user", "content": f"{inputs}"}], | |
| "temperature" : temperature_chatgpt, #1.0, | |
| "top_p": top_p_chatgpt, #1.0, | |
| "n" : 1, | |
| "stream": True, | |
| "presence_penalty":0, | |
| "frequency_penalty":0, | |
| } | |
| chat_counter_chatgpt+=1 | |
| history.append(inputs) | |
| # make a POST request to the API endpoint using the requests.post method, passing in stream=True | |
| response = requests.post(API_URL, headers=headers, json=payload, stream=True) | |
| token_counter = 0 | |
| partial_words = "" | |
| counter=0 | |
| for chunk in response.iter_lines(): | |
| #Skipping the first chunk | |
| if counter == 0: | |
| counter+=1 | |
| continue | |
| # check whether each line is non-empty | |
| if chunk.decode() : | |
| chunk = chunk.decode() | |
| # decode each line as response data is in bytes | |
| if len(chunk) > 13 and "content" in json.loads(chunk[6:])['choices'][0]["delta"]: | |
| partial_words = partial_words + json.loads(chunk[6:])['choices'][0]["delta"]["content"] | |
| if token_counter == 0: | |
| history.append(" " + partial_words) | |
| else: | |
| history[-1] = partial_words | |
| chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2) ] # convert to tuples of list | |
| token_counter+=1 | |
| yield chat, history, chat_counter_chatgpt # this resembles {chatbot: chat, state: history} | |
| # Define function to generate model predictions and update the history | |
| def predict_glm(input, history=[]): | |
| response, history = model_glm.chat(tokenizer_glm, input, history) | |
| # translate Chinese to English | |
| history = [(query, translate_Chinese_English(response)) for query, response in history] | |
| return history, history #[history] + updates | |
| def translate_Chinese_English(chinese_text): | |
| # translate Chinese to English | |
| tokenizer_chtoen.src_lang = "zh" | |
| encoded_zh = tokenizer_chtoen(chinese_text, return_tensors="pt") | |
| generated_tokens = model_chtoen.generate(**encoded_zh, forced_bos_token_id=tokenizer_chtoen.get_lang_id("en")) | |
| trans_eng_text = tokenizer_chtoen.batch_decode(generated_tokens, skip_special_tokens=True) | |
| return trans_eng_text[0] | |
| # Define function to generate model predictions and update the history | |
| def predict_glm_stream(input, history=[]): #, top_p, temperature): | |
| response, history = model_glm.chat(tokenizer_glm, input, history) | |
| print(f"outside for loop resonse is ^^- {response}") | |
| print(f"outside for loop history is ^^- {history}") | |
| top_p = 1.0 | |
| temperature = 1.0 | |
| for response, history in model.stream_chat(tokenizer_glm, input, history, top_p=1.0, temperature=1.0): #max_length=max_length, | |
| print(f"In for loop resonse is ^^- {response}") | |
| print(f"In for loop history is ^^- {history}") | |
| # translate Chinese to English | |
| history = [(query, translate_Chinese_English(response)) for query, response in history] | |
| print(f"In for loop translated history is ^^- {history}") | |
| yield history, history #[history] + updates | |
| """ | |
| def predict(input, max_length, top_p, temperature, history=None): | |
| if history is None: | |
| history = [] | |
| for response, history in model.stream_chat(tokenizer, input, history, max_length=max_length, top_p=top_p, | |
| temperature=temperature): | |
| updates = [] | |
| for query, response in history: | |
| updates.append(gr.update(visible=True, value="user：" + query)) #用户 | |
| updates.append(gr.update(visible=True, value="ChatGLM-6B：" + response)) | |
| if len(updates) < MAX_BOXES: | |
| updates = updates + [gr.Textbox.update(visible=False)] * (MAX_BOXES - len(updates)) | |
| yield [history] + updates | |
| """ | |
| def reset_textbox(): | |
| return gr.update(value="") | |
| def reset_chat(chatbot, state): | |
| # debug | |
| #print(f"^^chatbot value is - {chatbot}") | |
| #print(f"^^state value is - {state}") | |
| return None, [] | |
| #title = """<h1 align="center">🔥🔥Comparison: ChatGPT & OpenChatKit </h1><br><h3 align="center">🚀A Gradio Streaming Demo</h3><br>Official Demo: <a href="https://huggingface.co/spaces/togethercomputer/OpenChatKit">OpenChatKit feedback app</a>""" | |
| title = """<h1 align="center">🔥🔥Comparison: ChatGPT & Open Sourced CHatGLM-6B </h1><br><h3 align="center">🚀A Gradio Chatbot Demo</h3>""" | |
| description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form: | |
| ``` | |
| User: <utterance> | |
| Assistant: <utterance> | |
| User: <utterance> | |
| Assistant: <utterance> | |
| ... | |
| ``` | |
| In this app, you can explore the outputs of multiple LLMs when prompted in similar ways. | |
| """ | |
| with gr.Blocks(css="""#col_container {width: 1000px; margin-left: auto; margin-right: auto;} | |
| #chatgpt {height: 520px; overflow: auto;} | |
| #chatglm {height: 520px; overflow: auto;} """ ) as demo: | |
| #chattogether {height: 520px; overflow: auto;} """ ) as demo: | |
| #clear {width: 100px; height:50px; font-size:12px}""") as demo: | |
| gr.HTML(title) | |
| with gr.Row(): | |
| with gr.Column(scale=14): | |
| with gr.Box(): | |
| with gr.Row(): | |
| with gr.Column(scale=13): | |
| openai_api_key = gr.Textbox(type='password', label="Enter your OpenAI API key here for ChatGPT") | |
| inputs = gr.Textbox(placeholder="Hi there!", label="Type an input and press Enter ⤵️ " ) | |
| with gr.Column(scale=1): | |
| b1 = gr.Button('🏃Run', elem_id = 'run').style(full_width=True) | |
| b2 = gr.Button('🔄Clear up Chatbots!', elem_id = 'clear').style(full_width=True) | |
| state_chatgpt = gr.State([]) | |
| #state_together = gr.State([]) | |
| state_glm = gr.State([]) | |
| with gr.Box(): | |
| with gr.Row(): | |
| chatbot_chatgpt = gr.Chatbot(elem_id="chatgpt", label='ChatGPT API - OPENAI') | |
| #chatbot_together = gr.Chatbot(elem_id="chattogether", label='OpenChatKit - Text Generation') | |
| chatbot_glm = gr.Chatbot(elem_id="chatglm", label='THUDM-ChatGLM6B') | |
| with gr.Column(scale=2, elem_id='parameters'): | |
| with gr.Box(): | |
| gr.HTML("Parameters for #OpenCHAtKit", visible=False) | |
| top_p = gr.Slider(minimum=-0, maximum=1.0,value=0.25, step=0.05,interactive=True, label="Top-p", visible=False) | |
| temperature = gr.Slider(minimum=-0, maximum=5.0, value=0.6, step=0.1, interactive=True, label="Temperature", visible=False) | |
| top_k = gr.Slider( minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k", visible=False) | |
| repetition_penalty = gr.Slider( minimum=0.1, maximum=3.0, value=1.01, step=0.01, interactive=True, label="Repetition Penalty", visible=False) | |
| watermark = gr.Checkbox(value=True, label="Text watermarking", visible=False) | |
| model = gr.CheckboxGroup(value="Rallio67/joi2_20B_instruct_alpha", | |
| choices=["togethercomputer/GPT-NeoXT-Chat-Base-20B", "Rallio67/joi2_20B_instruct_alpha", "google/flan-t5-xxl", "google/flan-ul2", "bigscience/bloomz", "EleutherAI/gpt-neox-20b",], | |
| label="Model",visible=False,) | |
| temp_textbox_together = gr.Textbox(value=model.choices[0], visible=False) | |
| with gr.Box(): | |
| gr.HTML("Parameters for OpenAI's ChatGPT") | |
| top_p_chatgpt = gr.Slider( minimum=-0, maximum=1.0, value=1.0, step=0.05, interactive=True, label="Top-p",) | |
| temperature_chatgpt = gr.Slider( minimum=-0, maximum=5.0, value=1.0, step=0.1, interactive=True, label="Temperature",) | |
| chat_counter_chatgpt = gr.Number(value=0, visible=False, precision=0) | |
| inputs.submit(reset_textbox, [], [inputs]) | |
| inputs.submit( predict_chatgpt, | |
| [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], | |
| [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) | |
| #inputs.submit( predict_glm, | |
| # [inputs, state_glm, ], | |
| # [chatbot_glm, state_glm],) | |
| #b1.click( predict_glm, | |
| # [inputs, state_glm, ], | |
| # [chatbot_glm, state_glm],) | |
| inputs.submit( predict_glm_stream, | |
| [inputs, state_glm, ], | |
| [chatbot_glm, state_glm],) | |
| b1.click( predict_glm_stream, | |
| [inputs, state_glm, ], | |
| [chatbot_glm, state_glm],) | |
| b1.click( predict_chatgpt, | |
| [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], | |
| [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) | |
| b2.click(reset_chat, [chatbot_chatgpt, state_chatgpt], [chatbot_chatgpt, state_chatgpt]) | |
| #b2.click(reset_chat, [chatbot_together, state_together], [chatbot_together, state_together]) | |
| b2.click(reset_chat, [chatbot_glm, state_glm], [chatbot_glm, state_glm]) | |
| gr.HTML('''<center><a href="https://huggingface.co/spaces/ysharma/OpenChatKit_ChatGPT_Comparison?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>Duplicate the Space and run securely with your OpenAI API Key</center>''') | |
| gr.Markdown(description) | |
| demo.queue(concurrency_count=16).launch(height= 2500, debug=True) |