import gradio as gr import pandas as pd import random from datasets import load_dataset, Dataset, DatasetDict from huggingface_hub import HfApi, login import os from datetime import datetime import markdown hf_api = HfApi() HF_TOKEN = os.getenv('HF_TOKEN') login(token=HF_TOKEN) log_dataset = "HumanLLMs/log" selected_indices = set() dataset_1 = load_dataset("HumanLLMs/LlamaPair")["train"] #dataset_2 = load_dataset("HumanLLMs/QwenPair")["train"] #dataset_3 = load_dataset("HumanLLMs/MistralPair")["train"]""" df_log = pd.DataFrame(columns=["instruction", "selected_model", "pair", "submission_time"]) def remove_emojis(text): return text.encode('ascii', 'ignore').decode('ascii') def get_random_row(): global selected_indices #selected_dataset = random.choice([dataset_1, dataset_2, dataset_3]) #pair_name = ("LlamaPair" if selected_dataset == dataset_1 # else "QwenPair" if selected_dataset == dataset_2 # else "MistralPair" pair_name = "LlamaPair" if len(selected_indices) >= len(dataset_1): raise ValueError("All rows in the dataset have been used.") idx = random.randint(0, len(dataset_1) - 1) while idx in selected_indices: idx = random.randint(0, len(dataset_1) - 1) selected_indices.add(idx) row = dataset_1[idx] instruction = row["instruction"] response_human = row["response_human_like_model"] response_official = row["response_offical_instruct_model"] responses = [("Human-like Model", response_human), ("Official Model", response_official)] random.shuffle(responses) return (instruction, remove_emojis(responses[0][1]), remove_emojis(responses[1][1]), responses[0][0], responses[1][0], pair_name) def format_response_1_html(response): return f'''
Answer 1:
{markdown.markdown(response)}
''' def format_response_2_html(response): ## Near duplicate return f'''
Answer 2:
{markdown.markdown(response)}
''' counter = 0 accumulated_log = pd.DataFrame(columns=["instruction", "selected_model", "pair", "submission_time"]) def submit_choice(selected_response, instruction, label_1, label_2, pair_name): global counter, accumulated_log try: # Use the existing dataset if exists, create the dataset if not df_log = pd.DataFrame(load_dataset(log_dataset)["train"]) except: df_log = pd.DataFrame(columns=["instruction", "selected_model", "pair", "submission_time"]) selected_model = label_1 if selected_response == "Answer 1" else label_2 submission_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') new_instruction, new_response_1, new_response_2, new_label_1, new_label_2, new_pair_name = get_random_row() new_entry = pd.DataFrame({ "instruction": [new_instruction], "selected_model": [selected_model], "pair": [pair_name], "submission_time": [submission_time] }) accumulated_log = pd.concat([accumulated_log, new_entry], ignore_index=True) counter += 1 if counter % 10 == 0: # Save the dataset every 10 vote df_log = pd.concat([df_log, accumulated_log], ignore_index=True) df_log.to_csv("annotations_log.csv", index=False) accumulated_log = pd.DataFrame(columns=["instruction", "selected_model", "pair", "submission_time"]) log = Dataset.from_pandas(df_log) log.push_to_hub(log_dataset) question = f"""
Question:
{new_instruction}
""" return ( question, format_response_1_html(new_response_1), format_response_2_html(new_response_2), new_label_1, new_label_2, new_pair_name, "Your choice has been recorded. A new question is loaded!" ) def create_interface(): instruction, response_1, response_2, label_1, label_2, pair_name = get_random_row() with gr.Blocks(theme=gr.themes.Default()) as demo: gr.HTML("""

Human-Likeness Voting System

""") gr.Markdown("This interface has been created to compare the performance of the human-like LLMs developed by our team with the models on which they were trained. The results of this study will be presented in a paper. Please ensure that your responses are fair and accurate when casting your vote and selecting the appropriate answer. We thank you for your contributions on behalf of the research team.") gr.Markdown("## Instructions") gr.Markdown( """ 1. First, read the provided question carefully. 2. Second, read both responses carefully. 3. Finally, select the model that best resembles a human in terms of response quality.""" ) current_instruction = gr.State(instruction) label_1_state = gr.State(label_1) label_2_state = gr.State(label_2) pair_name_state = gr.State(pair_name) question_display = gr.HTML( value=f"""
Question:
{instruction}
""" ) with gr.Row(): with gr.Column(): response_1_display = gr.HTML(format_response_1_html(response_1)) with gr.Column(): response_2_display = gr.HTML(format_response_2_html(response_2)) with gr.Row(): selected_response = gr.Radio( ["Answer 1", "Answer 2"], label="Which answer is better?", interactive=True, ) submit_btn = gr.Button("Submit Choice") status_output = gr.Textbox( interactive=False, label="Status", value="Select an answer and click Submit" ) submit_btn.click( fn=submit_choice, inputs=[ selected_response, current_instruction, label_1_state, label_2_state, pair_name_state ], outputs=[ question_display, response_1_display, response_2_display, label_1_state, label_2_state, pair_name_state, status_output ] ) return demo if __name__ == "__main__": interface = create_interface() interface.launch(share=True)