Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import subprocess | |
| import os | |
| import sys | |
| import time | |
| import pandas as pd | |
| from threading import Thread | |
| import numpy as np | |
| # Add the path to the "src" directory of detect-pretrain-code-contamination to the sys.path | |
| project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "detect-pretrain-code-contamination")) | |
| src_dir = os.path.join(project_root, "src") | |
| sys.path.insert(0, src_dir) | |
| import run as evaluator # Import the run module | |
| from src.css_html import custom_css | |
| from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_2 | |
| from src.envs import API, H4_TOKEN, REPO_ID | |
| from huggingface_hub import HfApi | |
| from src.utils import ( | |
| AutoEvalColumn, | |
| fields, | |
| is_model_on_hub, | |
| make_clickable_names, | |
| styled_error, | |
| styled_message, | |
| EVAL_COLS, | |
| EVAL_TYPES | |
| ) | |
| COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] | |
| TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden] | |
| COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden] | |
| TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden] | |
| # CONFIGURATION: | |
| test_datasets = ["truthful_qa","cais/mmlu","ai2_arc","gsm8k","Rowan/hellaswag","winogrande"] | |
| modelQueue = (pd.read_csv('data/queue.csv')).values.tolist() | |
| print(modelQueue) | |
| def restart_space(): #Most dumbest update function to ever exist, I'm sobbing in tears as I've tried to make gradio update the leaderboard literally any other way. | |
| API.restart_space(repo_id=REPO_ID, token=H4_TOKEN) | |
| def formatr(result): | |
| result = str(result) | |
| result = result.split(",")[2].replace(")","") | |
| result = result.replace(" ","") | |
| return result | |
| def save_to_txt(model, results, model_type,ref_model): | |
| file_path = "data/code_eval_board.csv" | |
| with open(file_path, "a") as f: | |
| f.write(f"\n{model_type},{model}," + str(formatr(results["arc"])) + "," + str(formatr(results["hellaswag"])) + "," + str(formatr(results["mmlu"])) + "," + str(formatr(results["truthfulQA"])) + "," + str(formatr(results["winogrande"])) + "," + str(formatr(results["gsm8k"])) + f",{ref_model}") | |
| print(f"Finished evaluation of model: {model} using ref_model: {ref_model}") | |
| print(f"\n{model_type},{model}," + str(formatr(results["arc"])) + "," + str(formatr(results["hellaswag"])) + "," + str(formatr(results["mmlu"])) + "," + str(formatr(results["truthfulQA"])) + "," + str(formatr(results["winogrande"])) + "," + str(formatr(results["gsm8k"])) + f",{ref_model}") | |
| f.close() | |
| def run_test(model,ref_model,data): | |
| print(f"|| TESTING {data} ||") | |
| return evaluator.main( | |
| target_model=f"{model}", | |
| ref_model=f"{ref_model}", | |
| output_dir="out", | |
| data=f"{data}", | |
| length=64, | |
| key_name="input", | |
| ratio_gen=0.4 | |
| ) # Call the main function in detect-pretrain-code-contamination/src/run.py | |
| def evaluate(model,model_type,ref_model): | |
| print(f"|| EVALUATING {model} ||") | |
| results = { | |
| "arc": run_test(model, ref_model, test_datasets[2]), | |
| "hellaswag": run_test(model, ref_model, test_datasets[4]), | |
| "mmlu": run_test(model, ref_model, test_datasets[1]), | |
| "truthfulQA": run_test(model, ref_model, test_datasets[0]), | |
| "winogrande": run_test(model, ref_model, test_datasets[5]), | |
| "gsm8k": run_test(model, ref_model, test_datasets[3]), | |
| "ref_model": ref_model, | |
| } | |
| # Save to .txt file in /Evaluations/{model} | |
| save_to_txt(model, results, model_type,ref_model) | |
| return "\n".join([f"{k}:{results[k]}" for k in results]) | |
| def worker_thread(): | |
| global modelQueue, server | |
| while True: | |
| for submission in modelQueue: | |
| #evaluate(submission[1],submission[0].split(" ")[0],submission[2]) | |
| #modelQueue.pop(modelQueue.index(submission)) | |
| #exit() | |
| #The exit above is temporal while I figure out how to unload a model from a thread or similar. | |
| # Uncomment those lines in order to begin testing, I test these models outside of this space and later commit the results back. | |
| # I highly encourage you to try to reproduce the results I get using your own implementation. | |
| # Do NOT take anything listed here as fact, as I'm not 100% my implementation works as intended. | |
| # Take whatever you see in the leaderboard as a grain of salt, do NOT accuse models of cheating just because of their placement here alone. | |
| time.sleep(1) | |
| time.sleep(1) | |
| def queue(model,model_type,ref_model): | |
| global modelQueue | |
| modelQueue.append([model_type,model,ref_model]) | |
| file_path = "data/queue.csv" | |
| with open(file_path, "a") as f: | |
| model = model.strip() | |
| ref_model = ref_model.strip() | |
| f.write(f"\n{model_type},{model},{ref_model}") | |
| f.close() | |
| print(f"QUEUE:\n{modelQueue}") | |
| ### bigcode/bigcode-models-leaderboard | |
| def add_new_eval( | |
| model: str, | |
| revision: str, | |
| ref_model: str, | |
| model_type: str, | |
| ): | |
| ref_model = ref_model | |
| if model_type is None or model_type == "" or model_type == []: | |
| return styled_error("Please select a model type.") | |
| print(model_type) | |
| # check the model actually exists before adding the eval | |
| if revision == "": | |
| revision = "main" | |
| model_on_hub, error = is_model_on_hub(model, revision) | |
| if not model_on_hub: | |
| return styled_error(f'Model "{model}" {error}') | |
| print("Adding new eval") | |
| queue(model,model_type,ref_model) | |
| return styled_message("Your request has been submitted to the evaluation queue!\n") | |
| def select_columns(df, columns): | |
| always_here_cols = [ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| # We use COLS to maintain sorting | |
| filtered_df = df[ | |
| always_here_cols + [c for c in COLS if c in df.columns and c in columns] | |
| ] | |
| return filtered_df | |
| def filter_items(df, leaderboard_table, query): | |
| if query == "All": | |
| return df[leaderboard_table.columns] | |
| else: | |
| query = query[0] # take only the emoji character | |
| filtered_df = df[(df["T"] == query)] | |
| return filtered_df[leaderboard_table.columns] | |
| def search_table(df, leaderboard_table, query): | |
| filtered_df = df[(df["Models"].str.contains(query, case=False))] | |
| return filtered_df[leaderboard_table.columns] | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| with gr.Row(): | |
| gr.Markdown( | |
| """<div style="text-align: center;"><h1> 📄 LLM Contamination Detector </h1></div>\ | |
| <br>\ | |
| <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">🤗 Big Code Models Leaderboard ⭐</a>, we use an implementation of <a href="https://huggingface.co/papers/2310.16789">Detecting Pretraining Data from Large Language Models</a> paper found in <a href="https://github.com/swj0419/detect-pretrain-code-contamination/tree/master">this github repo</a>, to provide contamination scores for LLMs on the datasets used by Open LLM Leaderboard.\ | |
| This space should NOT be used to flag or accuse models of cheating / being contamined, instead, it should form part of a holistic assesment by the parties involved.</p>""", | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.Column(): | |
| with gr.Tabs(elem_classes="A100-tabs") as A100_tabs: | |
| with gr.TabItem("🔍 Evaluations", id=0): | |
| with gr.Column(): | |
| with gr.Accordion("➡️ See filters", open=False): | |
| shown_columns = gr.CheckboxGroup( | |
| choices=[ | |
| c | |
| for c in COLS | |
| if c | |
| not in [ | |
| AutoEvalColumn.dummy.name, | |
| AutoEvalColumn.model.name, | |
| AutoEvalColumn.model_type_symbol.name, | |
| ] | |
| ], | |
| value=[ | |
| c | |
| for c in COLS_LITE | |
| if c | |
| not in [ | |
| AutoEvalColumn.dummy.name, | |
| AutoEvalColumn.model.name, | |
| AutoEvalColumn.model_type_symbol.name, | |
| ] | |
| ], | |
| label="", | |
| elem_id="column-select", | |
| interactive=True, | |
| ) | |
| # with gr.Column(min_width=780): | |
| with gr.Row(): | |
| search_bar = gr.Textbox( | |
| placeholder="🔍 Search for a model and press ENTER...", | |
| show_label=False, | |
| elem_id="search-bar", | |
| ) | |
| filter_columns = gr.Radio( | |
| label="⏚ Filter model types", | |
| choices=["All", "🟢 Base", "🔶 Finetuned"], | |
| value="All", | |
| elem_id="filter-columns", | |
| ) | |
| df = pd.read_csv("data/code_eval_board.csv") | |
| leaderboard_df = gr.components.Dataframe( | |
| value=df[ | |
| [ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| + shown_columns.value | |
| ], | |
| headers=[ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| + shown_columns.value, | |
| datatype=TYPES, | |
| elem_id="leaderboard-table", | |
| interactive=False, | |
| ) | |
| hidden_leaderboard_df = gr.components.Dataframe( | |
| value=df, | |
| headers=COLS, | |
| datatype=["str" for _ in range(len(COLS))], | |
| visible=False, | |
| ) | |
| search_bar.submit( | |
| search_table, | |
| [hidden_leaderboard_df, leaderboard_df, search_bar], | |
| leaderboard_df, | |
| ) | |
| filter_columns.change( | |
| filter_items, | |
| [hidden_leaderboard_df, leaderboard_df, filter_columns], | |
| leaderboard_df, | |
| ) | |
| shown_columns.change( | |
| select_columns, | |
| [hidden_leaderboard_df, shown_columns], | |
| leaderboard_df, | |
| ) | |
| gr.Markdown( | |
| """ | |
| **Notes:** | |
| - The Huggingface team is working on their own implementation of this paper as a space, I'll be leaving this space up until that's available. | |
| - Some scores may not be entirely accurate according to the paper cited as I still work out the kinks and innacuracies of this implementation. | |
| - For any issues, questions, or comments either open a discussion in this space's community tab or message me directly to my discord: yeyito777. | |
| - Make sure to check the pinned discussion in this space's community tab for implementation details I'm not 100% about. | |
| """, | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.TabItem("📝 About", id=2): | |
| gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") | |
| with gr.TabItem("🛠️ Submit models", id=3): | |
| gr.Markdown(SUBMISSION_TEXT) | |
| gr.Markdown( | |
| "## 📤 Submit a model here:", elem_classes="markdown-text" | |
| ) | |
| with gr.Column(): | |
| with gr.Column(): | |
| with gr.Accordion( | |
| f"⏳ Evaluation Queue ({len(modelQueue)})", | |
| open=False, | |
| ): | |
| with gr.Row(): | |
| finished_eval_table = gr.components.Dataframe( | |
| value=pd.DataFrame(modelQueue, columns=['Type','Model','Reference Model']), | |
| ) | |
| with gr.Row(): | |
| model_name = gr.Textbox(label="Model name") | |
| revision_name = gr.Textbox( | |
| label="revision", placeholder="main" | |
| ) | |
| with gr.Row(): | |
| ref_model = gr.Dropdown( | |
| choices=[ | |
| "mistralai/Mistral-7B-v0.1", | |
| "huggyllama/llama-7b", | |
| "NousResearch/Llama-2-7b-hf", | |
| "upstage/SOLAR-10.7B-v1.0", | |
| ], | |
| label="Reference Model", | |
| multiselect=False, | |
| value="mistralai/Mistral-7B-v0.1", | |
| interactive=True, | |
| ) | |
| model_type = gr.Dropdown( | |
| choices=["🟢 base", "🔶 finetuned"], | |
| label="Model type", | |
| multiselect=False, | |
| value=None, | |
| interactive=True, | |
| ) | |
| submit_button = gr.Button("Submit Eval") | |
| submission_result = gr.Markdown() | |
| submit_button.click( | |
| add_new_eval, | |
| inputs=[model_name, revision_name, ref_model, model_type], | |
| outputs=[submission_result], | |
| ) | |
| gr.Markdown(SUBMISSION_TEXT_2) | |
| thread = Thread(target=worker_thread) | |
| thread.start() | |
| demo.launch(share=True) | |
| # Some worries: | |
| # 1. Am I testing things correctly in eval.py, following the template format? | |
| # 2. Am I choosing the correct splits in run.py? The higherarchy I use is: test > val > train | |
| # (As in: if test exists, I go with that, then validation, then default) | |
| # 3. I decided to go with winogrande_debiased instead of winogrande_l arbitrarily. | |
| # (Not sure which one open llm leaderboard uses, or what is the standard) | |
| # 4. I'm unsure why in eval.py we append the output at the end of the input. | |
| # 5. Currently I'm using huggyllama/llama-7b as ref_model, should I switch to llama2-7B? Maybe Mistral-7B? | |