init

app.py

@@ -8,20 +8,15 @@ import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
-from transformers import AutoConfig
 
-from src.auto_leaderboard.get_model_metadata import apply_metadata
 from src.assets.text_content import *
 from src.elo_leaderboard.load_results import get_elo_plots, get_elo_results_dicts
-from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
-from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
-from src.assets.css_html_js import custom_css, get_window_url_params
-from src.utils_display import AutoEvalColumn, EvalQueueColumn, EloEvalColumn, fields, styled_error, styled_warning, styled_message
+from src.assets.css_html_js import custom_css, get_window_url_params # left in case you need them
+from src.utils_display import EloEvalColumn, fields, styled_error, styled_warning, styled_message
 from src.init import load_all_info_from_hub
 
 # clone / pull the lmeh eval data
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
-LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
 HUMAN_EVAL_REPO = "HuggingFaceH4/scale-human-eval"
 GPT_4_EVAL_REPO = "HuggingFaceH4/open_llm_leaderboard_oai_evals"
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
@@ -37,21 +32,7 @@ def restart_space():
         repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN
     )
 
-auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models = load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
-
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-
-if not IS_PUBLIC:
-    COLS.insert(2, AutoEvalColumn.is_8bit.name)
-    TYPES.insert(2, AutoEvalColumn.is_8bit.type)
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
+human_eval_repo, gpt_4_eval_repo = load_all_info_from_hub(HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
 
 ELO_COLS = [c.name for c in fields(EloEvalColumn)]
 ELO_TYPES = [c.type for c in fields(EloEvalColumn)]
@@ -66,78 +47,6 @@ def has_nan_values(df, columns):
     return df[columns].isna().any(axis=1)
 
 
-def get_leaderboard_df():
-    if auto_eval_repo:
-        print("Pulling evaluation results for the leaderboard.")
-        auto_eval_repo.git_pull()
-
-    all_data = get_eval_results_dicts(IS_PUBLIC)
-
-    if not IS_PUBLIC:
-        all_data.append(gpt4_values)
-        all_data.append(gpt35_values)
-
-    all_data.append(baseline)
-    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
-
-    df = pd.DataFrame.from_records(all_data)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[COLS]
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
-    return df
-
-
-def get_evaluation_queue_df():
-    # todo @saylortwift: replace the repo by the one you created for the eval queue
-    if auto_eval_repo:
-        print("Pulling changes for the evaluation queue.")
-        auto_eval_repo.git_pull()
-
-    entries = [
-        entry
-        for entry in os.listdir(EVAL_REQUESTS_PATH)
-        if not entry.startswith(".")
-    ]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data["# params"] = "unknown"
-            data["model"] = make_clickable_model(data["model"])
-            data["revision"] = data.get("revision", "main")
-
-            all_evals.append(data)
-        else:
-            # this is a folder
-            sub_entries = [
-                e
-                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
-                if not e.startswith(".")
-            ]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                # data["# params"] = get_n_params(data["model"])
-                data["model"] = make_clickable_model(data["model"])
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
-    df_pending = pd.DataFrame.from_records(pending_list)
-    df_running = pd.DataFrame.from_records(running_list)
-    df_finished = pd.DataFrame.from_records(finished_list)
-    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
-
-
 def get_elo_leaderboard(df_instruct, df_code_instruct, tie_allowed=False):
     if human_eval_repo:
         print("Pulling human_eval_repo changes")
@@ -173,14 +82,6 @@ def get_elo_elements():
         plot_4,
     )
 
-
-original_df = get_leaderboard_df()
-leaderboard_df = original_df.copy()
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df()
 (
     elo_leaderboard,
     elo_leaderboard_with_tie_allowed,
@@ -191,309 +92,46 @@ leaderboard_df = original_df.copy()
 ) = get_elo_elements()
 
 
-def is_model_on_hub(model_name, revision) -> bool:
-    try:
-        AutoConfig.from_pretrained(model_name, revision=revision)
-        return True, None
-    
-    except ValueError as e:
-        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
-
-    except Exception as e:
-        print("Could not get the model config from the hub.: \n", e)
-        return False, "was not found on hub!"
-
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    is_8_bit_eval: bool,
-    private: bool,
-    is_delta_weight: bool,
-):
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    # check the model actually exists before adding the eval
-    if revision == "":
-        revision = "main"
-
-    if is_delta_weight: 
-        base_model_on_hub, error = is_model_on_hub(base_model, revision)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    model_on_hub, error = is_model_on_hub(model, revision)
-    if not model_on_hub:
-        return styled_error(f'Model "{model}" {error}')
-
-    print("adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "private": private,
-        "8bit_eval": is_8_bit_eval,
-        "is_delta_weight": is_delta_weight,
-        "status": "PENDING",
-        "submitted_time": current_time,
-    }
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
-
-    # Check for duplicate submission
-    if out_path.split("eval_requests/")[1].lower() in requested_models:
-        return styled_warning("This model has been already submitted.")
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    api.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path,
-        repo_id=LMEH_REPO,
-        token=H4_TOKEN,
-        repo_type="dataset",
-    )
-
-    return styled_message("Your request has been submitted to the evaluation queue!")
-
-
-def refresh():
-    leaderboard_df = get_leaderboard_df()
-    (
-        finished_eval_queue_df,
-        running_eval_queue_df,
-        pending_eval_queue_df,
-    ) = get_evaluation_queue_df()
-    return (
-        leaderboard_df,
-        finished_eval_queue_df,
-        running_eval_queue_df,
-        pending_eval_queue_df,
-    )
-
-
-def search_table(df, query):
-    filtered_df = df[df[AutoEvalColumn.dummy.name].str.contains(query, case=False)]
-    return filtered_df
-
-
-def change_tab(query_param):
-    query_param = query_param.replace("'", '"')
-    query_param = json.loads(query_param)
-
-    if (
-        isinstance(query_param, dict)
-        and "tab" in query_param
-        and query_param["tab"] == "evaluation"
-    ):
-        return gr.Tabs.update(selected=1)
-    else:
-        return gr.Tabs.update(selected=0)
-
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     with gr.Row():
         gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-    with gr.Row():
-        with gr.Column():
-            with gr.Accordion("📙 Citation", open=False):
-                citation_button = gr.Textbox(
-                    value=CITATION_BUTTON_TEXT,
-                    label=CITATION_BUTTON_LABEL,
-                    elem_id="citation-button",
-                ).style(show_copy_button=True)
-        with gr.Column():
-            with gr.Accordion("✨ CHANGELOG", open=False):
-                changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
+    with gr.Column():
+        with gr.Row():
+            with gr.Column(scale=2):
+                gr.Markdown(HUMAN_GPT_EVAL_TEXT, elem_classes="markdown-text")
+            with gr.Column(scale=1):
+                gr.Image(
+                    "src/assets/scale-hf-logo.png", elem_id="scale-logo", show_label=False
+                )
+        gr.Markdown("## No tie allowed")
+        elo_leaderboard_table = gr.components.Dataframe(
+            value=elo_leaderboard,
+            headers=ELO_COLS,
+            datatype=ELO_TYPES,
+            max_rows=5,
+        )
+
+        gr.Markdown("## Tie allowed*")
+        elo_leaderboard_table_with_tie_allowed = gr.components.Dataframe(
+            value=elo_leaderboard_with_tie_allowed,
+            headers=ELO_COLS,
+            datatype=ELO_TYPES,
+            max_rows=5,
+        )
+
+        gr.Markdown(
+            "\* Results when the scores of 4 and 5 were treated as ties.",
+            elem_classes="markdown-text",
+        )
+
+        gr.Markdown(
+            "Let us know in [this discussion](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/65) which models we should add!",
+            elem_id="models-to-add-text",
+        )
 
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("📊 LLM Benchmarks", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Column():
-                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-                with gr.Box(elem_id="search-bar-table-box"):
-                    search_bar = gr.Textbox(
-                        placeholder="🔍 Search your model and press ENTER...",
-                        show_label=False,
-                        elem_id="search-bar",
-                    )
-                    with gr.Tabs(elem_classes="tab-buttons"):
-                        with gr.TabItem("Light View"):
-                            leaderboard_table_lite = gr.components.Dataframe(
-                                value=leaderboard_df[COLS_LITE],
-                                headers=COLS_LITE,
-                                datatype=TYPES_LITE,
-                                max_rows=None,
-                                elem_id="leaderboard-table-lite",
-                            )
-                        with gr.TabItem("Extended Model View"):
-                            leaderboard_table = gr.components.Dataframe(
-                                value=leaderboard_df,
-                                headers=COLS,
-                                datatype=TYPES,
-                                max_rows=None,
-                                elem_id="leaderboard-table",
-                            )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=original_df,
-                        headers=COLS,
-                        datatype=TYPES,
-                        max_rows=None,
-                        visible=False,
-                    )
-                    search_bar.submit(
-                        search_table,
-                        [hidden_leaderboard_table_for_search, search_bar],
-                        leaderboard_table,
-                    )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search_lite = gr.components.Dataframe(
-                        value=original_df[COLS_LITE],
-                        headers=COLS_LITE,
-                        datatype=TYPES_LITE,
-                        max_rows=None,
-                        visible=False,
-                    )
-                    search_bar.submit(
-                        search_table,
-                        [hidden_leaderboard_table_for_search_lite, search_bar],
-                        leaderboard_table_lite,
-                    )
-
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Accordion("✅ Finished Evaluations", open=False):
-                    with gr.Row():
-                        finished_eval_table = gr.components.Dataframe(
-                            value=finished_eval_queue_df,
-                            headers=EVAL_COLS,
-                            datatype=EVAL_TYPES,
-                            max_rows=5,
-                        )
-                with gr.Accordion("🔄 Running Evaluation Queue", open=False):
-                    with gr.Row():
-                        running_eval_table = gr.components.Dataframe(
-                            value=running_eval_queue_df,
-                            headers=EVAL_COLS,
-                            datatype=EVAL_TYPES,
-                            max_rows=5,
-                        )
-
-                with gr.Accordion("⏳ Pending Evaluation Queue", open=False):
-                    with gr.Row():
-                        pending_eval_table = gr.components.Dataframe(
-                            value=pending_eval_queue_df,
-                            headers=EVAL_COLS,
-                            datatype=EVAL_TYPES,
-                            max_rows=5,
-                        )
-
-                with gr.Row():
-                    refresh_button = gr.Button("Refresh")
-                    refresh_button.click(
-                        refresh,
-                        inputs=[],
-                        outputs=[
-                            leaderboard_table,
-                            finished_eval_table,
-                            running_eval_table,
-                            pending_eval_table,
-                        ],
-                    )
-                with gr.Accordion("Submit a new model for evaluation"):
-                    with gr.Row():
-                        with gr.Column():
-                            model_name_textbox = gr.Textbox(label="Model name")
-                            revision_name_textbox = gr.Textbox(
-                                label="revision", placeholder="main"
-                            )
-
-                        with gr.Column():
-                            is_8bit_toggle = gr.Checkbox(
-                                False, label="8 bit eval", visible=not IS_PUBLIC
-                            )
-                            private = gr.Checkbox(
-                                False, label="Private", visible=not IS_PUBLIC
-                            )
-                            is_delta_weight = gr.Checkbox(False, label="Delta weights")
-                            base_model_name_textbox = gr.Textbox(
-                                label="base model (for delta)"
-                            )
-
-                    submit_button = gr.Button("Submit Eval")
-                    submission_result = gr.Markdown()
-                    submit_button.click(
-                        add_new_eval,
-                        [
-                            model_name_textbox,
-                            base_model_name_textbox,
-                            revision_name_textbox,
-                            is_8bit_toggle,
-                            private,
-                            is_delta_weight,
-                        ],
-                        submission_result,
-                    )
-        with gr.TabItem(
-            "🧑‍⚖️ Human & GPT-4 Evaluations 🤖", elem_id="human-gpt-tab-table", id=1
-        ):
-            with gr.Row():
-                with gr.Column(scale=2):
-                    gr.Markdown(HUMAN_GPT_EVAL_TEXT, elem_classes="markdown-text")
-                with gr.Column(scale=1):
-                    gr.Image(
-                        "src/assets/scale-hf-logo.png", elem_id="scale-logo", show_label=False
-                    )
-            gr.Markdown("## No tie allowed")
-            elo_leaderboard_table = gr.components.Dataframe(
-                value=elo_leaderboard,
-                headers=ELO_COLS,
-                datatype=ELO_TYPES,
-                max_rows=5,
-            )
-
-            gr.Markdown("## Tie allowed*")
-            elo_leaderboard_table_with_tie_allowed = gr.components.Dataframe(
-                value=elo_leaderboard_with_tie_allowed,
-                headers=ELO_COLS,
-                datatype=ELO_TYPES,
-                max_rows=5,
-            )
-
-            gr.Markdown(
-                "\* Results when the scores of 4 and 5 were treated as ties.",
-                elem_classes="markdown-text",
-            )
-
-            gr.Markdown(
-                "Let us know in [this discussion](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/65) which models we should add!",
-                elem_id="models-to-add-text",
-            )
-
-    dummy = gr.Textbox(visible=False)
-    demo.load(
-        change_tab,
-        dummy,
-        tabs,
-        _js=get_window_url_params,
-    )
     if ADD_PLOTS:
         with gr.Box():
             visualization_title = gr.HTML(VISUALIZATION_TITLE)
@@ -512,6 +150,20 @@ with demo:
                     gr.Markdown(f"#### Figure 4: {PLOT_4_TITLE}")
                     plot_4 = gr.Plot(plot_4, show_label=False)
 
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    elem_id="citation-button",
+                ).style(show_copy_button=True)
+        with gr.Column():
+            with gr.Accordion("✨ CHANGELOG", open=False):
+                changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
+
+
+
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()

src/assets/hardcoded_evals.py

@@ -1,38 +0,0 @@
-from src.utils_display import AutoEvalColumn, model_hyperlink
-
-gpt4_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.is_8bit.name: None,
-    AutoEvalColumn.average.name: 84.3,
-    AutoEvalColumn.arc.name: 96.3,
-    AutoEvalColumn.hellaswag.name:  95.3,
-    AutoEvalColumn.mmlu.name:  86.4,
-    AutoEvalColumn.truthfulqa.name:  59.0,
-    AutoEvalColumn.dummy.name: "GPT-4",
-}
-
-gpt35_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.is_8bit.name: None,
-    AutoEvalColumn.average.name: 71.9,
-    AutoEvalColumn.arc.name: 85.2,
-    AutoEvalColumn.hellaswag.name:  85.5,
-    AutoEvalColumn.mmlu.name:  70.0,
-    AutoEvalColumn.truthfulqa.name:  47.0,
-    AutoEvalColumn.dummy.name: "GPT-3.5",
-}
-
-baseline = {
-    AutoEvalColumn.model.name: "<p>Baseline</p>",
-    AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.is_8bit.name: None,
-    AutoEvalColumn.average.name: 25.0,
-    AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name:  25.0,
-    AutoEvalColumn.mmlu.name:  25.0,
-    AutoEvalColumn.truthfulqa.name:  25.0,
-    AutoEvalColumn.dummy.name: "baseline",
-}
-

src/assets/text_content.py

@@ -54,24 +54,12 @@ CHANGELOG_TEXT = f"""
 - Release the leaderboard to public
 """
 
-TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard (Humans and GPT4 evaluations) </h1>"""
 
 INTRODUCTION_TEXT = f"""
 📐 With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art. The 🤗 Open LLM Leaderboard aims to track, rank and evaluate LLMs and chatbots as they are released. 
 
-🤗 A key advantage of this leaderboard is that anyone from the community can submit a model for automated evaluation on the 🤗 GPU cluster, as long as it is a 🤗 Transformers model with weights on the Hub. We also support evaluation of models with delta-weights for non-commercial licensed models, such as LLaMa.
-
-📈 In the **first tab (LLM Benchmarks)**, we evaluate models on 4 key benchmarks from the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks. In the **second tab (Human & GPT Evaluations)**, the evaluations are performed by having humans and GPT-4 compare completions from a set of popular open-source language models (LLMs) on a secret set of instruction prompts.
-"""
-
-LLM_BENCHMARKS_TEXT = f"""
-Evaluation is performed against 4 popular benchmarks:
-- <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
-- <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
-- <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
-- <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model’s propensity to reproduce falsehoods commonly found online.
-
-We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
+📈 Here, the evaluations are performed by having humans and GPT-4 compare completions from a set of popular open-source language models (LLMs) on a secret set of instruction prompts.
 """
 
 HUMAN_GPT_EVAL_TEXT = f"""
@@ -83,10 +71,6 @@ For more information on the calibration and initiation of these measurements, pl
 """
 
 
-EVALUATION_QUEUE_TEXT = f"""
-# Evaluation Queue for the 🤗 Open LLM Leaderboard, these models will be automatically evaluated on the 🤗 cluster
-"""
-
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
   author = {Edward Beeching, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, Thomas Wolf},
@@ -121,38 +105,6 @@ CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
   version      = {v0.0.1},
   doi          = {10.5281/zenodo.5371628},
   url          = {https://doi.org/10.5281/zenodo.5371628}
-}
-@misc{clark2018think,
-      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge}, 
-      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
-      year={2018},
-      eprint={1803.05457},
-      archivePrefix={arXiv},
-      primaryClass={cs.AI}
-}
-@misc{zellers2019hellaswag,
-      title={HellaSwag: Can a Machine Really Finish Your Sentence?}, 
-      author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
-      year={2019},
-      eprint={1905.07830},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-@misc{hendrycks2021measuring,
-      title={Measuring Massive Multitask Language Understanding}, 
-      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
-      year={2021},
-      eprint={2009.03300},
-      archivePrefix={arXiv},
-      primaryClass={cs.CY}
-}
-@misc{lin2022truthfulqa,
-      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
-      author={Stephanie Lin and Jacob Hilton and Owain Evans},
-      year={2022},
-      eprint={2109.07958},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
 }"""
 
 VISUALIZATION_TITLE = """<h1 align="center" id="space-title">📊 Visualizations</h1>"""

src/auto_leaderboard/get_model_metadata.py

@@ -1,56 +0,0 @@
-import re
-from typing import List
-
-from src.utils_display import AutoEvalColumn
-from src.auto_leaderboard.model_metadata_type import get_model_type
-
-from huggingface_hub import HfApi
-import huggingface_hub
-api = HfApi()
-
-
-def get_model_infos_from_hub(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        model_name = model_data["model_name_for_query"]
-        try:
-            model_info = api.model_info(model_name)
-        except huggingface_hub.utils._errors.RepositoryNotFoundError:
-            model_data[AutoEvalColumn.license.name] = None
-            model_data[AutoEvalColumn.likes.name] = None
-            model_data[AutoEvalColumn.params.name] = None
-            continue
-
-        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
-        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
-        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
-
-
-def get_model_license(model_info):
-    try:
-        return model_info.cardData["license"]
-    except Exception:
-        return None
-
-def get_model_likes(model_info):
-    return model_info.likes
-
-size_pattern = re.compile(r"\d+(b|m)")
-
-def get_model_size(model_name, model_info):
-    # In billions
-    try:
-        return round(model_info.safetensors["total"] / 1e9, 3) 
-    except AttributeError:
-        #print(f"Repository {model_id} does not have safetensors weights")
-        pass
-    try:
-        size_match = re.search(size_pattern, model_name.lower())
-        size = size_match.group(0)
-        return round(int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3, 3)
-    except AttributeError:
-        return None
-
-
-def apply_metadata(leaderboard_data: List[dict]):
-    get_model_type(leaderboard_data)
-    get_model_infos_from_hub(leaderboard_data)

src/auto_leaderboard/load_results.py

@@ -1,116 +0,0 @@
-from dataclasses import dataclass
-
-import glob
-import json
-from typing import Dict, List, Tuple
-
-from src.utils_display import AutoEvalColumn, make_clickable_model
-import numpy as np
-
-# clone / pull the lmeh eval data
-METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
-BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
-BENCH_TO_NAME = {
-    "arc_challenge": AutoEvalColumn.arc.name,
-    "hellaswag": AutoEvalColumn.hellaswag.name,
-    "hendrycks": AutoEvalColumn.mmlu.name,
-    "truthfulqa_mc": AutoEvalColumn.truthfulqa.name,
-}
-
-
-@dataclass
-class EvalResult:
-    eval_name: str
-    org: str
-    model: str
-    revision: str
-    is_8bit: bool
-    results: dict
-
-    def to_dict(self):
-        if self.org is not None:
-            base_model = f"{self.org}/{self.model}"
-        else:
-            base_model = f"{self.model}"
-        data_dict = {}
-
-        data_dict["eval_name"] = self.eval_name # not a column, just a save name
-        data_dict[AutoEvalColumn.is_8bit.name] = self.is_8bit
-        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
-        data_dict[AutoEvalColumn.dummy.name] = base_model
-        data_dict[AutoEvalColumn.revision.name] = self.revision
-        data_dict[AutoEvalColumn.average.name] = round(
-            sum([v for k, v in self.results.items()]) / 4.0, 1
-        )
-
-        for benchmark in BENCHMARKS:
-            if not benchmark in self.results.keys():
-                self.results[benchmark] = None
-
-        for k, v in BENCH_TO_NAME.items():
-            data_dict[v] = self.results[k]
-
-        return data_dict
-
-
-def parse_eval_result(json_filepath: str) -> Tuple[str, dict]:
-    with open(json_filepath) as fp:
-        data = json.load(fp)
-
-    path_split = json_filepath.split("/")
-    org = None
-    model = path_split[-4]
-    is_8bit = path_split[-2] == "8bit"
-    revision = path_split[-3]
-    if len(path_split) == 7:
-        # handles gpt2 type models that don't have an org
-        result_key = f"{model}_{revision}_{is_8bit}"
-    else:
-        org = path_split[-5]
-        result_key =  f"{org}_{model}_{revision}_{is_8bit}"
-
-    eval_result = None
-    for benchmark, metric in zip(BENCHMARKS, METRICS):
-        if benchmark in json_filepath:
-            accs = np.array([v[metric] for v in data["results"].values()])
-            mean_acc = round(np.mean(accs) * 100.0, 1)
-            eval_result = EvalResult(
-                result_key, org, model, revision, is_8bit, {benchmark: mean_acc}
-            )
-
-    return result_key, eval_result
-
-
-def get_eval_results(is_public) -> List[EvalResult]:
-    json_filepaths = glob.glob(
-        "auto_evals/eval_results/public/**/16bit/*.json", recursive=True
-    )
-    if not is_public:
-        json_filepaths += glob.glob(
-            "auto_evals/eval_results/private/**/*.json", recursive=True
-        )
-        json_filepaths += glob.glob(
-            "auto_evals/eval_results/private/**/*.json", recursive=True
-        )
-        # include the 8bit evals of public models
-        json_filepaths += glob.glob(
-            "auto_evals/eval_results/public/**/8bit/*.json", recursive=True
-        )  
-    eval_results = {}
-
-    for json_filepath in json_filepaths:
-        result_key, eval_result = parse_eval_result(json_filepath)
-        if result_key in eval_results.keys():
-            eval_results[result_key].results.update(eval_result.results)
-        else:
-            eval_results[result_key] = eval_result
-
-    eval_results = [v for v in eval_results.values()]
-
-    return eval_results
-
-
-def get_eval_results_dicts(is_public=True) -> List[Dict]:
-    eval_results = get_eval_results(is_public)
-
-    return [e.to_dict() for e in eval_results]

src/auto_leaderboard/model_metadata_type.py

@@ -1,163 +0,0 @@
-from enum import Enum
-from typing import Dict, List
-
-class ModelType(Enum):
-    PT = "pretrained"
-    SFT = "finetuned"
-    RL = "with RL"
-
-
-TYPE_METADATA: Dict[str, ModelType] = {
-    "aisquared/dlite-v1-355m": ModelType.SFT,
-    "aisquared/dlite-v2-774m": ModelType.SFT,
-    "aisquared/dlite-v2-1_5b": ModelType.SFT,
-    "TheBloke/wizardLM-7B-HF": ModelType.SFT,
-    "TheBloke/dromedary-65b-lora-HF": ModelType.SFT,
-    "TheBloke/vicuna-13B-1.1-HF": ModelType.SFT,
-    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.SFT,
-    "wordcab/llama-natural-instructions-13b": ModelType.SFT,
-    "JosephusCheung/Guanaco": ModelType.SFT,
-    "AlekseyKorshuk/vicuna-7b": ModelType.SFT,
-    "AlekseyKorshuk/chatml-pyg-v1": ModelType.SFT,
-    "concedo/OPT-19M-ChatSalad": ModelType.SFT,
-    "digitous/Javalion-R": ModelType.SFT,
-    "digitous/Alpacino30b": ModelType.SFT,
-    "digitous/Javelin-GPTJ": ModelType.SFT,
-    "anton-l/gpt-j-tiny-random": ModelType.SFT,
-    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.SFT,
-    "gpt2-medium": ModelType.PT,
-    "PygmalionAI/pygmalion-6b": ModelType.SFT,
-    "medalpaca/medalpaca-7b": ModelType.SFT,
-    "medalpaca/medalpaca-13b": ModelType.SFT,
-    "chavinlo/alpaca-13b": ModelType.SFT,
-    "chavinlo/alpaca-native": ModelType.SFT,
-    "chavinlo/gpt4-x-alpaca": ModelType.SFT,
-    "hakurei/lotus-12B": ModelType.SFT,
-    "amazon/LightGPT": ModelType.SFT,
-    "shibing624/chinese-llama-plus-13b-hf": ModelType.SFT,
-    "mosaicml/mpt-7b": ModelType.PT,
-    "PSanni/Deer-3b": ModelType.SFT,
-    "bigscience/bloom-1b1": ModelType.PT,
-    "MetaIX/GPT4-X-Alpasta-30b": ModelType.SFT,
-    "EleutherAI/gpt-neox-20b": ModelType.PT,
-    "EleutherAI/gpt-j-6b": ModelType.PT,
-    "roneneldan/TinyStories-28M": ModelType.SFT,
-    "lmsys/vicuna-13b-delta-v1.1": ModelType.SFT,
-    "lmsys/vicuna-7b-delta-v1.1": ModelType.SFT,
-    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.SFT,
-    "pillowtalks-ai/delta13b": ModelType.SFT,
-    "bigcode/starcoderplus": ModelType.SFT,
-    "microsoft/DialoGPT-large": ModelType.SFT,
-    "microsoft/CodeGPT-small-py": ModelType.SFT,
-    "Pirr/pythia-13b-deduped-green_devil": ModelType.SFT,
-    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.SFT,
-    "Aeala/VicUnlocked-alpaca-30b": ModelType.SFT,
-    "dvruette/llama-13b-pretrained-sft-epoch-2": ModelType.SFT,
-    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.SFT,
-    "openlm-research/open_llama_3b_350bt_preview": ModelType.PT,
-    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
-    "openlm-research/open_llama_7b": ModelType.PT,
-    "openlm-research/open_llama_3b": ModelType.PT,
-    "openlm-research/open_llama_7b_400bt_preview": ModelType.PT,
-    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.SFT,
-    "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k": ModelType.SFT,
-    "databricks/dolly-v2-7b": ModelType.SFT,
-    "databricks/dolly-v2-3b": ModelType.SFT,
-    "databricks/dolly-v2-12b": ModelType.SFT,
-    "pinkmanlove/llama-65b-hf": ModelType.SFT,
-    "Rachneet/gpt2-xl-alpaca": ModelType.SFT,
-    "Locutusque/gpt2-conversational-or-qa": ModelType.SFT,
-    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.SFT,
-    "Fredithefish/ScarletPajama-3B-HF": ModelType.SFT,
-    "eachadea/vicuna-7b-1.1": ModelType.SFT,
-    "eachadea/vicuna-13b": ModelType.SFT,
-    "openaccess-ai-collective/wizard-mega-13b": ModelType.SFT,
-    "openaccess-ai-collective/manticore-13b": ModelType.SFT,
-    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.SFT,
-    "openaccess-ai-collective/minotaur-13b": ModelType.SFT,
-    "lamini/instruct-tuned-3b": ModelType.SFT,
-    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.SFT,
-    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.SFT,
-    "stabilityai/stablelm-tuned-alpha-7b": ModelType.SFT,
-    "CalderaAI/30B-Lazarus": ModelType.SFT,
-    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.SFT,
-    "distilgpt2": ModelType.PT,
-    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.SFT,
-    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.SFT,
-    "junelee/wizard-vicuna-13b": ModelType.SFT,
-    "BreadAi/StoryPy": ModelType.SFT,
-    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
-    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
-    "Writer/camel-5b-hf": ModelType.SFT,
-    "Writer/palmyra-base": ModelType.PT,
-    "MBZUAI/lamini-neo-125m": ModelType.SFT,
-    "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4": ModelType.SFT,
-    "vicgalle/gpt2-alpaca-gpt4": ModelType.SFT,
-    "facebook/opt-350m": ModelType.PT,
-    "facebook/opt-125m": ModelType.PT,
-    "facebook/opt-13b": ModelType.PT,
-    "facebook/opt-1.3b": ModelType.PT,
-    "facebook/opt-66b": ModelType.PT,
-    "facebook/galactica-120b": ModelType.PT,
-    "Abe13/jgpt2-v1": ModelType.SFT,
-    "gpt2-xl": ModelType.PT,
-    "HuggingFaceH4/stable-vicuna-13b-2904": ModelType.RL,
-    "HuggingFaceH4/llama-7b-ift-alpaca": ModelType.SFT,
-    "HuggingFaceH4/starchat-alpha": ModelType.SFT,
-    "HuggingFaceH4/starchat-beta": ModelType.SFT,
-    "ausboss/Llama30B-SuperHOT": ModelType.SFT,
-    "ausboss/llama-13b-supercot": ModelType.SFT,
-    "ausboss/llama-30b-supercot": ModelType.SFT,
-    "Neko-Institute-of-Science/metharme-7b": ModelType.SFT,
-    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.SFT,
-    "victor123/WizardLM-13B-1.0": ModelType.SFT,
-    "AlpinDale/pygmalion-instruct": ModelType.SFT,
-    "tiiuae/falcon-7b-instruct": ModelType.SFT,
-    "tiiuae/falcon-40b-instruct": ModelType.SFT,
-    "tiiuae/falcon-40b": ModelType.PT,
-    "tiiuae/falcon-7b": ModelType.PT,
-    "cyl/awsome-llama": ModelType.SFT,
-    "xzuyn/Alpacino-SuperCOT-13B": ModelType.SFT,
-    "xzuyn/MedicWizard-7B": ModelType.SFT,
-    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.SFT,
-    "chainyo/alpaca-lora-7b": ModelType.SFT,
-    "Salesforce/codegen-16B-nl": ModelType.PT,
-    "Salesforce/codegen-16B-multi": ModelType.SFT,
-    "ai-forever/rugpt3large_based_on_gpt2": ModelType.SFT,
-    "gpt2-large": ModelType.PT,
-    "huggingface/llama-13b": ModelType.PT,
-    "huggingface/llama-7b": ModelType.PT,
-    "huggingface/llama-65b": ModelType.PT,
-    "huggingface/llama-30b": ModelType.PT,
-    "jondurbin/airoboros-7b": ModelType.SFT,
-    "jondurbin/airoboros-13b": ModelType.SFT,
-    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
-    "cerebras/Cerebras-GPT-111M": ModelType.PT,
-    "NousResearch/Nous-Hermes-13b": ModelType.SFT,
-    "project-baize/baize-v2-7b": ModelType.SFT,
-    "project-baize/baize-v2-13b": ModelType.SFT,
-    "LLMs/AlpacaGPT4-7B-elina": ModelType.SFT,
-    "LLMs/Vicuna-EvolInstruct-13B": ModelType.SFT,
-    "huggingtweets/jerma985": ModelType.SFT,
-    "huggyllama/llama-65b": ModelType.PT,
-    "WizardLM/WizardLM-13B-1.0": ModelType.SFT,
-    "gpt2": ModelType.PT,
-    "alessandropalla/instruct_gpt2": ModelType.SFT,
-    "MayaPH/FinOPT-Lincoln": ModelType.SFT,
-    "MayaPH/FinOPT-Franklin": ModelType.SFT,
-    "timdettmers/guanaco-33b-merged": ModelType.SFT,
-    "timdettmers/guanaco-65b-merged": ModelType.SFT,
-    "elinas/llama-30b-hf-transformers-4.29": ModelType.SFT,
-    "elinas/chronos-33b": ModelType.SFT,
-    "nmitchko/medguanaco-65b-GPTQ": ModelType.SFT,
-    "xhyi/PT_GPTNEO350_ATG": ModelType.SFT,
-    "h2oai/h2ogpt-oasst1-512-20b": ModelType.SFT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.SFT,
-    "nomic-ai/gpt4all-13b-snoozy": ModelType.SFT,
-    "nomic-ai/gpt4all-j": ModelType.SFT,
-}
-
-
-def get_model_type(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")

src/elo_leaderboard/load_results.py

@@ -143,6 +143,7 @@ def get_elo_results(df_instruct, df_code_instruct, tie_allowed):
         "gpt_4_evals/data/",
         split="train",
         revision="e007baaf6e505731c08a0bc1a833a1f8f8cb8846",
+        
     ).to_pandas()
 
     dfs = [df_instruct, df_code_instruct, df_all]

src/init.py

@@ -15,27 +15,7 @@ def get_all_requested_models(requested_models_dir):
 
     return set([file_name.lower().split("eval_requests/")[1] for file_name in file_names])
 
-def load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
-    auto_eval_repo = None
-    requested_models = None
-    if H4_TOKEN:
-        print("Pulling evaluation requests and results.")
-        # try:
-        #     shutil.rmtree("./auto_evals/")
-        # except:
-        #     pass
-
-        auto_eval_repo = Repository(
-            local_dir="./auto_evals/",
-            clone_from=LMEH_REPO,
-            use_auth_token=H4_TOKEN,
-            repo_type="dataset",
-        )
-        auto_eval_repo.git_pull()
-
-        requested_models_dir = "./auto_evals/eval_requests"
-        requested_models = get_all_requested_models(requested_models_dir)
-
+def load_all_info_from_hub(HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
     human_eval_repo = None
     if H4_TOKEN and not os.path.isdir("./human_evals"):
         print("Pulling human evaluation repo")
@@ -58,7 +38,7 @@ def load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
         )
         gpt_4_eval_repo.git_pull()
 
-    return auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models
+    return human_eval_repo, gpt_4_eval_repo
 
 
 #def load_results(model, benchmark, metric):

src/utils_display.py

@@ -12,22 +12,6 @@ class ColumnContent:
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
-@dataclass(frozen=True)
-class AutoEvalColumn: # Auto evals column
-    model = ColumnContent("Model", "markdown", True)
-    revision = ColumnContent("Revision", "str", True, True)
-    model_type = ColumnContent("Type", "bool", False)
-    is_8bit = ColumnContent("8bit", "bool", False, True)
-    license = ColumnContent("Hub License", "str", False)
-    params = ColumnContent("#Params (B)", "number", False)
-    likes = ColumnContent("Hub ❤️", "number", False)
-    average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("ARC (25-s) ⬆️", "number", True)
-    hellaswag = ColumnContent("HellaSwag (10-s) ⬆️", "number", True)
-    mmlu = ColumnContent("MMLU (5-s) ⬆️", "number", True)
-    truthfulqa = ColumnContent("TruthfulQA (MC) (0-s) ⬆️", "number", True)
-    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
-
 @dataclass(frozen=True)
 class EloEvalColumn: # Elo evals column
     model = ColumnContent("Model", "markdown", True)
@@ -36,16 +20,6 @@ class EloEvalColumn: # Elo evals column
     human_instruct = ColumnContent("Human (instruct)", "number", True)
     human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
 
-
-@dataclass(frozen=True)
-class EvalQueueColumn: # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    is_8bit = ColumnContent("8bit_eval", "bool", True)
-    has_delta_weight = ColumnContent("is_delta_weight", "bool", True)
-    status = ColumnContent("status", "str", True)
-
 LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]