made lots of changes

app.py

@@ -95,7 +95,7 @@ with gr.Blocks(css=custom_css) as app:
             # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
             # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
             gr.Markdown("""
-                        ![](file/src/logo.png)
+                        <img src="file/src/logo.png" height="200">
                         """)
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏆 HREF Leaderboard"):
@@ -149,8 +149,7 @@ with gr.Blocks(css=custom_css) as app:
         with gr.TabItem("Dataset Viewer"):
             with gr.Row():
                 # loads one sample
-                gr.Markdown("""## Random Dataset Sample Viewer
-Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
+                gr.Markdown("""## Random Dataset Sample Viewer""")
                 subset_selector = gr.Dropdown(subsets, label="Category", value=None, multiselect=True)
                 button = gr.Button("Show Random Sample")
 

app_old.py

@@ -1,464 +0,0 @@
-import gradio as gr
-import os
-from huggingface_hub import HfApi, snapshot_download
-from apscheduler.schedulers.background import BackgroundScheduler
-from datasets import load_dataset
-from src.utils_old import load_all_data
-from src.md import ABOUT_TEXT, TOP_TEXT
-from src.constants import subset_mapping, length_categories, example_counts
-from src.css import custom_css
-import numpy as np
-
-api = HfApi()
-
-COLLAB_TOKEN = os.environ.get("COLLAB_TOKEN")
-evals_repo = "allenai/reward-bench-results"
-
-eval_set_repo = "allenai/reward-bench"
-repo_dir_rewardbench = "./evals/rewardbench/"
-
-def restart_space():
-    api.restart_space(repo_id="allenai/reward-bench", token=COLLAB_TOKEN)
-
-print("Pulling evaluation results")
-repo = snapshot_download(
-    local_dir=repo_dir_rewardbench,
-    ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
-    repo_id=evals_repo,
-    use_auth_token=COLLAB_TOKEN,
-    tqdm_class=None, 
-    etag_timeout=30,
-    repo_type="dataset",
-)
-
-
-def avg_over_rewardbench(dataframe_core, dataframe_prefs):
-    """
-    Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
-
-    We average over 4 core sections (per prompt weighting):
-    1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
-    2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
-    3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
-    4. Reasoning: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
-    5. Prior Sets (0.5 weight): Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
-    """
-    new_df = dataframe_core.copy()
-    dataframe_prefs = dataframe_prefs.copy()
-
-    # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
-    for subset, sub_subsets in subset_mapping.items():
-        subset_cols = [col for col in new_df.columns if col in sub_subsets]
-        sub_data = new_df[subset_cols].values # take the relevant column values
-        sub_counts = [example_counts[s] for s in subset_cols] # take the example counts
-        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
-        # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
-
-    data_cols = list(subset_mapping.keys())
-    keep_columns = ["model",] + ["model_type"] + data_cols
-    # keep_columns = ["model", "average"] + subsets
-    new_df = new_df[keep_columns]
-
-    # selected average from pref_sets
-    pref_columns = ["anthropic_helpful", "anthropic_hhh", "shp", "summarize"]
-    pref_data = dataframe_prefs[pref_columns].values
-
-    # add column test sets knowing the rows are not identical, take superset
-    dataframe_prefs["Prior Sets (0.5 weight)"] = np.nanmean(pref_data, axis=1)
-
-    # add column Test Sets empty to new_df
-    new_df["Prior Sets (0.5 weight)"] = np.nan
-    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets (0.5 weight)"]
-    values = []
-    for i, row in new_df.iterrows():
-        model = row["model"]
-        if model in dataframe_prefs["model"].values:
-            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0])
-            # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
-        else:
-            values.append(np.nan)
-    
-    new_df["Prior Sets (0.5 weight)"] = values
-
-    # add total average
-    data_cols += ["Prior Sets (0.5 weight)"]
-    final_data = new_df[data_cols].values
-    masked_data = np.ma.masked_array(final_data, np.isnan(final_data))
-    weights = [2, 2, 2, 2, 1]
-    average = np.ma.average(masked_data, axis=1, weights=weights)
-    new_df["average"] = average.filled(np.nan)
-    # new_df["average"] = np.nanmean(new_df[data_cols].values, axis=1)
-
-    # make average third column
-    keep_columns = ["model", "model_type", "average"] + data_cols
-    new_df = new_df[keep_columns]
-    return new_df
-
-def expand_subsets(dataframe):
-    # TODO need to modify data/ script to do this
-    pass
-
-
-def length_bias_check(dataframe):
-    """
-    Takes the raw rewardbench dataframe and splits the data into new buckets according to length_categories.
-    Then, take the average of the three buckets as "average"
-    """
-    new_df = dataframe.copy()
-    existing_subsets = new_df.columns[3:] # model, model_type, average
-    final_subsets = ["Length Bias", "Neutral", "Terse Bias"]
-    # new data is empty list dict for each final subset
-    new_data = {s: [] for s in final_subsets}
-
-    # now, subsets correspond to those with True, Nuetral, and False length bias
-    # check if length_categories[subset] == "True" or "False" or "Neutral"
-    for subset in existing_subsets:
-        subset_data = new_df[subset].values
-        subset_length = length_categories[subset]
-        # route to the correct bucket
-        if subset_length == "True":
-            new_data["Length Bias"].append(subset_data)
-        elif subset_length == "Neutral":
-            new_data["Neutral"].append(subset_data)
-        elif subset_length == "False":
-            new_data["Terse Bias"].append(subset_data)
-
-    # take average of new_data and add to new_df (removing other columns than model)
-    for subset in final_subsets:
-        new_df[subset] = np.nanmean(new_data[subset], axis=0)
-    keep_columns = ["model"] + final_subsets
-    new_df = new_df[keep_columns]
-    # recompute average
-    # new_df["average"] = np.round(np.nanmean(new_df[final_subsets].values, axis=1), 2)
-
-    return new_df
-
-
-    
-rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False)
-rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False)
-prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False)
-# prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
-
-rewardbench_data_avg = avg_over_rewardbench(rewardbench_data, prefs_data).sort_values(by='average', ascending=False)
-
-def prep_df(df):
-    # add column to 0th entry with count (column name itself empty)
-    df.insert(0, '', range(1, 1 + len(df)))
-
-    # replace "model" with "Model" and "model_type" with "Model Type" and "average" with "Average"
-    df = df.rename(columns={"model": "Model", "model_type": "Model Type", "average": "Average"})
-
-    # if "Model Type" in columns
-    if "Model Type" in df.columns:
-        # get model_types that have generative in them
-        mask = df["Model Type"].str.contains("generative", case=False, na=False)
-        
-        # set these values to "Generative"
-        df.loc[mask, "Model Type"] = "Generative"
-
-    return df
-
-# add count column to all dataframes
-rewardbench_data = prep_df(rewardbench_data)
-rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
-# adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others
-
-rewardbench_data_length = prep_df(rewardbench_data_length)
-prefs_data = prep_df(prefs_data)
-
-col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
-col_types_rewardbench_avg = ["number"] + ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
-cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
-col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
-# col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1)
-
-# for showing random samples
-eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="filtered")
-def random_sample(r: gr.Request, subset):
-    if subset is None or subset == []:
-        sample_index = np.random.randint(0, len(eval_set) - 1)
-        sample = eval_set[sample_index]
-    else: # filter by subsets (can be list)
-        if isinstance(subset, str):
-            subset = [subset]
-        # filter down dataset to only include the subset(s)
-        eval_set_filtered = eval_set.filter(lambda x: x["subset"] in subset)
-        sample_index = np.random.randint(0, len(eval_set_filtered) - 1)
-        sample = eval_set_filtered[sample_index]
-
-    markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
-    return markdown_text
-
-subsets = eval_set.unique("subset")
-
-color_map = {
-    "Generative": "#7497db",
-    "Custom Classifier": "#E8ECF2",
-    "Seq. Classifier": "#ffcd75",
-    "DPO": "#75809c",
-}
-def color_model_type_column(df, color_map):
-    """
-    Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.
-
-    Parameters:
-    df (pd.DataFrame): The DataFrame containing the 'Model Type' column.
-    color_map (dict): A dictionary mapping model types to colors.
-
-    Returns:
-    pd.Styler: The styled DataFrame.
-    """
-    # Function to apply color based on the model type
-    def apply_color(val):
-        color = color_map.get(val, "default")  # Default color if not specified in color_map
-        return f'background-color: {color}'
-    
-    # Format for different columns
-    format_dict = {col: "{:.1f}" for col in df.columns if col not in ['Average', 'Model', 'Model Type']}
-    format_dict['Average'] = "{:.2f}"
-    format_dict[''] = "{:d}"
-
-    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')
-
-def regex_table(dataframe, regex, filter_button, style=True):
-    """
-    Takes a model name as a regex, then returns only the rows that has that in it.
-    """
-    # Split regex statement by comma and trim whitespace around regexes
-    regex_list = [x.strip() for x in regex.split(",")]
-    # Join the list into a single regex pattern with '|' acting as OR
-    combined_regex = '|'.join(regex_list)
-
-    # remove internal ai2 data
-    dataframe = dataframe[~dataframe["Model"].str.contains("ai2", case=False, na=False)]
-    
-    # if filter_button, remove all rows with "ai2" in the model name
-    update_scores = False
-    if isinstance(filter_button, list) or isinstance(filter_button, str):
-        if "Prior Sets" not in filter_button and 'Prior Sets (0.5 weight)' in dataframe.columns:
-            update_scores = True
-            # remove the column "Prior Sets (0.5 weight)" from the outputted table
-            dataframe = dataframe.drop(columns=['Prior Sets (0.5 weight)'])
-        if "Seq. Classifiers" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("Seq. Classifier", case=False, na=False)]
-        if "DPO" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("DPO", case=False, na=False)]
-        if "Custom Classifiers" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("Custom Classifier", case=False, na=False)]
-        if "Generative" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("generative", case=False, na=False)]
-    # Filter the dataframe such that 'model' contains any of the regex patterns
-    data = dataframe[dataframe["Model"].str.contains(combined_regex, case=False, na=False)]
-
-    # if update the score to not use prior sets, do so
-    if update_scores:
-        data["Score"] = (data["Chat"] + data["Chat Hard"] + data["Safety"] + data["Reasoning"]) / 4
-        # if "Prior Sets (0.5 weight)" in data.columns:
-        # data["Prior Sets (0.5 weight)"] = np.nan
-        # sort array by Score column
-        data = data.sort_values(by='Score', ascending=False)
-
-    data.reset_index(drop=True, inplace=True)
-
-    # replace column '' with count/rank
-    data[''] = np.arange(1, 1 + len(data))
-
-    # if Score exists, round to 2 decimals
-    if "Score" in data.columns:
-        data["Score"] = np.round(np.array(data["Score"].values).astype(float), 2)
-    if "Average" in data.columns:
-        data["Average"] = np.round(np.array(data["Average"].values).astype(float), 1)
-    # round all others to 1 decimal
-    for col in data.columns:
-        if col not in ["", "Model", "Model Type", "Score", "Average"]:
-            # replace any data[col].values == '' with np.nan
-            data[col] = data[col].replace('', np.nan)
-            data[col] = np.round(np.array(data[col].values).astype(float), 1)
-    if style:
-        # apply color
-        data = color_model_type_column(data, color_map)
-
-    return data
-
-
-def printout(df):
-    print(df.iloc[0])
-    print(df.iloc[1])
-
-# import ipdb; ipdb.set_trace()
-
-total_models = len(regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False).values)
-
-
-with gr.Blocks(css=custom_css) as app:
-    # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About"
-    with gr.Row():
-        with gr.Column(scale=6):
-            gr.Markdown(TOP_TEXT.format(str(total_models)))
-        with gr.Column(scale=4):
-            # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
-            # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
-            # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
-            gr.Markdown("""
-                        ![](file/src/logo.png)
-                        """)
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏆 RewardBench Leaderboard"):
-            with gr.Row():
-                search_1 = gr.Textbox(label="Model Search (delimit with , )", 
-                                      placeholder="Model Search (delimit with , )",
-                                      show_label=False)
-                model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative", "Prior Sets"], 
-                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include.",
-                                                 )
-            with gr.Row():
-                # reference data
-                rewardbench_table_hidden = gr.Dataframe(
-                    rewardbench_data_avg.values,
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
-                    visible=False,
-                )
-                rewardbench_table = gr.Dataframe(
-                    regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers",  "Generative"]),
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
-                    elem_id="rewardbench_dataframe_avg",
-                    max_height=1000,
-                )
-                
-        with gr.TabItem("🔍 RewardBench - Detailed"):
-            with gr.Row():
-                search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-                model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include."
-                                                 )
-            with gr.Row():
-                # ref data
-                rewardbench_table_detailed_hidden = gr.Dataframe(
-                    rewardbench_data.values,
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    visible=False,
-                )
-                rewardbench_table_detailed = gr.Dataframe(
-                    regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    elem_id="rewardbench_dataframe",
-                    max_height=1000,
-                )
-        # with gr.TabItem("rewardbench Eval Set - Length Bias"):
-        #     with gr.Row():
-        #         # backup
-        #         rewardbench_table_len_hidden = gr.Dataframe(
-        #             rewardbench_data_length.values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             visible=False,
-        #         )
-        #         rewardbench_table_len = gr.Dataframe(
-        #             regex_table(rewardbench_data_length.copy(), "", False).values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             elem_id="rewardbench_dataframe_length",
-        #             height=1000,
-        #         )
-        with gr.TabItem("Prior Test Sets"):
-            with gr.Row():
-                search_3 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-                model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
-                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers"], 
-                                                 label="Model Types",
-                                                 show_label=False, 
-                                                #  info="Which model types to include.",
-                                                 )
-            with gr.Row():
-                PREF_SET_TEXT = """
-                For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). Only the subsets Anthropic Helpful, Anthropic HHH, Stanford SHP, and OpenAI's Summarize data are used in the leaderboard ranking.
-                """
-                gr.Markdown(PREF_SET_TEXT)
-            with gr.Row():
-                # backup
-                pref_sets_table_hidden = gr.Dataframe(
-                    prefs_data.values,
-                    datatype=col_types_prefs,
-                    headers=prefs_data.columns.tolist(),
-                    visible=False,
-                )
-                pref_sets_table = gr.Dataframe(
-                    regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]),
-                    datatype=col_types_prefs,
-                    headers=prefs_data.columns.tolist(),
-                    elem_id="prefs_dataframe",
-                    max_height=1000,
-                )
-
-
-        with gr.TabItem("About"):
-            with gr.Row():
-                gr.Markdown(ABOUT_TEXT)
-
-        with gr.TabItem("Dataset Viewer"):
-            with gr.Row():
-                # loads one sample
-                gr.Markdown("""## Random Dataset Sample Viewer
-Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
-                subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
-                button = gr.Button("Show Random Sample")
-
-            with gr.Row():
-                sample_display = gr.Markdown("{sampled data loads here}")
-
-            button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
-        # removed plot because not pretty enough
-        # with gr.TabItem("Model Correlation"):
-        #     with gr.Row():
-        #         plot = plot_avg_correlation(rewardbench_data_avg, prefs_data)
-        #         gr.Plot(plot)
-
-    search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len)
-    search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)    
-    
-    model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)
-
-    with gr.Row():
-        with gr.Accordion("📚 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=r"""@misc{RewardBench,
-    title={RewardBench: Evaluating Reward Models for Language Modeling},
-    author={Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, LJ and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
-    year={2024},
-    howpublished={\url{https://huggingface.co/spaces/allenai/reward-bench}
-}""",
-                lines=7,
-                label="Copy the following to cite these results.",
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-# Load data when app starts, TODO make this used somewhere...
-# def load_data_on_start():
-#     data_rewardbench = load_all_data(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench)
-
-#     data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench_avg)
-
-#     data_prefs = load_all_data(repo_dir_prefs)
-#     pref_sets_table.update(data_prefs)
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
-scheduler.start()
-app.launch(allowed_paths=['src/']) # had .queue() before launch before... not sure if that's necessary

src/constants.py

@@ -1,60 +0,0 @@
-# reference for length bias categories
-length_categories = {
-    'alpacaeval-easy': 'True',
-    'alpacaeval-hard': 'True',
-    'alpacaeval-length': 'Neutral',
-    'donotanswer': 'False',
-    'hep-cpp': 'Neutral',
-    'hep-go': 'Neutral',
-    'hep-java': 'Neutral',
-    'hep-js': 'Neutral',
-    'hep-python': 'Neutral',
-    'hep-rust': 'Neutral',
-    'llmbar-adver-GPTInst': 'False',
-    'llmbar-adver-GPTOut': 'Neutral',
-    'llmbar-adver-manual': 'False',
-    'llmbar-adver-neighbor': 'False',
-    'llmbar-natural': 'Neutral',
-    'math-prm': 'Neutral',
-    'mt-bench-easy': 'False',
-    'mt-bench-hard': 'False',
-    'mt-bench-med': 'Neutral',
-    'refusals-dangerous': 'False',
-    'refusals-offensive': 'False',
-    'xstest-should-refuse': 'False',
-    'xstest-should-respond': 'True'
-}
-
-example_counts = {
-    "alpacaeval-easy": 100,
-    "alpacaeval-length": 95,
-    "alpacaeval-hard": 95,
-    "mt-bench-easy": 28,
-    "mt-bench-med": 40,
-    "mt-bench-hard": 37,
-    "math-prm": 984, # actual length 447, upweighting to be equal to code
-    "refusals-dangerous": 100,
-    "refusals-offensive": 100,
-    "llmbar-natural": 100,
-    "llmbar-adver-neighbor": 134,
-    "llmbar-adver-GPTInst": 92,
-    "llmbar-adver-GPTOut": 47,
-    "llmbar-adver-manual": 46,
-    "xstest-should-refuse": 154,
-    "xstest-should-respond": 250, # Note, refuse and respond were accidentally swapped until 9 Sept 2024
-    "donotanswer": 136,
-    "hep-cpp": 164,
-    "hep-go": 164,
-    "hep-java": 164,
-    "hep-js": 164,
-    "hep-python": 164,
-    "hep-rust": 164
-}
-
-# note, this order should match the dataframe.
-subset_mapping = {
-    "Chat": ['alpacaeval-easy', 'alpacaeval-hard', 'alpacaeval-length', 'mt-bench-easy', 'mt-bench-med'],
-    "Chat Hard": ['llmbar-adver-GPTInst', 'llmbar-adver-GPTOut', 'llmbar-adver-manual', 'llmbar-adver-neighbor', 'llmbar-natural', 'mt-bench-hard'],
-    "Safety": ['donotanswer', 'refusals-dangerous', 'refusals-offensive', 'xstest-should-refuse', 'xstest-should-respond'],
-    "Reasoning": ["hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust", "math-prm"]
-}

src/md.py

@@ -2,7 +2,24 @@ from datetime import datetime
 import pytz
 
 ABOUT_TEXT = """
-TODO
+HREF is evaluation benchmark that evaluates language models' capacity of following human instructions. It is consisted of 4,258 instructions covering 11 distinct categories, including Brainstorm ,Open QA ,Closed QA ,Extract ,Generation ,Rewrite ,Summarize ,Coding ,Classify ,Fact Checking or Attributed QA ,Multi-Document Synthesis , and Reasoning Over Numerical Data.
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/64dff1ddb5cc372803af964d/dSv3U11h936t_q-aiqbkV.png)
+
+## Why HREF
+| Benchmark          | Size  | Evaluation Method       | Baseline Model             | Judge Model        | Task Oriented | Contamination Resistant | Contains Human Reference|
+|--------------------|-------|------------|----------------|----------------|----------|------------|-----------|
+| MT-Bench          | 80    | Score      | ---            | gpt4           | ✓        | ✗          | ✗         |
+| AlpacaEval 2.0    | 805   | PWC        | gpt4-turbo     | gpt4-turbo     | ✗        | ✗          | ✗         |
+| Chatbot Arena     | ---   | PWC        | ---            | Human          | ✗        | ✓          | ✗         |
+| Arena-Hard        | 500   | PWC        | gpt4-0314      | gpt4-turbo     | ✗        | ✗          | ✗         |
+| WildBench         | 1,024 | Score/PWC  | gpt4-turbo     | three models   | ✗        | ✗          | ✗         |
+| **HREF**         | 4,258 | PWC        | Llama-3.1-405B-Instruct     | Llama-3.1-70B-Instruct      | ✓        | ✓          | ✓         |
+
+- **Human Reference**: HREF leverages human-written answer as reference to provide more reliable evaluation than previous method.
+- **Large**: HREF has the largest evaluation size among similar benchmarks, making its evaluation more reliable.
+- **Contamination-resistant**: HREF's evaluation set is hidden and uses public models for both the baseline model and judge model, which makes it completely free of contamination.
+- **Task Oriented**: Instead of naturally collected instructions from the user, HREF contains instructions that are written specifically targetting 8 distinct categories that are used in instruction tuning, which allows it to provide more insights about how to improve language models.
+
 """
 
 # Get Pacific time zone (handles PST/PDT automatically)
@@ -10,5 +27,5 @@ pacific_tz = pytz.timezone('America/Los_Angeles')
 current_time = datetime.now(pacific_tz).strftime("%H:%M %Z, %d %b %Y")
 
 TOP_TEXT = f"""# HREF: Human Reference Guided Evaluation for Instructiong Following
-[Code]() | [Eval. Dataset]() | [Prior Test Sets]() | [Results]() | [Paper]() | Total models: {{}} | * Unverified models | ⚠️ Dataset Contamination | Last restart (PST): {current_time}
+[Code]() | [Validation Set]() | [Human Agreement Set]() | [Results]() | [Paper]() | Total models: {{}} | * Unverified models | ⚠️ Dataset Contamination | Last restart (PST): {current_time}
 """

src/md_old.py

@@ -1,105 +0,0 @@
-from datetime import datetime
-import pytz
-
-ABOUT_TEXT = """
-We compute the win percentage for a reward model on hand curated chosen-rejected pairs for each prompt.
-A win is when the score for the chosen response is higher than the score for the rejected response.
-
-Note: Models with (*) after the model name are independently submitted model scores which have not been verified by the RewardBench team.
-
-## Overview
-
-We average over 4 core sections (per prompt weighting):
-1. **Chat**: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
-2. **Chat Hard**: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
-3. **Safety**: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
-4. **Reasoning**: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
-
-For Reasoning, we increase the weight of the PRM-Math subset so code and math abilities are weighed equally in the final number, rather than increasing the relevance of code.
-We add a final column, **Prior Sets** -- includes the test sets ([anthropic_helpful](https://huggingface.co/datasets/Anthropic/hh-rlhf), [anthropic_hhh](https://huggingface.co/datasets/HuggingFaceH4/hhh_alignment), [shp](https://huggingface.co/datasets/stanfordnlp/SHP), [summarize](https://huggingface.co/datasets/openai/summarize_from_feedback))
-Prior sets is weighted 0.5x in the final score to avoid gamification by training on the available training sets of Anthropic HH, SHP, and Summarize.
-
-Once all subsets weighted averages are achieved, the final RewardBench score is the average across the 5 subset scores.
-
-
-We include multiple types of reward models in this evaluation:
-1. **Sequence Classifiers** (Seq. Classifier): A model, normally trained with HuggingFace AutoModelForSequenceClassification, that takes in a prompt and a response and outputs a score.
-2. **Custom Classifiers**: Research models with different architectures and training objectives to either take in two inputs at once or generate scores differently (e.g. PairRM and Stanford SteamSHP).
-3. **DPO**: Models trained with Direct Preference Optimization (DPO), with modifiers such as `-ref-free` or `-norm` changing how scores are computed. *Note*: This also includes other models trained with implicit rewards, such as those trained with [KTO](https://arxiv.org/abs/2402.01306). 
-4. **Random**: Random choice baseline.
-4. **Generative**: Prompting fine-tuned models to choose between two answers, similar to MT Bench and AlpacaEval.
-
-All models are evaluated in fp16 expect for Starling-7B, which is evaluated in fp32.
-*Note*: The reference models for DPO models (and other implicit rewards) can be found in two ways.
-* Click on a specific model in results and you'll see a key `ref_model`, e.g. [Qwen](https://huggingface.co/datasets/allenai/reward-bench-results/blob/main/eval-set/Qwen/Qwen1.5-72B-Chat.json).
-* All the reference models are listed in the [evaluation configs](https://github.com/allenai/reward-bench/blob/main/scripts/configs/eval_configs.yaml).
-
-
-### Subset Details
-
-Total number of the prompts is: 2985, filtered from 5123.
-
-| Subset             | Num. Samples (Pre-filtering, post-filtering) | Description |
-| :---------- | :-----: | :---------: |
-| alpacaeval-easy    | 805, 100          | Great model vs poor model            |
-| alpacaeval-length    | 805, 95          | Good model vs low model, equal length            |
-| alpacaeval-hard    | 805, 95          | Great model vs baseline model            |
-| mt-bench-easy      | 28, 28           | MT Bench 10s vs 1s            |
-| mt-bench-medium    | 45, 40           | MT Bench 9s vs 2-5s            |
-| mt-bench-hard      | 45, 37          | MT Bench 7-8 vs 5-6            |
-| refusals-dangerous | 505, 100          | Dangerous response vs no response            |
-| refusals-offensive | 704, 100          | Offensive response vs no response            |
-| llmbar-natural     | 100          | (See [paper](https://arxiv.org/abs/2310.07641)) Manually curated instruction pairs |
-| llmbar-adver-neighbor | 134          | (See [paper](https://arxiv.org/abs/2310.07641)) Instruction response vs. off-topic prompt response |
-| llmbar-adver-GPTInst | 92          | (See [paper](https://arxiv.org/abs/2310.07641)) Instruction response vs. GPT4 generated off-topic prompt response |
-| llmbar-adver-GPTOut |  47          | (See [paper](https://arxiv.org/abs/2310.07641)) Instruction response vs. unhelpful-prompted GPT4 responses |
-| llmbar-adver-manual |  46          | (See [paper](https://arxiv.org/abs/2310.07641)) Challenge set chosen vs. rejected |
-| xstest-should-refuse | 450, 154         | False response dataset (see [paper](https://arxiv.org/abs/2308.01263))        |
-| xstest-should-respond | 450, 250         | False refusal dataset (see [paper](https://arxiv.org/abs/2308.01263))        |
-| do not answer | 939, 136         | [Prompts which responsible LLMs do not answer](https://huggingface.co/datasets/LibrAI/do-not-answer)        |
-| math-prm | 447         | Human references vs. model error from OpenAI's Let's Verify Step by Step        |
-| hep-cpp | 164         | C++ code revisions (See [dataset](https://huggingface.co/datasets/bigcode/humanevalpack) or [paper](https://arxiv.org/abs/2308.07124))        |
-| hep-go | 164         |   Go code       |
-| hep-java | 164         |  Java code        |
-| hep-js | 164         |    Javascript code        |
-| hep-python | 164         |  Python code         |
-| hep-rust | 164         |   Rust code        |
-
-Lengths (mean, std. dev.) include the prompt
-
-| subset                | length bias | chosen_chars   | rejected_chars   | chosen_tokens   | rejected_tokens   | chosen_unique_tokens   | rejected_unique_tokens   |
-|-----------------------|-------------|----------------|------------------|-----------------|-------------------|------------------------|--------------------------|
-| alpacaeval-easy       | True        | 2283 (1138)    | 646 (482)        | 591 (303)       | 167 (139)         | 253 (117)              | 83 (46)                  |
-| alpacaeval-hard       | True        | 1590 (769)     | 526 (430)        | 412 (199)       | 137 (117)         | 173 (67)               | 71 (48)                  |
-| alpacaeval-length     | Neutral       | 2001 (1137)    | 2127 (1787)      | 511 (283)       | 597 (530)         | 192 (85)               | 189 (99)                 |
-| donotanswer           | False       | 755 (722)      | 1389 (695)       | 170 (161)       | 320 (164)         | 104 (82)               | 157 (73)                 |
-| hep-cpp               | Neutral     | 709 (341)      | 705 (342)        | 261 (125)       | 259 (125)         | 100 (29)               | 99 (29)                  |
-| hep-go                | Neutral     | 738 (361)      | 734 (361)        | 266 (118)       | 265 (118)         | 100 (29)               | 99 (29)                  |
-| hep-java              | Neutral     | 821 (393)      | 814 (390)        | 263 (123)       | 261 (122)         | 102 (30)               | 102 (30)                 |
-| hep-js                | Neutral     | 677 (341)      | 673 (339)        | 251 (129)       | 250 (128)         | 93 (29)                | 93 (29)                  |
-| hep-python            | Neutral     | 618 (301)      | 616 (300)        | 212 (98)        | 211 (98)          | 86 (26)                | 85 (26)                  |
-| hep-rust              | Neutral     | 666 (391)      | 660 (391)        | 221 (132)       | 219 (132)         | 95 (29)                | 95 (29)                  |
-| llmbar-adver-GPTInst  | False       | 735 (578)      | 1623 (1055)      | 170 (135)       | 377 (245)         | 93 (59)                | 179 (106)                |
-| llmbar-adver-GPTOut   | Neutral     | 378 (339)      | 359 (319)        | 96 (81)         | 101 (94)          | 60 (45)                | 55 (41)                  |
-| llmbar-adver-manual   | False       | 666 (584)      | 1139 (866)       | 160 (134)       | 264 (194)         | 92 (63)                | 140 (90)                 |
-| llmbar-adver-neighbor | False       | 287 (297)      | 712 (749)        | 70 (76)         | 173 (175)         | 43 (31)                | 91 (70)                  |
-| llmbar-natural        | Neutral     | 553 (644)      | 530 (597)        | 139 (162)       | 130 (140)         | 75 (71)                | 70 (62)                  |
-| mt-bench-easy         | False       | 1563 (720)     | 2129 (1520)      | 377 (159)       | 551 (415)         | 166 (55)               | 116 (62)                 |
-| mt-bench-hard         | False       | 1225 (499)     | 1471 (1016)      | 284 (116)       | 349 (234)         | 131 (45)               | 136 (58)                 |
-| mt-bench-med          | Neutral       | 1558 (729)     | 1733 (1312)      | 377 (170)       | 410 (311)         | 162 (58)               | 145 (88)                 |
-| refusals-dangerous    | False       | 597 (81)       | 1828 (547)       | 131 (20)        | 459 (136)         | 90 (12)                | 211 (50)                 |
-| refusals-offensive    | False       | 365 (116)      | 1092 (1146)      | 82 (25)         | 299 (278)         | 64 (15)                | 134 (101)                |
-| xstest-should-refuse  | False       | 584 (419)      | 904 (493)        | 129 (89)        | 217 (115)         | 81 (47)                | 116 (53)                 |
-| xstest-should-respond | True        | 771 (420)      | 466 (427)        | 189 (105)       | 107 (94)          | 104 (48)               | 67 (48)                  |
-
-For more details, see the [dataset](https://huggingface.co/datasets/allenai/reward-bench).
-"""
-
-# Get Pacific time zone (handles PST/PDT automatically)
-pacific_tz = pytz.timezone('America/Los_Angeles')
-current_time = datetime.now(pacific_tz).strftime("%H:%M %Z, %d %b %Y")
-
-TOP_TEXT = f"""# RewardBench: Evaluating Reward Models
-### Evaluating the capabilities, safety, and pitfalls of reward models
-[Code](https://github.com/allenai/reward-bench) | [Eval. Dataset](https://huggingface.co/datasets/allenai/reward-bench) | [Prior Test Sets](https://huggingface.co/datasets/allenai/pref-test-sets) | [Results](https://huggingface.co/datasets/allenai/reward-bench-results) | [Paper](https://arxiv.org/abs/2403.13787) | Total models: {{}} | * Unverified models | ⚠️ Dataset Contamination | Last restart (PST): {current_time}
-"""

src/utils_old.py

@@ -1,171 +0,0 @@
-import pandas as pd
-from pathlib import Path
-from datasets import load_dataset
-import numpy as np
-import os
-import re
-
-UNVERIFIED_MODELS = [
-    "nvidia/Nemotron-4-340B-Reward",
-    "nvidia/Llama3-70B-SteerLM-RM",
-    "Cohere May 2024",
-    "google/gemini-1.5-pro-0514",
-    "google/flame-24b-july-2024",
-    "Cohere March 2024",
-    "facebook/Self-taught-Llama-3-70B",
-    "facebook/Self-taught-evaluator-llama3.1-70B",
-    "google/flame-1.0-24B-july-2024",
-    "Salesforce/SFR-LLaMa-3.1-70B-Judge-r",
-    "Salesforce/SFR-nemo-12B-Judge-r",
-    "Salesforce/SFR-LLaMa-3.1-8B-Judge-r",
-    "SF-Foundation/TextEval-OffsetBias-12B",
-    "SF-Foundation/TextEval-Llama3.1-70B",
-    "nvidia/Llama-3.1-Nemotron-70B-Reward",
-]
-
-CONTAMINATED_MODELS = [
-    "Skywork/Skywork-Reward-Gemma-2-27B",
-    "Skywork/Skywork-Critic-Llama-3.1-70B",
-    "LxzGordon/URM-LLaMa-3.1-8B",
-    "Skywork/Skywork-Reward-Llama-3.1-8B",
-    "Ray2333/GRM-Llama3-8B-rewardmodel-ft",
-    "nicolinho/QRM-Llama3.1-8B",
-    "nicolinho/QRM-Llama3-8B",
-    "general-preference/GPM-Llama-3.1-8B",
-    "general-preference/GPM-Gemma-2B"
-]
-
-# From Open LLM Leaderboard
-def model_hyperlink(link, model_name):
-    # if model_name is above 50 characters, return first 47 characters and "..."
-    if len(model_name) > 50:
-        model_name = model_name[:47] + "..."
-    if model_name == "random":
-        output = "random"
-    elif model_name == "Cohere March 2024":
-        output = f'<a target="_blank" href="https://huggingface.co/Cohere" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-    elif "openai" == model_name.split("/")[0]:
-        output = f'<a target="_blank" href="https://huggingface.co/openai" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-    elif "Anthropic" == model_name.split("/")[0]:
-        output = f'<a target="_blank" href="https://huggingface.co/Anthropic" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-    elif "google" == model_name.split("/")[0]:
-        output = f'<a target="_blank" href="https://huggingface.co/google" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-    elif "PoLL" == model_name.split("/")[0]:
-        output = model_name
-    output = f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-    if model_name in UNVERIFIED_MODELS:
-        output += " *"
-    if model_name in CONTAMINATED_MODELS:
-        output += " ⚠️"
-    return output
-
-def undo_hyperlink(html_string):
-    # Regex pattern to match content inside > and <
-    pattern = r'>[^<]+<'
-    match = re.search(pattern, html_string)
-    if match:
-        # Extract the matched text and remove leading '>' and trailing '<'
-        return match.group(0)[1:-1]
-    else:
-        return "No text found"
-
-
-# Define a function to fetch and process data
-def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to pull the git repo
-    dir = Path(data_repo)
-    data_dir = dir / subdir
-    orgs = [d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d))]
-    # get all files within the sub folders orgs
-    models_results = []
-    for org in orgs:
-        org_dir = data_dir / org
-        files = [f for f in os.listdir(org_dir) if os.path.isfile(os.path.join(org_dir, f))]
-        for file in files:
-            if file.endswith(".json"):
-                models_results.append(org + "/" + file)
-
-    # create empty dataframe to add all data to
-    df = pd.DataFrame()
-
-    # load all json data in the list models_results one by one to avoid not having the same entries
-    for model in models_results:
-        model_data = load_dataset("json", data_files=data_repo + subdir+ "/" + model, split="train")
-        df2 = pd.DataFrame(model_data)
-        # add to df
-        df = pd.concat([df2, df])
-
-
-    # remove chat_template comlumn
-    df = df.drop(columns=["chat_template"])
-
-    # sort columns alphabetically
-    df = df.reindex(sorted(df.columns), axis=1)
-    
-    # move column "model" to the front
-    cols = list(df.columns)
-    cols.insert(0, cols.pop(cols.index('model')))
-    df = df.loc[:, cols]
-
-    # select all columns except "model"
-    cols = df.columns.tolist()
-    cols.remove("model")
-    # if model_type is a column (pref tests may not have it)
-    if "model_type" in cols:
-        cols.remove("model_type")
-    # remove ref_model if in columns
-    if "ref_model" in cols:
-        cols.remove("ref_model")
-    # remove model_beaker from dataframe
-    if "model_beaker" in cols:
-        cols.remove("model_beaker")
-        df = df.drop(columns=["model_beaker"])
-    
-    # remove column xstest (outdated data)
-    # if xstest is a column
-    if "xstest" in cols:
-        df = df.drop(columns=["xstest"])
-        cols.remove("xstest")
-
-    if "ref_model" in df.columns:
-        df = df.drop(columns=["ref_model"])
-
-    # remove column anthropic and summarize_prompted (outdated data)
-    if "anthropic" in cols:
-        df = df.drop(columns=["anthropic"])
-        cols.remove("anthropic")
-    if "summarize_prompted" in cols:
-        df = df.drop(columns=["summarize_prompted"])
-        cols.remove("summarize_prompted")
-    # remove pku_better and pku_safer (removed from the leaderboard)
-    if "pku_better" in cols:
-        df = df.drop(columns=["pku_better"])
-        cols.remove("pku_better")
-    if "pku_safer" in cols:
-        df = df.drop(columns=["pku_safer"])
-        cols.remove("pku_safer")
-
-    # convert to score 
-    df[cols] = (df[cols]*100)
-    avg = np.nanmean(df[cols].values,axis=1)
-    # add average column
-    df["average"] = avg
-    
-    # apply model_hyperlink function to column "model"
-    df["model"] = df["model"].apply(lambda x: model_hyperlink(f"https://huggingface.co/{x}", x))
-
-    # move average column to the second
-    cols = list(df.columns)
-    cols.insert(1, cols.pop(cols.index('average')))
-    df = df.loc[:, cols]
-
-    # move model_type column to first
-    if "model_type" in cols:
-        cols = list(df.columns)
-        cols.insert(1, cols.pop(cols.index('model_type')))
-        df = df.loc[:, cols]
-
-    # remove models with DPO Ref. Free as type (future work)
-    df = df[~df["model_type"].str.contains("DPO Ref. Free", na=False)]
-
-    return df