changed website

.gitignore

@@ -1,3 +1,4 @@
 evals/
+results/
 __pycache__/*
 *.pyc

app.py

@@ -3,288 +3,87 @@ import os
 from huggingface_hub import HfApi, snapshot_download
 from apscheduler.schedulers.background import BackgroundScheduler
 from datasets import load_dataset
-from src.utils import load_all_data
+from src.utils import load_all_data, prep_df, sort_by_category
 from src.md import ABOUT_TEXT, TOP_TEXT
-from src.plt import plot_avg_correlation
-from src.constants import subset_mapping, length_categories, example_counts
 from src.css import custom_css
 import numpy as np
 
 api = HfApi()
 
 COLLAB_TOKEN = os.environ.get("COLLAB_TOKEN")
-evals_repo = "allenai/reward-bench-results"
+evals_repo = "alrope/href_results"
 
-eval_set_repo = "allenai/reward-bench"
-repo_dir_rewardbench = "./evals/rewardbench/"
+eval_set_repo = "alrope/test_dev"
+local_result_dir = "./results/"
 
 def restart_space():
-    api.restart_space(repo_id="allenai/reward-bench", token=COLLAB_TOKEN)
+    api.restart_space(repo_id="alrope/href", token=COLLAB_TOKEN)
 
 print("Pulling evaluation results")
 repo = snapshot_download(
-    local_dir=repo_dir_rewardbench,
-    ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
+    local_dir=local_result_dir,
+    ignore_patterns=[],
     repo_id=evals_repo,
     use_auth_token=COLLAB_TOKEN,
     tqdm_class=None, 
     etag_timeout=30,
     repo_type="dataset",
 )
-
-
-def avg_over_rewardbench(dataframe_core, dataframe_prefs):
-    """
-    Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
-
-    We average over 4 core sections (per prompt weighting):
-    1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
-    2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
-    3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
-    4. Reasoning: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
-    5. Prior Sets (0.5 weight): Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
-    """
-    new_df = dataframe_core.copy()
-    dataframe_prefs = dataframe_prefs.copy()
-
-    # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
-    for subset, sub_subsets in subset_mapping.items():
-        subset_cols = [col for col in new_df.columns if col in sub_subsets]
-        sub_data = new_df[subset_cols].values # take the relevant column values
-        sub_counts = [example_counts[s] for s in subset_cols] # take the example counts
-        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
-        # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
-
-    data_cols = list(subset_mapping.keys())
-    keep_columns = ["model",] + ["model_type"] + data_cols
-    # keep_columns = ["model", "average"] + subsets
-    new_df = new_df[keep_columns]
-
-    # selected average from pref_sets
-    pref_columns = ["anthropic_helpful", "anthropic_hhh", "shp", "summarize"]
-    pref_data = dataframe_prefs[pref_columns].values
-
-    # add column test sets knowing the rows are not identical, take superset
-    dataframe_prefs["Prior Sets (0.5 weight)"] = np.nanmean(pref_data, axis=1)
-
-    # add column Test Sets empty to new_df
-    new_df["Prior Sets (0.5 weight)"] = np.nan
-    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets (0.5 weight)"]
-    values = []
-    for i, row in new_df.iterrows():
-        model = row["model"]
-        if model in dataframe_prefs["model"].values:
-            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0])
-            # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
-        else:
-            values.append(np.nan)
-    
-    new_df["Prior Sets (0.5 weight)"] = values
-
-    # add total average
-    data_cols += ["Prior Sets (0.5 weight)"]
-    final_data = new_df[data_cols].values
-    masked_data = np.ma.masked_array(final_data, np.isnan(final_data))
-    weights = [2, 2, 2, 2, 1]
-    average = np.ma.average(masked_data, axis=1, weights=weights)
-    new_df["average"] = average.filled(np.nan)
-    # new_df["average"] = np.nanmean(new_df[data_cols].values, axis=1)
-
-    # make average third column
-    keep_columns = ["model", "model_type", "average"] + data_cols
-    new_df = new_df[keep_columns]
-    return new_df
-
-def expand_subsets(dataframe):
-    # TODO need to modify data/ script to do this
-    pass
-
-
-def length_bias_check(dataframe):
-    """
-    Takes the raw rewardbench dataframe and splits the data into new buckets according to length_categories.
-    Then, take the average of the three buckets as "average"
-    """
-    new_df = dataframe.copy()
-    existing_subsets = new_df.columns[3:] # model, model_type, average
-    final_subsets = ["Length Bias", "Neutral", "Terse Bias"]
-    # new data is empty list dict for each final subset
-    new_data = {s: [] for s in final_subsets}
-
-    # now, subsets correspond to those with True, Nuetral, and False length bias
-    # check if length_categories[subset] == "True" or "False" or "Neutral"
-    for subset in existing_subsets:
-        subset_data = new_df[subset].values
-        subset_length = length_categories[subset]
-        # route to the correct bucket
-        if subset_length == "True":
-            new_data["Length Bias"].append(subset_data)
-        elif subset_length == "Neutral":
-            new_data["Neutral"].append(subset_data)
-        elif subset_length == "False":
-            new_data["Terse Bias"].append(subset_data)
-
-    # take average of new_data and add to new_df (removing other columns than model)
-    for subset in final_subsets:
-        new_df[subset] = np.nanmean(new_data[subset], axis=0)
-    keep_columns = ["model"] + final_subsets
-    new_df = new_df[keep_columns]
-    # recompute average
-    # new_df["average"] = np.round(np.nanmean(new_df[final_subsets].values, axis=1), 2)
-
-    return new_df
-
-
     
-rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False)
-rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False)
-prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False)
-# prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
-
-rewardbench_data_avg = avg_over_rewardbench(rewardbench_data, prefs_data).sort_values(by='average', ascending=False)
-
-def prep_df(df):
-    # add column to 0th entry with count (column name itself empty)
-    df.insert(0, '', range(1, 1 + len(df)))
-
-    # replace "model" with "Model" and "model_type" with "Model Type" and "average" with "Average"
-    df = df.rename(columns={"model": "Model", "model_type": "Model Type", "average": "Average"})
+href_data_greedy = prep_df(load_all_data(local_result_dir, subdir="temperature=0.0"))
+href_data_nongreedy = prep_df(load_all_data(local_result_dir, subdir="temperature=1.0"))
 
-    # if "Model Type" in columns
-    if "Model Type" in df.columns:
-        # get model_types that have generative in them
-        mask = df["Model Type"].str.contains("generative", case=False, na=False)
-        
-        # set these values to "Generative"
-        df.loc[mask, "Model Type"] = "Generative"
 
-    return df
-
-# add count column to all dataframes
-rewardbench_data = prep_df(rewardbench_data)
-rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
-# adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others
-
-rewardbench_data_length = prep_df(rewardbench_data_length)
-prefs_data = prep_df(prefs_data)
-
-col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
-col_types_rewardbench_avg = ["number"] + ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
-cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
-col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
-# col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1)
+col_types_href = ["number"] + ["markdown"] + ["number"] * int((len(href_data_greedy.columns) - 1) / 2)
+col_types_href_hidden = ["number"] + ["markdown"] + ["number"] * (len(href_data_greedy.columns) - 1)
+categories = ['Brainstorm', 'Open QA', 'Closed QA', 'Extract', 'Generation', 'Rewrite', 'Summarize', 'Classify', "Reasoning Over Numerical Data", "Multi-Document Synthesis", "Fact Checking or Attributed QA"]
+# categories = ['Average', 'Brainstorm', 'Open QA', 'Closed QA', 'Extract', 'Generation', 'Rewrite', 'Summarize', 'Classify']
 
 # for showing random samples
-eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="filtered")
-def random_sample(r: gr.Request, subset):
-    if subset is None or subset == []:
+eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="dev")
+def random_sample(r: gr.Request, category):
+    if category is None or category == []:
         sample_index = np.random.randint(0, len(eval_set) - 1)
         sample = eval_set[sample_index]
-    else: # filter by subsets (can be list)
-        if isinstance(subset, str):
-            subset = [subset]
-        # filter down dataset to only include the subset(s)
-        eval_set_filtered = eval_set.filter(lambda x: x["subset"] in subset)
+    else: # filter by category (can be list)
+        if isinstance(category, str):
+            category = [category]
+        # filter down dataset to only include the category(s)
+        eval_set_filtered = eval_set.filter(lambda x: x["category"] in category)
         sample_index = np.random.randint(0, len(eval_set_filtered) - 1)
         sample = eval_set_filtered[sample_index]
 
     markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
     return markdown_text
 
-subsets = eval_set.unique("subset")
+subsets = eval_set.unique("category")
 
-color_map = {
-    "Generative": "#7497db",
-    "Custom Classifier": "#E8ECF2",
-    "Seq. Classifier": "#ffcd75",
-    "DPO": "#75809c",
-}
-def color_model_type_column(df, color_map):
-    """
-    Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.
 
-    Parameters:
-    df (pd.DataFrame): The DataFrame containing the 'Model Type' column.
-    color_map (dict): A dictionary mapping model types to colors.
-
-    Returns:
-    pd.Styler: The styled DataFrame.
-    """
-    # Function to apply color based on the model type
-    def apply_color(val):
-        color = color_map.get(val, "default")  # Default color if not specified in color_map
-        return f'background-color: {color}'
-    
-    # Format for different columns
-    format_dict = {col: "{:.1f}" for col in df.columns if col not in ['Average', 'Model', 'Model Type']}
-    format_dict['Average'] = "{:.2f}"
-    format_dict[''] = "{:d}"
-
-    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')
-
-def regex_table(dataframe, regex, filter_button, style=True):
+def regex_table(dataframe, regex, selected_category, style=True):
     """
     Takes a model name as a regex, then returns only the rows that has that in it.
     """
+    dataframe = sort_by_category(dataframe, selected_category)
+
     # Split regex statement by comma and trim whitespace around regexes
     regex_list = [x.strip() for x in regex.split(",")]
     # Join the list into a single regex pattern with '|' acting as OR
     combined_regex = '|'.join(regex_list)
 
-    # remove internal ai2 data
-    dataframe = dataframe[~dataframe["Model"].str.contains("ai2", case=False, na=False)]
-    
-    # if filter_button, remove all rows with "ai2" in the model name
-    update_scores = False
-    if isinstance(filter_button, list) or isinstance(filter_button, str):
-        if "Prior Sets" not in filter_button and 'Prior Sets (0.5 weight)' in dataframe.columns:
-            update_scores = True
-            # remove the column "Prior Sets (0.5 weight)" from the outputted table
-            dataframe = dataframe.drop(columns=['Prior Sets (0.5 weight)'])
-        if "Seq. Classifiers" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("Seq. Classifier", case=False, na=False)]
-        if "DPO" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("DPO", case=False, na=False)]
-        if "Custom Classifiers" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("Custom Classifier", case=False, na=False)]
-        if "Generative" not in filter_button:
-            dataframe = dataframe[~dataframe["Model Type"].str.contains("generative", case=False, na=False)]
     # Filter the dataframe such that 'model' contains any of the regex patterns
     data = dataframe[dataframe["Model"].str.contains(combined_regex, case=False, na=False)]
 
-    # if update the score to not use prior sets, do so
-    if update_scores:
-        data["Score"] = (data["Chat"] + data["Chat Hard"] + data["Safety"] + data["Reasoning"]) / 4
-        # if "Prior Sets (0.5 weight)" in data.columns:
-        # data["Prior Sets (0.5 weight)"] = np.nan
-        # sort array by Score column
-        data = data.sort_values(by='Score', ascending=False)
-
     data.reset_index(drop=True, inplace=True)
 
-    # replace column '' with count/rank
-    data[''] = np.arange(1, 1 + len(data))
-
-    # if Score exists, round to 2 decimals
-    if "Score" in data.columns:
-        data["Score"] = np.round(np.array(data["Score"].values).astype(float), 2)
-    if "Average" in data.columns:
-        data["Average"] = np.round(np.array(data["Average"].values).astype(float), 1)
-    # round all others to 1 decimal
-    for col in data.columns:
-        if col not in ["", "Model", "Model Type", "Score", "Average"]:
-            # replace any data[col].values == '' with np.nan
-            data[col] = data[col].replace('', np.nan)
-            data[col] = np.round(np.array(data[col].values).astype(float), 1)
     if style:
-        # apply color
-        data = color_model_type_column(data, color_map)
-
+         # Format for different columns
+        format_dict = {col: "{:.1f}" for col in data.columns if col not in ['Average', 'Model', 'Rank']}
+        format_dict['Average'] = "{:.2f}"
+        data = data.style.format(format_dict, na_rep='').set_properties(**{'text-align': 'right'})
     return data
 
-# import ipdb; ipdb.set_trace()
 
-total_models = len(regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False).values)
+total_models = len(regex_table(href_data_greedy.copy(), "", "Average", style=False).values)
 
 with gr.Blocks(css=custom_css) as app:
     # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About"
@@ -299,104 +98,50 @@ with gr.Blocks(css=custom_css) as app:
                         ![](file/src/logo.png)
                         """)
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏆 RewardBench Leaderboard"):
+        with gr.TabItem("🏆 HREF Leaderboard"):
             with gr.Row():
                 search_1 = gr.Textbox(label="Model Search (delimit with , )", 
-                                      placeholder="Model Search (delimit with , )",
-                                      show_label=False)
-                model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative", "Prior Sets"], 
-                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include.",
-                                                 )
+                                    #   placeholder="Model Search (delimit with , )",
+                                      show_label=True)
+                category_selector_1 = gr.Dropdown(categories, label="Sorted By", value="Average", multiselect=False, show_label=True)
             with gr.Row():
                 # reference data
                 rewardbench_table_hidden = gr.Dataframe(
-                    rewardbench_data_avg.values,
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
+                    href_data_greedy.values,
+                    datatype=col_types_href_hidden,
+                    headers=href_data_greedy.columns.tolist(),
                     visible=False,
                 )
                 rewardbench_table = gr.Dataframe(
-                    regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers",  "Generative"]),
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
-                    elem_id="rewardbench_dataframe_avg",
-                    height=1000,
-                )
-                
-        with gr.TabItem("🔍 RewardBench - Detailed"):
-            with gr.Row():
-                search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-                model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include."
-                                                 )
-            with gr.Row():
-                # ref data
-                rewardbench_table_detailed_hidden = gr.Dataframe(
-                    rewardbench_data.values,
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    visible=False,
-                )
-                rewardbench_table_detailed = gr.Dataframe(
-                    regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    elem_id="rewardbench_dataframe",
-                    height=1000,
+                    regex_table(href_data_greedy.copy(), "", "Average"),
+                    datatype=col_types_href,
+                    headers=href_data_greedy.columns.tolist(),
+                    elem_id="href_data_greedy",
+                    interactive=False,
+                    max_height=1000,
                 )
-        # with gr.TabItem("rewardbench Eval Set - Length Bias"):
-        #     with gr.Row():
-        #         # backup
-        #         rewardbench_table_len_hidden = gr.Dataframe(
-        #             rewardbench_data_length.values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             visible=False,
-        #         )
-        #         rewardbench_table_len = gr.Dataframe(
-        #             regex_table(rewardbench_data_length.copy(), "", False).values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             elem_id="rewardbench_dataframe_length",
-        #             height=1000,
-        #         )
-        with gr.TabItem("Prior Test Sets"):
+        with gr.TabItem("Non-Greedy"):
             with gr.Row():
-                search_3 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-                model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
-                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers"], 
-                                                 label="Model Types",
-                                                 show_label=False, 
-                                                #  info="Which model types to include.",
-                                                 )
+                search_2 = gr.Textbox(label="Model Search (delimit with , )", 
+                                    #   placeholder="Model Search (delimit with , )",
+                                      show_label=True)
+                category_selector_2 = gr.Dropdown(categories, label="Sorted By", value="Average", multiselect=False, show_label=True)
             with gr.Row():
-                PREF_SET_TEXT = """
-                For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). Only the subsets Anthropic Helpful, Anthropic HHH, Stanford SHP, and OpenAI's Summarize data are used in the leaderboard ranking.
-                """
-                gr.Markdown(PREF_SET_TEXT)
-            with gr.Row():
-                # backup
-                pref_sets_table_hidden = gr.Dataframe(
-                    prefs_data.values,
-                    datatype=col_types_prefs,
-                    headers=prefs_data.columns.tolist(),
+                # reference data
+                rewardbench_table_hidden_nongreedy = gr.Dataframe(
+                    href_data_nongreedy.values,
+                    datatype=col_types_href_hidden,
+                    headers=href_data_nongreedy.columns.tolist(),
                     visible=False,
                 )
-                pref_sets_table = gr.Dataframe(
-                    regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]),
-                    datatype=col_types_prefs,
-                    headers=prefs_data.columns.tolist(),
-                    elem_id="prefs_dataframe",
-                    height=1000,
+                rewardbench_table_nongreedy = gr.Dataframe(
+                    regex_table(href_data_nongreedy.copy(), "", "Average"),
+                    datatype=col_types_href,
+                    headers=href_data_nongreedy.columns.tolist(),
+                    elem_id="href_data_nongreedy",
+                    interactive=False,
+                    max_height=1000,
                 )
-
-
         with gr.TabItem("About"):
             with gr.Row():
                 gr.Markdown(ABOUT_TEXT)
@@ -406,27 +151,18 @@ with gr.Blocks(css=custom_css) as app:
                 # loads one sample
                 gr.Markdown("""## Random Dataset Sample Viewer
 Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
-                subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
+                subset_selector = gr.Dropdown(subsets, label="Category", value=None, multiselect=True)
                 button = gr.Button("Show Random Sample")
 
             with gr.Row():
                 sample_display = gr.Markdown("{sampled data loads here}")
 
             button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
-        # removed plot because not pretty enough
-        # with gr.TabItem("Model Correlation"):
-        #     with gr.Row():
-        #         plot = plot_avg_correlation(rewardbench_data_avg, prefs_data)
-        #         gr.Plot(plot)
 
-    search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len)
-    search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)    
-    
-    model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)
+    search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, category_selector_1], outputs=rewardbench_table)  
+    category_selector_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, category_selector_1], outputs=rewardbench_table)
+    search_2.change(regex_table, inputs=[rewardbench_table_hidden_nongreedy, search_2, category_selector_2], outputs=rewardbench_table_nongreedy)  
+    category_selector_2.change(regex_table, inputs=[rewardbench_table_hidden_nongreedy, search_2, category_selector_2], outputs=rewardbench_table_nongreedy)
 
     with gr.Row():
         with gr.Accordion("📚 Citation", open=False):
@@ -442,16 +178,7 @@ Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
                 elem_id="citation-button",
                 show_copy_button=True,
             )
-# Load data when app starts, TODO make this used somewhere...
-# def load_data_on_start():
-#     data_rewardbench = load_all_data(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench)
-
-#     data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench_avg)
 
-#     data_prefs = load_all_data(repo_dir_prefs)
-#     pref_sets_table.update(data_prefs)
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h

app_old.py

@@ -0,0 +1,464 @@
+import gradio as gr
+import os
+from huggingface_hub import HfApi, snapshot_download
+from apscheduler.schedulers.background import BackgroundScheduler
+from datasets import load_dataset
+from src.utils_old import load_all_data
+from src.md import ABOUT_TEXT, TOP_TEXT
+from src.constants import subset_mapping, length_categories, example_counts
+from src.css import custom_css
+import numpy as np
+
+api = HfApi()
+
+COLLAB_TOKEN = os.environ.get("COLLAB_TOKEN")
+evals_repo = "allenai/reward-bench-results"
+
+eval_set_repo = "allenai/reward-bench"
+repo_dir_rewardbench = "./evals/rewardbench/"
+
+def restart_space():
+    api.restart_space(repo_id="allenai/reward-bench", token=COLLAB_TOKEN)
+
+print("Pulling evaluation results")
+repo = snapshot_download(
+    local_dir=repo_dir_rewardbench,
+    ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
+    repo_id=evals_repo,
+    use_auth_token=COLLAB_TOKEN,
+    tqdm_class=None, 
+    etag_timeout=30,
+    repo_type="dataset",
+)
+
+
+def avg_over_rewardbench(dataframe_core, dataframe_prefs):
+    """
+    Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
+
+    We average over 4 core sections (per prompt weighting):
+    1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
+    2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
+    3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
+    4. Reasoning: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
+    5. Prior Sets (0.5 weight): Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
+    """
+    new_df = dataframe_core.copy()
+    dataframe_prefs = dataframe_prefs.copy()
+
+    # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
+    for subset, sub_subsets in subset_mapping.items():
+        subset_cols = [col for col in new_df.columns if col in sub_subsets]
+        sub_data = new_df[subset_cols].values # take the relevant column values
+        sub_counts = [example_counts[s] for s in subset_cols] # take the example counts
+        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
+        # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
+
+    data_cols = list(subset_mapping.keys())
+    keep_columns = ["model",] + ["model_type"] + data_cols
+    # keep_columns = ["model", "average"] + subsets
+    new_df = new_df[keep_columns]
+
+    # selected average from pref_sets
+    pref_columns = ["anthropic_helpful", "anthropic_hhh", "shp", "summarize"]
+    pref_data = dataframe_prefs[pref_columns].values
+
+    # add column test sets knowing the rows are not identical, take superset
+    dataframe_prefs["Prior Sets (0.5 weight)"] = np.nanmean(pref_data, axis=1)
+
+    # add column Test Sets empty to new_df
+    new_df["Prior Sets (0.5 weight)"] = np.nan
+    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets (0.5 weight)"]
+    values = []
+    for i, row in new_df.iterrows():
+        model = row["model"]
+        if model in dataframe_prefs["model"].values:
+            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0])
+            # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
+        else:
+            values.append(np.nan)
+    
+    new_df["Prior Sets (0.5 weight)"] = values
+
+    # add total average
+    data_cols += ["Prior Sets (0.5 weight)"]
+    final_data = new_df[data_cols].values
+    masked_data = np.ma.masked_array(final_data, np.isnan(final_data))
+    weights = [2, 2, 2, 2, 1]
+    average = np.ma.average(masked_data, axis=1, weights=weights)
+    new_df["average"] = average.filled(np.nan)
+    # new_df["average"] = np.nanmean(new_df[data_cols].values, axis=1)
+
+    # make average third column
+    keep_columns = ["model", "model_type", "average"] + data_cols
+    new_df = new_df[keep_columns]
+    return new_df
+
+def expand_subsets(dataframe):
+    # TODO need to modify data/ script to do this
+    pass
+
+
+def length_bias_check(dataframe):
+    """
+    Takes the raw rewardbench dataframe and splits the data into new buckets according to length_categories.
+    Then, take the average of the three buckets as "average"
+    """
+    new_df = dataframe.copy()
+    existing_subsets = new_df.columns[3:] # model, model_type, average
+    final_subsets = ["Length Bias", "Neutral", "Terse Bias"]
+    # new data is empty list dict for each final subset
+    new_data = {s: [] for s in final_subsets}
+
+    # now, subsets correspond to those with True, Nuetral, and False length bias
+    # check if length_categories[subset] == "True" or "False" or "Neutral"
+    for subset in existing_subsets:
+        subset_data = new_df[subset].values
+        subset_length = length_categories[subset]
+        # route to the correct bucket
+        if subset_length == "True":
+            new_data["Length Bias"].append(subset_data)
+        elif subset_length == "Neutral":
+            new_data["Neutral"].append(subset_data)
+        elif subset_length == "False":
+            new_data["Terse Bias"].append(subset_data)
+
+    # take average of new_data and add to new_df (removing other columns than model)
+    for subset in final_subsets:
+        new_df[subset] = np.nanmean(new_data[subset], axis=0)
+    keep_columns = ["model"] + final_subsets
+    new_df = new_df[keep_columns]
+    # recompute average
+    # new_df["average"] = np.round(np.nanmean(new_df[final_subsets].values, axis=1), 2)
+
+    return new_df
+
+
+    
+rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False)
+rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False)
+prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False)
+# prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
+
+rewardbench_data_avg = avg_over_rewardbench(rewardbench_data, prefs_data).sort_values(by='average', ascending=False)
+
+def prep_df(df):
+    # add column to 0th entry with count (column name itself empty)
+    df.insert(0, '', range(1, 1 + len(df)))
+
+    # replace "model" with "Model" and "model_type" with "Model Type" and "average" with "Average"
+    df = df.rename(columns={"model": "Model", "model_type": "Model Type", "average": "Average"})
+
+    # if "Model Type" in columns
+    if "Model Type" in df.columns:
+        # get model_types that have generative in them
+        mask = df["Model Type"].str.contains("generative", case=False, na=False)
+        
+        # set these values to "Generative"
+        df.loc[mask, "Model Type"] = "Generative"
+
+    return df
+
+# add count column to all dataframes
+rewardbench_data = prep_df(rewardbench_data)
+rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
+# adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others
+
+rewardbench_data_length = prep_df(rewardbench_data_length)
+prefs_data = prep_df(prefs_data)
+
+col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
+col_types_rewardbench_avg = ["number"] + ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
+cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
+col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
+# col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1)
+
+# for showing random samples
+eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="filtered")
+def random_sample(r: gr.Request, subset):
+    if subset is None or subset == []:
+        sample_index = np.random.randint(0, len(eval_set) - 1)
+        sample = eval_set[sample_index]
+    else: # filter by subsets (can be list)
+        if isinstance(subset, str):
+            subset = [subset]
+        # filter down dataset to only include the subset(s)
+        eval_set_filtered = eval_set.filter(lambda x: x["subset"] in subset)
+        sample_index = np.random.randint(0, len(eval_set_filtered) - 1)
+        sample = eval_set_filtered[sample_index]
+
+    markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
+    return markdown_text
+
+subsets = eval_set.unique("subset")
+
+color_map = {
+    "Generative": "#7497db",
+    "Custom Classifier": "#E8ECF2",
+    "Seq. Classifier": "#ffcd75",
+    "DPO": "#75809c",
+}
+def color_model_type_column(df, color_map):
+    """
+    Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.
+
+    Parameters:
+    df (pd.DataFrame): The DataFrame containing the 'Model Type' column.
+    color_map (dict): A dictionary mapping model types to colors.
+
+    Returns:
+    pd.Styler: The styled DataFrame.
+    """
+    # Function to apply color based on the model type
+    def apply_color(val):
+        color = color_map.get(val, "default")  # Default color if not specified in color_map
+        return f'background-color: {color}'
+    
+    # Format for different columns
+    format_dict = {col: "{:.1f}" for col in df.columns if col not in ['Average', 'Model', 'Model Type']}
+    format_dict['Average'] = "{:.2f}"
+    format_dict[''] = "{:d}"
+
+    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')
+
+def regex_table(dataframe, regex, filter_button, style=True):
+    """
+    Takes a model name as a regex, then returns only the rows that has that in it.
+    """
+    # Split regex statement by comma and trim whitespace around regexes
+    regex_list = [x.strip() for x in regex.split(",")]
+    # Join the list into a single regex pattern with '|' acting as OR
+    combined_regex = '|'.join(regex_list)
+
+    # remove internal ai2 data
+    dataframe = dataframe[~dataframe["Model"].str.contains("ai2", case=False, na=False)]
+    
+    # if filter_button, remove all rows with "ai2" in the model name
+    update_scores = False
+    if isinstance(filter_button, list) or isinstance(filter_button, str):
+        if "Prior Sets" not in filter_button and 'Prior Sets (0.5 weight)' in dataframe.columns:
+            update_scores = True
+            # remove the column "Prior Sets (0.5 weight)" from the outputted table
+            dataframe = dataframe.drop(columns=['Prior Sets (0.5 weight)'])
+        if "Seq. Classifiers" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Seq. Classifier", case=False, na=False)]
+        if "DPO" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("DPO", case=False, na=False)]
+        if "Custom Classifiers" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Custom Classifier", case=False, na=False)]
+        if "Generative" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("generative", case=False, na=False)]
+    # Filter the dataframe such that 'model' contains any of the regex patterns
+    data = dataframe[dataframe["Model"].str.contains(combined_regex, case=False, na=False)]
+
+    # if update the score to not use prior sets, do so
+    if update_scores:
+        data["Score"] = (data["Chat"] + data["Chat Hard"] + data["Safety"] + data["Reasoning"]) / 4
+        # if "Prior Sets (0.5 weight)" in data.columns:
+        # data["Prior Sets (0.5 weight)"] = np.nan
+        # sort array by Score column
+        data = data.sort_values(by='Score', ascending=False)
+
+    data.reset_index(drop=True, inplace=True)
+
+    # replace column '' with count/rank
+    data[''] = np.arange(1, 1 + len(data))
+
+    # if Score exists, round to 2 decimals
+    if "Score" in data.columns:
+        data["Score"] = np.round(np.array(data["Score"].values).astype(float), 2)
+    if "Average" in data.columns:
+        data["Average"] = np.round(np.array(data["Average"].values).astype(float), 1)
+    # round all others to 1 decimal
+    for col in data.columns:
+        if col not in ["", "Model", "Model Type", "Score", "Average"]:
+            # replace any data[col].values == '' with np.nan
+            data[col] = data[col].replace('', np.nan)
+            data[col] = np.round(np.array(data[col].values).astype(float), 1)
+    if style:
+        # apply color
+        data = color_model_type_column(data, color_map)
+
+    return data
+
+
+def printout(df):
+    print(df.iloc[0])
+    print(df.iloc[1])
+
+# import ipdb; ipdb.set_trace()
+
+total_models = len(regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False).values)
+
+
+with gr.Blocks(css=custom_css) as app:
+    # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About"
+    with gr.Row():
+        with gr.Column(scale=6):
+            gr.Markdown(TOP_TEXT.format(str(total_models)))
+        with gr.Column(scale=4):
+            # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
+            # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
+            # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
+            gr.Markdown("""
+                        ![](file/src/logo.png)
+                        """)
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏆 RewardBench Leaderboard"):
+            with gr.Row():
+                search_1 = gr.Textbox(label="Model Search (delimit with , )", 
+                                      placeholder="Model Search (delimit with , )",
+                                      show_label=False)
+                model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative", "Prior Sets"], 
+                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
+                                                 label="Model Types", 
+                                                 show_label=False,
+                                                #  info="Which model types to include.",
+                                                 )
+            with gr.Row():
+                # reference data
+                rewardbench_table_hidden = gr.Dataframe(
+                    rewardbench_data_avg.values,
+                    datatype=col_types_rewardbench_avg,
+                    headers=rewardbench_data_avg.columns.tolist(),
+                    visible=False,
+                )
+                rewardbench_table = gr.Dataframe(
+                    regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers",  "Generative"]),
+                    datatype=col_types_rewardbench_avg,
+                    headers=rewardbench_data_avg.columns.tolist(),
+                    elem_id="rewardbench_dataframe_avg",
+                    max_height=1000,
+                )
+                
+        with gr.TabItem("🔍 RewardBench - Detailed"):
+            with gr.Row():
+                search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
+                model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
+                                                 value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"], 
+                                                 label="Model Types", 
+                                                 show_label=False,
+                                                #  info="Which model types to include."
+                                                 )
+            with gr.Row():
+                # ref data
+                rewardbench_table_detailed_hidden = gr.Dataframe(
+                    rewardbench_data.values,
+                    datatype=col_types_rewardbench,
+                    headers=rewardbench_data.columns.tolist(),
+                    visible=False,
+                )
+                rewardbench_table_detailed = gr.Dataframe(
+                    regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
+                    datatype=col_types_rewardbench,
+                    headers=rewardbench_data.columns.tolist(),
+                    elem_id="rewardbench_dataframe",
+                    max_height=1000,
+                )
+        # with gr.TabItem("rewardbench Eval Set - Length Bias"):
+        #     with gr.Row():
+        #         # backup
+        #         rewardbench_table_len_hidden = gr.Dataframe(
+        #             rewardbench_data_length.values,
+        #             datatype=cols_rewardbench_data_length,
+        #             headers=rewardbench_data_length.columns.tolist(),
+        #             visible=False,
+        #         )
+        #         rewardbench_table_len = gr.Dataframe(
+        #             regex_table(rewardbench_data_length.copy(), "", False).values,
+        #             datatype=cols_rewardbench_data_length,
+        #             headers=rewardbench_data_length.columns.tolist(),
+        #             elem_id="rewardbench_dataframe_length",
+        #             height=1000,
+        #         )
+        with gr.TabItem("Prior Test Sets"):
+            with gr.Row():
+                search_3 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
+                model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
+                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers"], 
+                                                 label="Model Types",
+                                                 show_label=False, 
+                                                #  info="Which model types to include.",
+                                                 )
+            with gr.Row():
+                PREF_SET_TEXT = """
+                For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). Only the subsets Anthropic Helpful, Anthropic HHH, Stanford SHP, and OpenAI's Summarize data are used in the leaderboard ranking.
+                """
+                gr.Markdown(PREF_SET_TEXT)
+            with gr.Row():
+                # backup
+                pref_sets_table_hidden = gr.Dataframe(
+                    prefs_data.values,
+                    datatype=col_types_prefs,
+                    headers=prefs_data.columns.tolist(),
+                    visible=False,
+                )
+                pref_sets_table = gr.Dataframe(
+                    regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]),
+                    datatype=col_types_prefs,
+                    headers=prefs_data.columns.tolist(),
+                    elem_id="prefs_dataframe",
+                    max_height=1000,
+                )
+
+
+        with gr.TabItem("About"):
+            with gr.Row():
+                gr.Markdown(ABOUT_TEXT)
+
+        with gr.TabItem("Dataset Viewer"):
+            with gr.Row():
+                # loads one sample
+                gr.Markdown("""## Random Dataset Sample Viewer
+Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
+                subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
+                button = gr.Button("Show Random Sample")
+
+            with gr.Row():
+                sample_display = gr.Markdown("{sampled data loads here}")
+
+            button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
+        # removed plot because not pretty enough
+        # with gr.TabItem("Model Correlation"):
+        #     with gr.Row():
+        #         plot = plot_avg_correlation(rewardbench_data_avg, prefs_data)
+        #         gr.Plot(plot)
+
+    search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
+    search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
+    # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len)
+    search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)    
+    
+    model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
+    model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
+    model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)
+
+    with gr.Row():
+        with gr.Accordion("📚 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=r"""@misc{RewardBench,
+    title={RewardBench: Evaluating Reward Models for Language Modeling},
+    author={Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, LJ and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
+    year={2024},
+    howpublished={\url{https://huggingface.co/spaces/allenai/reward-bench}
+}""",
+                lines=7,
+                label="Copy the following to cite these results.",
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+# Load data when app starts, TODO make this used somewhere...
+# def load_data_on_start():
+#     data_rewardbench = load_all_data(repo_dir_rewardbench)
+#     rewardbench_table.update(data_rewardbench)
+
+#     data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench)
+#     rewardbench_table.update(data_rewardbench_avg)
+
+#     data_prefs = load_all_data(repo_dir_prefs)
+#     pref_sets_table.update(data_prefs)
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
+scheduler.start()
+app.launch(allowed_paths=['src/']) # had .queue() before launch before... not sure if that's necessary

src/constants.py

@@ -57,4 +57,4 @@ subset_mapping = {
     "Chat Hard": ['llmbar-adver-GPTInst', 'llmbar-adver-GPTOut', 'llmbar-adver-manual', 'llmbar-adver-neighbor', 'llmbar-natural', 'mt-bench-hard'],
     "Safety": ['donotanswer', 'refusals-dangerous', 'refusals-offensive', 'xstest-should-refuse', 'xstest-should-respond'],
     "Reasoning": ["hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust", "math-prm"]
-}
+}

src/md.py

@@ -102,5 +102,4 @@ current_time = datetime.now(pacific_tz).strftime("%H:%M %Z, %d %b %Y")
 TOP_TEXT = f"""# RewardBench: Evaluating Reward Models
 ### Evaluating the capabilities, safety, and pitfalls of reward models
 [Code](https://github.com/allenai/reward-bench) | [Eval. Dataset](https://huggingface.co/datasets/allenai/reward-bench) | [Prior Test Sets](https://huggingface.co/datasets/allenai/pref-test-sets) | [Results](https://huggingface.co/datasets/allenai/reward-bench-results) | [Paper](https://arxiv.org/abs/2403.13787) | Total models: {{}} | * Unverified models | ⚠️ Dataset Contamination | Last restart (PST): {current_time}
-
-⚠️ Many of the top models were trained on unintentionally contaminated, AI-generated data, for more information, see this [gist](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300)."""
+"""

src/plt.py

@@ -1,53 +0,0 @@
-import matplotlib.pyplot as plt
-import pandas as pd
-from .utils import undo_hyperlink
-
-def plot_avg_correlation(df1, df2):
-    """
-    Plots the "average" column for each unique model that appears in both dataframes.
-    
-    Parameters:
-    - df1: pandas DataFrame containing columns "model" and "average".
-    - df2: pandas DataFrame containing columns "model" and "average".
-    """
-    # Identify the unique models that appear in both DataFrames
-    common_models = pd.Series(list(set(df1['model']) & set(df2['model'])))
-    
-    # Set up the plot
-    plt.figure(figsize=(13, 6), constrained_layout=True)
-
-    # axes from 0 to 1 for x and y 
-    plt.xlim(0.475, 0.8)
-    plt.ylim(0.475, 0.8)
-
-    # larger font (16)
-    plt.rcParams.update({'font.size': 12, 'axes.labelsize': 14,'axes.titlesize': 14})
-    # plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.1)
-    # plt.tight_layout()
-    # plt.margins(0,0)
-
-    for model in common_models:
-        # Filter data for the current model
-        df1_model_data = df1[df1['model'] == model]['average'].values
-        df2_model_data = df2[df2['model'] == model]['average'].values
-        
-        # Plotting
-        plt.scatter(df1_model_data, df2_model_data, label=model)
-        m_name = undo_hyperlink(model)
-        if m_name == "No text found":
-            m_name = "Random"
-        # Add text above each point like 
-        # plt.text(x[i] + 0.1, y[i] + 0.1, label, ha='left', va='bottom')
-        plt.text(df1_model_data - .005, df2_model_data, m_name, horizontalalignment='right', verticalalignment='center')
-
-    # add correlation line to scatter plot
-    # first, compute correlation
-    corr = df1['average'].corr(df2['average'])
-    # add correlation line based on corr
-    
-
-
-    plt.xlabel('HERM Eval. Set Avg.', fontsize=16)
-    plt.ylabel('Pref. Test Sets Avg.', fontsize=16)
-    # plt.legend(title='Model', bbox_to_anchor=(1.05, 1), loc='upper left')
-    return plt

src/utils.py

@@ -6,33 +6,11 @@ import os
 import re
 
 UNVERIFIED_MODELS = [
-    "nvidia/Nemotron-4-340B-Reward",
-    "nvidia/Llama3-70B-SteerLM-RM",
-    "Cohere May 2024",
-    "google/gemini-1.5-pro-0514",
-    "google/flame-24b-july-2024",
-    "Cohere March 2024",
-    "facebook/Self-taught-Llama-3-70B",
-    "facebook/Self-taught-evaluator-llama3.1-70B",
-    "google/flame-1.0-24B-july-2024",
-    "Salesforce/SFR-LLaMa-3.1-70B-Judge-r",
-    "Salesforce/SFR-nemo-12B-Judge-r",
-    "Salesforce/SFR-LLaMa-3.1-8B-Judge-r",
-    "SF-Foundation/TextEval-OffsetBias-12B",
-    "SF-Foundation/TextEval-Llama3.1-70B",
-    "nvidia/Llama-3.1-Nemotron-70B-Reward",
+
 ]
 
 CONTAMINATED_MODELS = [
-    "Skywork/Skywork-Reward-Gemma-2-27B",
-    "Skywork/Skywork-Critic-Llama-3.1-70B",
-    "LxzGordon/URM-LLaMa-3.1-8B",
-    "Skywork/Skywork-Reward-Llama-3.1-8B",
-    "Ray2333/GRM-Llama3-8B-rewardmodel-ft",
-    "nicolinho/QRM-Llama3.1-8B",
-    "nicolinho/QRM-Llama3-8B",
-    "general-preference/GPM-Llama-3.1-8B",
-    "general-preference/GPM-Gemma-2B"
+
 ]
 
 # From Open LLM Leaderboard
@@ -75,97 +53,103 @@ def undo_hyperlink(html_string):
 def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to pull the git repo
     dir = Path(data_repo)
     data_dir = dir / subdir
-    orgs = [d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d))]
-    # get all files within the sub folders orgs
-    models_results = []
-    for org in orgs:
-        org_dir = data_dir / org
-        files = [f for f in os.listdir(org_dir) if os.path.isfile(os.path.join(org_dir, f))]
-        for file in files:
-            if file.endswith(".json"):
-                models_results.append(org + "/" + file)
-
+    
+    # get all files
+    models_names = [f.split(".json")[0] for f in os.listdir(data_dir) 
+                          if os.path.isfile(os.path.join(data_dir, f)) and f.endswith(".json")]
     # create empty dataframe to add all data to
     df = pd.DataFrame()
 
     # load all json data in the list models_results one by one to avoid not having the same entries
-    for model in models_results:
-        model_data = load_dataset("json", data_files=data_repo + subdir+ "/" + model, split="train")
+    for model_name in models_names:
+        model_data = load_dataset("json", data_files=os.path.join(data_dir, model_name + ".json"), split="train")
+        model_data = model_data.add_column("model", [model_name])
         df2 = pd.DataFrame(model_data)
         # add to df
         df = pd.concat([df2, df])
+    
+    return df
 
 
-    # remove chat_template comlumn
-    df = df.drop(columns=["chat_template"])
-
+def prep_df(df):
     # sort columns alphabetically
     df = df.reindex(sorted(df.columns), axis=1)
-    
+
     # move column "model" to the front
     cols = list(df.columns)
     cols.insert(0, cols.pop(cols.index('model')))
     df = df.loc[:, cols]
+    
+    # apply model_hyperlink function to column "model"
+    df["model"] = df.apply(lambda row: model_hyperlink(f"https://huggingface.co/{row['path']}", row['model']), axis=1)
+    df = df.drop(columns=["path"])
 
-    # select all columns except "model"
+    # select all columns except "model" and convert to score 
     cols = df.columns.tolist()
     cols.remove("model")
-    # if model_type is a column (pref tests may not have it)
-    if "model_type" in cols:
-        cols.remove("model_type")
-    # remove ref_model if in columns
-    if "ref_model" in cols:
-        cols.remove("ref_model")
-    # remove model_beaker from dataframe
-    if "model_beaker" in cols:
-        cols.remove("model_beaker")
-        df = df.drop(columns=["model_beaker"])
-    
-    # remove column xstest (outdated data)
-    # if xstest is a column
-    if "xstest" in cols:
-        df = df.drop(columns=["xstest"])
-        cols.remove("xstest")
-
-    if "ref_model" in df.columns:
-        df = df.drop(columns=["ref_model"])
-
-    # remove column anthropic and summarize_prompted (outdated data)
-    if "anthropic" in cols:
-        df = df.drop(columns=["anthropic"])
-        cols.remove("anthropic")
-    if "summarize_prompted" in cols:
-        df = df.drop(columns=["summarize_prompted"])
-        cols.remove("summarize_prompted")
-    # remove pku_better and pku_safer (removed from the leaderboard)
-    if "pku_better" in cols:
-        df = df.drop(columns=["pku_better"])
-        cols.remove("pku_better")
-    if "pku_safer" in cols:
-        df = df.drop(columns=["pku_safer"])
-        cols.remove("pku_safer")
-
-    # convert to score 
+    cols = [c for c in cols if "rank" not in c]
     df[cols] = (df[cols]*100)
-    avg = np.nanmean(df[cols].values,axis=1)
-    # add average column
-    df["average"] = avg
-    
-    # apply model_hyperlink function to column "model"
-    df["model"] = df["model"].apply(lambda x: model_hyperlink(f"https://huggingface.co/{x}", x))
 
     # move average column to the second
     cols = list(df.columns)
     cols.insert(1, cols.pop(cols.index('average')))
     df = df.loc[:, cols]
 
-    # move model_type column to first
-    if "model_type" in cols:
-        cols = list(df.columns)
-        cols.insert(1, cols.pop(cols.index('model_type')))
-        df = df.loc[:, cols]
-
-    # remove models with DPO Ref. Free as type (future work)
-    df = df[~df["model_type"].str.contains("DPO Ref. Free", na=False)]
+    df = df.rename(columns={
+        "model": "Model",
+        "average": "Average",
+        "brainstorm": "Brainstorm",
+        "open_qa": "Open QA",
+        "closed_qa": "Closed QA",
+        "extract": "Extract",
+        "generation": "Generation",
+        "rewrite": "Rewrite",
+        "summarize": "Summarize",
+        "classify": "Classify",
+        "reasoning_over_numerical_data": "Reasoning Over Numerical Data",
+        "multi-document_synthesis": "Multi-Document Synthesis",
+        "fact_checking_or_attributed_qa": "Fact Checking or Attributed QA",
+    })
+
+    # Format for different columns
+    # if Score exists, round to 2 decimals
+    # if "Average" in df.columns:
+    #     df["Average"] = np.array([f"{v:.2f}" for v in df["Average"].values])
+         
+    # # round all others to 1 decimal
+    # for col in df.columns:
+    #     if col not in ["Model", "Average"]:
+    #         # replace any df[col].values == '' with np.nan
+    #         df[col] = df[col].replace('', np.nan)
+    #         df[col] = np.array([f"{v:.1f}" for v in df[col].values])
 
     return df
+
+
+def sort_by_category(df, category):
+    new_df = df.copy()
+    col_rank = category.lower().replace(" ", "_") + "_rank"
+
+    # sort
+    new_df = new_df.sort_values(by=[col_rank, category], ascending=[True, False])
+
+    # move column ranking to the front
+    cols = list(new_df.columns)
+    cols.insert(0, cols.pop(cols.index(col_rank)))
+    new_df = new_df.loc[:, cols]
+    new_df = new_df.rename(columns={col_rank: "Rank"})
+
+    # move selected column to the third
+    cols = list(new_df.columns)
+    cols.insert(2, cols.pop(cols.index(category)))
+    new_df = new_df.loc[:, cols]
+
+    # # move selected column to the fourth
+    # cols = list(new_df.columns)
+    # cols.insert(3, cols.pop(cols.index("Average")))
+    # new_df = new_df.loc[:, cols]
+
+    # drop all ranking 
+    new_df = new_df.drop(columns=[c for c in new_df.columns if c.endswith("rank")])
+
+    return new_df

src/utils_old.py

@@ -0,0 +1,171 @@
+import pandas as pd
+from pathlib import Path
+from datasets import load_dataset
+import numpy as np
+import os
+import re
+
+UNVERIFIED_MODELS = [
+    "nvidia/Nemotron-4-340B-Reward",
+    "nvidia/Llama3-70B-SteerLM-RM",
+    "Cohere May 2024",
+    "google/gemini-1.5-pro-0514",
+    "google/flame-24b-july-2024",
+    "Cohere March 2024",
+    "facebook/Self-taught-Llama-3-70B",
+    "facebook/Self-taught-evaluator-llama3.1-70B",
+    "google/flame-1.0-24B-july-2024",
+    "Salesforce/SFR-LLaMa-3.1-70B-Judge-r",
+    "Salesforce/SFR-nemo-12B-Judge-r",
+    "Salesforce/SFR-LLaMa-3.1-8B-Judge-r",
+    "SF-Foundation/TextEval-OffsetBias-12B",
+    "SF-Foundation/TextEval-Llama3.1-70B",
+    "nvidia/Llama-3.1-Nemotron-70B-Reward",
+]
+
+CONTAMINATED_MODELS = [
+    "Skywork/Skywork-Reward-Gemma-2-27B",
+    "Skywork/Skywork-Critic-Llama-3.1-70B",
+    "LxzGordon/URM-LLaMa-3.1-8B",
+    "Skywork/Skywork-Reward-Llama-3.1-8B",
+    "Ray2333/GRM-Llama3-8B-rewardmodel-ft",
+    "nicolinho/QRM-Llama3.1-8B",
+    "nicolinho/QRM-Llama3-8B",
+    "general-preference/GPM-Llama-3.1-8B",
+    "general-preference/GPM-Gemma-2B"
+]
+
+# From Open LLM Leaderboard
+def model_hyperlink(link, model_name):
+    # if model_name is above 50 characters, return first 47 characters and "..."
+    if len(model_name) > 50:
+        model_name = model_name[:47] + "..."
+    if model_name == "random":
+        output = "random"
+    elif model_name == "Cohere March 2024":
+        output = f'<a target="_blank" href="https://huggingface.co/Cohere" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    elif "openai" == model_name.split("/")[0]:
+        output = f'<a target="_blank" href="https://huggingface.co/openai" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    elif "Anthropic" == model_name.split("/")[0]:
+        output = f'<a target="_blank" href="https://huggingface.co/Anthropic" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    elif "google" == model_name.split("/")[0]:
+        output = f'<a target="_blank" href="https://huggingface.co/google" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    elif "PoLL" == model_name.split("/")[0]:
+        output = model_name
+    output = f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+    if model_name in UNVERIFIED_MODELS:
+        output += " *"
+    if model_name in CONTAMINATED_MODELS:
+        output += " ⚠️"
+    return output
+
+def undo_hyperlink(html_string):
+    # Regex pattern to match content inside > and <
+    pattern = r'>[^<]+<'
+    match = re.search(pattern, html_string)
+    if match:
+        # Extract the matched text and remove leading '>' and trailing '<'
+        return match.group(0)[1:-1]
+    else:
+        return "No text found"
+
+
+# Define a function to fetch and process data
+def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to pull the git repo
+    dir = Path(data_repo)
+    data_dir = dir / subdir
+    orgs = [d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d))]
+    # get all files within the sub folders orgs
+    models_results = []
+    for org in orgs:
+        org_dir = data_dir / org
+        files = [f for f in os.listdir(org_dir) if os.path.isfile(os.path.join(org_dir, f))]
+        for file in files:
+            if file.endswith(".json"):
+                models_results.append(org + "/" + file)
+
+    # create empty dataframe to add all data to
+    df = pd.DataFrame()
+
+    # load all json data in the list models_results one by one to avoid not having the same entries
+    for model in models_results:
+        model_data = load_dataset("json", data_files=data_repo + subdir+ "/" + model, split="train")
+        df2 = pd.DataFrame(model_data)
+        # add to df
+        df = pd.concat([df2, df])
+
+
+    # remove chat_template comlumn
+    df = df.drop(columns=["chat_template"])
+
+    # sort columns alphabetically
+    df = df.reindex(sorted(df.columns), axis=1)
+    
+    # move column "model" to the front
+    cols = list(df.columns)
+    cols.insert(0, cols.pop(cols.index('model')))
+    df = df.loc[:, cols]
+
+    # select all columns except "model"
+    cols = df.columns.tolist()
+    cols.remove("model")
+    # if model_type is a column (pref tests may not have it)
+    if "model_type" in cols:
+        cols.remove("model_type")
+    # remove ref_model if in columns
+    if "ref_model" in cols:
+        cols.remove("ref_model")
+    # remove model_beaker from dataframe
+    if "model_beaker" in cols:
+        cols.remove("model_beaker")
+        df = df.drop(columns=["model_beaker"])
+    
+    # remove column xstest (outdated data)
+    # if xstest is a column
+    if "xstest" in cols:
+        df = df.drop(columns=["xstest"])
+        cols.remove("xstest")
+
+    if "ref_model" in df.columns:
+        df = df.drop(columns=["ref_model"])
+
+    # remove column anthropic and summarize_prompted (outdated data)
+    if "anthropic" in cols:
+        df = df.drop(columns=["anthropic"])
+        cols.remove("anthropic")
+    if "summarize_prompted" in cols:
+        df = df.drop(columns=["summarize_prompted"])
+        cols.remove("summarize_prompted")
+    # remove pku_better and pku_safer (removed from the leaderboard)
+    if "pku_better" in cols:
+        df = df.drop(columns=["pku_better"])
+        cols.remove("pku_better")
+    if "pku_safer" in cols:
+        df = df.drop(columns=["pku_safer"])
+        cols.remove("pku_safer")
+
+    # convert to score 
+    df[cols] = (df[cols]*100)
+    avg = np.nanmean(df[cols].values,axis=1)
+    # add average column
+    df["average"] = avg
+    
+    # apply model_hyperlink function to column "model"
+    df["model"] = df["model"].apply(lambda x: model_hyperlink(f"https://huggingface.co/{x}", x))
+
+    # move average column to the second
+    cols = list(df.columns)
+    cols.insert(1, cols.pop(cols.index('average')))
+    df = df.loc[:, cols]
+
+    # move model_type column to first
+    if "model_type" in cols:
+        cols = list(df.columns)
+        cols.insert(1, cols.pop(cols.index('model_type')))
+        df = df.loc[:, cols]
+
+    # remove models with DPO Ref. Free as type (future work)
+    df = df[~df["model_type"].str.contains("DPO Ref. Free", na=False)]
+
+    return df