File size: 22,810 Bytes
507a14d
 
9ceb843
e5d5995
8e499f4
9ceb843
ab74236
0b8c16d
8799e00
bbe05a0
507a14d
 
 
 
 
31bff5a
ab74236
31bff5a
 
9ceb843
e5d5995
31bff5a
507a14d
 
9ceb843
31bff5a
f5220e7
9ceb843
e4cd4cd
9ceb843
 
507a14d
 
 
9ceb843
31bff5a
9ceb843
 
8799e00
 
 
 
 
61c1fca
7eaa6d2
9ceb843
f5220e7
 
8799e00
 
 
 
 
54b0338
7eaa6d2
8799e00
 
f5220e7
 
8799e00
e5d5995
8799e00
f5220e7
9f4ce43
f5220e7
 
 
7eaa6d2
f5220e7
 
7eaa6d2
 
f5220e7
 
 
 
7eaa6d2
 
f5220e7
 
 
7eaa6d2
f5220e7
 
7eaa6d2
 
 
 
 
 
 
f5220e7
 
 
 
e5d5995
9ceb843
 
 
 
56fcfaf
 
 
 
31bff5a
56fcfaf
 
 
b7aaef4
56fcfaf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7eaa6d2
56fcfaf
 
 
 
 
 
 
 
507a14d
31bff5a
 
 
9ceb843
507a14d
31bff5a
f5220e7
f89f357
 
 
 
 
 
6ce351e
 
 
 
 
 
 
 
 
f89f357
 
 
 
7eaa6d2
 
 
f89f357
 
 
 
 
31bff5a
f89f357
9ceb843
507a14d
8e499f4
 
e5d5995
 
 
 
 
 
 
 
 
 
 
 
ab74236
8e499f4
 
e5d5995
 
4a1518a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8799e00
 
 
 
 
 
 
31bff5a
0de05c0
 
 
06fd8bd
0de05c0
31bff5a
0de05c0
 
777fcbc
 
31bff5a
f89f357
31bff5a
f89f357
31bff5a
f89f357
1d33a30
 
8799e00
9f4ce43
 
0de05c0
 
 
777fcbc
93916f2
0de05c0
 
 
4a1518a
 
590ad01
 
 
7eaa6d2
 
874c0c9
7eaa6d2
874c0c9
7eaa6d2
 
 
93916f2
 
874c0c9
4a1518a
 
 
 
9f4ce43
8799e00
874c0c9
 
4a1518a
8799e00
bbe05a0
31bff5a
507a14d
521165c
874c0c9
521165c
31bff5a
 
 
 
 
 
9ceb843
31bff5a
 
f89f357
 
 
0de05c0
777fcbc
31bff5a
f89f357
31bff5a
 
9ceb843
06fd8bd
31bff5a
 
 
 
06fd8bd
9ceb843
31bff5a
777fcbc
31bff5a
 
 
06fd8bd
8799e00
06fd8bd
31bff5a
 
f89f357
0de05c0
fe666e1
31bff5a
f89f357
31bff5a
 
9ceb843
06fd8bd
31bff5a
 
 
 
06fd8bd
9ceb843
31bff5a
4a1518a
31bff5a
 
 
06fd8bd
8799e00
31bff5a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9f4ce43
56fcfaf
f89f357
0de05c0
31bff5a
f89f357
 
31bff5a
 
ab74236
 
9f4ce43
ab74236
 
 
8799e00
 
 
 
 
 
 
06fd8bd
4a1518a
06fd8bd
 
 
 
 
 
9ceb843
 
 
 
8e499f4
 
 
 
6fda62c
 
e5d5995
8e499f4
 
 
 
 
e5d5995
0b8c16d
 
 
31bff5a
0b8c16d
8799e00
31bff5a
 
 
 
 
 
 
 
 
bbe05a0
 
 
149a173
17f167a
bd17252
 
 
 
149a173
bbe05a0
 
 
 
8e499f4
e5d5995
31bff5a
 
e5d5995
31bff5a
 
e5d5995
 
 
9ceb843
e5d5995
 
 
c8a4819
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
import gradio as gr
import os
from huggingface_hub import HfApi, snapshot_download
from apscheduler.schedulers.background import BackgroundScheduler
from datasets import load_dataset
from src.utils import load_all_data
from src.md import ABOUT_TEXT, TOP_TEXT
from src.plt import plot_avg_correlation
from src.constants import subset_mapping, length_categories, example_counts
from src.css import custom_css
import numpy as np

api = HfApi()

COLLAB_TOKEN = os.environ.get("COLLAB_TOKEN")
evals_repo = "allenai/reward-bench-results"

eval_set_repo = "allenai/reward-bench"
repo_dir_rewardbench = "./evals/rewardbench/"

def restart_space():
    api.restart_space(repo_id="allenai/reward-bench", token=COLLAB_TOKEN)

print("Pulling evaluation results")
repo = snapshot_download(
    local_dir=repo_dir_rewardbench,
    ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
    repo_id=evals_repo,
    use_auth_token=COLLAB_TOKEN,
    tqdm_class=None, 
    etag_timeout=30,
    repo_type="dataset",
)


def avg_over_rewardbench(dataframe_core, dataframe_prefs):
    """
    Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.

    We average over 4 core sections (per prompt weighting):
    1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
    2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
    3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
    4. Reasoning: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
    5. Prior Sets (0.5 weight): Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
    """
    new_df = dataframe_core.copy()
    dataframe_prefs = dataframe_prefs.copy()

    # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
    for subset, sub_subsets in subset_mapping.items():
        subset_cols = [col for col in new_df.columns if col in sub_subsets]
        sub_data = new_df[subset_cols].values # take the relevant column values
        sub_counts = [example_counts[s] for s in subset_cols] # take the example counts
        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
        # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)

    data_cols = list(subset_mapping.keys())
    keep_columns = ["model",] + ["model_type"] + data_cols
    # keep_columns = ["model", "average"] + subsets
    new_df = new_df[keep_columns]

    # selected average from pref_sets
    pref_columns = ["anthropic_helpful", "anthropic_hhh", "shp", "summarize"]
    pref_data = dataframe_prefs[pref_columns].values

    # add column test sets knowing the rows are not identical, take superset
    dataframe_prefs["Prior Sets (0.5 weight)"] = np.nanmean(pref_data, axis=1)

    # add column Test Sets empty to new_df
    new_df["Prior Sets (0.5 weight)"] = np.nan
    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets (0.5 weight)"]
    values = []
    for i, row in new_df.iterrows():
        model = row["model"]
        if model in dataframe_prefs["model"].values:
            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0])
            # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
        else:
            values.append(np.nan)
    
    new_df["Prior Sets (0.5 weight)"] = values

    # add total average
    data_cols += ["Prior Sets (0.5 weight)"]
    final_data = new_df[data_cols].values
    masked_data = np.ma.masked_array(final_data, np.isnan(final_data))
    weights = [2, 2, 2, 2, 1]
    average = np.ma.average(masked_data, axis=1, weights=weights)
    new_df["average"] = average.filled(np.nan)
    # new_df["average"] = np.nanmean(new_df[data_cols].values, axis=1)

    # make average third column
    keep_columns = ["model", "model_type", "average"] + data_cols
    new_df = new_df[keep_columns]
    return new_df

def expand_subsets(dataframe):
    # TODO need to modify data/ script to do this
    pass


def length_bias_check(dataframe):
    """
    Takes the raw rewardbench dataframe and splits the data into new buckets according to length_categories.
    Then, take the average of the three buckets as "average"
    """
    new_df = dataframe.copy()
    existing_subsets = new_df.columns[3:] # model, model_type, average
    final_subsets = ["Length Bias", "Neutral", "Terse Bias"]
    # new data is empty list dict for each final subset
    new_data = {s: [] for s in final_subsets}

    # now, subsets correspond to those with True, Nuetral, and False length bias
    # check if length_categories[subset] == "True" or "False" or "Neutral"
    for subset in existing_subsets:
        subset_data = new_df[subset].values
        subset_length = length_categories[subset]
        # route to the correct bucket
        if subset_length == "True":
            new_data["Length Bias"].append(subset_data)
        elif subset_length == "Neutral":
            new_data["Neutral"].append(subset_data)
        elif subset_length == "False":
            new_data["Terse Bias"].append(subset_data)

    # take average of new_data and add to new_df (removing other columns than model)
    for subset in final_subsets:
        new_df[subset] = np.nanmean(new_data[subset], axis=0)
    keep_columns = ["model"] + final_subsets
    new_df = new_df[keep_columns]
    # recompute average
    # new_df["average"] = np.round(np.nanmean(new_df[final_subsets].values, axis=1), 2)

    return new_df


    
rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False)
rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False)
prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False)
# prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)

rewardbench_data_avg = avg_over_rewardbench(rewardbench_data, prefs_data).sort_values(by='average', ascending=False)

def prep_df(df):
    # add column to 0th entry with count (column name itself empty)
    df.insert(0, '', range(1, 1 + len(df)))

    # replace "model" with "Model" and "model_type" with "Model Type" and "average" with "Average"
    df = df.rename(columns={"model": "Model", "model_type": "Model Type", "average": "Average"})

    # if "Model Type" in columns
    if "Model Type" in df.columns:
        # get model_types that have generative in them
        mask = df["Model Type"].str.contains("generative", case=False, na=False)
        
        # set these values to "Generative"
        df.loc[mask, "Model Type"] = "Generative"

    return df

# add count column to all dataframes
rewardbench_data = prep_df(rewardbench_data)
rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
# adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others

rewardbench_data_length = prep_df(rewardbench_data_length)
prefs_data = prep_df(prefs_data)

col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
col_types_rewardbench_avg = ["number"] + ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
# col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1)

# for showing random samples
eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="filtered")
def random_sample(r: gr.Request, subset):
    if subset is None or subset == []:
        sample_index = np.random.randint(0, len(eval_set) - 1)
        sample = eval_set[sample_index]
    else: # filter by subsets (can be list)
        if isinstance(subset, str):
            subset = [subset]
        # filter down dataset to only include the subset(s)
        eval_set_filtered = eval_set.filter(lambda x: x["subset"] in subset)
        sample_index = np.random.randint(0, len(eval_set_filtered) - 1)
        sample = eval_set_filtered[sample_index]

    markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
    return markdown_text

subsets = eval_set.unique("subset")

color_map = {
    "Generative": "#7497db",
    "Custom Classifier": "#E8ECF2",
    "Seq. Classifier": "#ffcd75",
    "DPO": "#75809c",
}
def color_model_type_column(df, color_map):
    """
    Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.

    Parameters:
    df (pd.DataFrame): The DataFrame containing the 'Model Type' column.
    color_map (dict): A dictionary mapping model types to colors.

    Returns:
    pd.Styler: The styled DataFrame.
    """
    # Function to apply color based on the model type
    def apply_color(val):
        color = color_map.get(val, "default")  # Default color if not specified in color_map
        return f'background-color: {color}'
    
    # Format for different columns
    format_dict = {col: "{:.1f}" for col in df.columns if col not in ['Average', 'Model', 'Model Type']}
    format_dict['Average'] = "{:.2f}"
    format_dict[''] = "{:d}"

    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')

def regex_table(dataframe, regex, filter_button, style=True):
    """
    Takes a model name as a regex, then returns only the rows that has that in it.
    """
    # Split regex statement by comma and trim whitespace around regexes
    regex_list = [x.strip() for x in regex.split(",")]
    # Join the list into a single regex pattern with '|' acting as OR
    combined_regex = '|'.join(regex_list)

    # remove internal ai2 data
    dataframe = dataframe[~dataframe["Model"].str.contains("ai2", case=False, na=False)]
    
    # if filter_button, remove all rows with "ai2" in the model name
    update_scores = False
    if isinstance(filter_button, list) or isinstance(filter_button, str):
        if "Prior Sets" not in filter_button and 'Prior Sets (0.5 weight)' in dataframe.columns:
            update_scores = True
            # remove the column "Prior Sets (0.5 weight)" from the outputted table
            dataframe = dataframe.drop(columns=['Prior Sets (0.5 weight)'])
        if "Seq. Classifiers" not in filter_button:
            dataframe = dataframe[~dataframe["Model Type"].str.contains("Seq. Classifier", case=False, na=False)]
        if "DPO" not in filter_button:
            dataframe = dataframe[~dataframe["Model Type"].str.contains("DPO", case=False, na=False)]
        if "Custom Classifiers" not in filter_button:
            dataframe = dataframe[~dataframe["Model Type"].str.contains("Custom Classifier", case=False, na=False)]
        if "Generative" not in filter_button:
            dataframe = dataframe[~dataframe["Model Type"].str.contains("generative", case=False, na=False)]
    # Filter the dataframe such that 'model' contains any of the regex patterns
    data = dataframe[dataframe["Model"].str.contains(combined_regex, case=False, na=False)]

    # if update the score to not use prior sets, do so
    if update_scores:
        data["Score"] = (data["Chat"] + data["Chat Hard"] + data["Safety"] + data["Reasoning"]) / 4
        # if "Prior Sets (0.5 weight)" in data.columns:
        # data["Prior Sets (0.5 weight)"] = np.nan
        # sort array by Score column
        data = data.sort_values(by='Score', ascending=False)

    data.reset_index(drop=True, inplace=True)

    # replace column '' with count/rank
    data[''] = np.arange(1, 1 + len(data))

    # if Score exists, round to 2 decimals
    if "Score" in data.columns:
        data["Score"] = np.round(np.array(data["Score"].values).astype(float), 2)
    if "Average" in data.columns:
        data["Average"] = np.round(np.array(data["Average"].values).astype(float), 1)
    # round all others to 1 decimal
    for col in data.columns:
        if col not in ["", "Model", "Model Type", "Score", "Average"]:
            # replace any data[col].values == '' with np.nan
            data[col] = data[col].replace('', np.nan)
            data[col] = np.round(np.array(data[col].values).astype(float), 1)
    if style:
        # apply color
        data = color_model_type_column(data, color_map)

    return data

# import ipdb; ipdb.set_trace()

total_models = len(regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False).values)

with gr.Blocks(css=custom_css) as app:
    # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About"
    with gr.Row():
        with gr.Column(scale=6):
            gr.Markdown(TOP_TEXT.format(str(total_models)))
        with gr.Column(scale=4):
            # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
            # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
            # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
            gr.Markdown("""
                        ![](file/src/logo.png)
                        """)
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("πŸ† RewardBench Leaderboard"):
            with gr.Row():
                search_1 = gr.Textbox(label="Model Search (delimit with , )", 
                                      placeholder="Model Search (delimit with , )",
                                      show_label=False)
                model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative", "Prior Sets"], 
                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
                                                 label="Model Types", 
                                                 show_label=False,
                                                #  info="Which model types to include.",
                                                 )
            with gr.Row():
                # reference data
                rewardbench_table_hidden = gr.Dataframe(
                    rewardbench_data_avg.values,
                    datatype=col_types_rewardbench_avg,
                    headers=rewardbench_data_avg.columns.tolist(),
                    visible=False,
                )
                rewardbench_table = gr.Dataframe(
                    regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers",  "Generative"]),
                    datatype=col_types_rewardbench_avg,
                    headers=rewardbench_data_avg.columns.tolist(),
                    elem_id="rewardbench_dataframe_avg",
                    height=1000,
                )
                
        with gr.TabItem("πŸ” RewardBench - Detailed"):
            with gr.Row():
                search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
                model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
                                                 value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"], 
                                                 label="Model Types", 
                                                 show_label=False,
                                                #  info="Which model types to include."
                                                 )
            with gr.Row():
                # ref data
                rewardbench_table_detailed_hidden = gr.Dataframe(
                    rewardbench_data.values,
                    datatype=col_types_rewardbench,
                    headers=rewardbench_data.columns.tolist(),
                    visible=False,
                )
                rewardbench_table_detailed = gr.Dataframe(
                    regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
                    datatype=col_types_rewardbench,
                    headers=rewardbench_data.columns.tolist(),
                    elem_id="rewardbench_dataframe",
                    height=1000,
                )
        # with gr.TabItem("rewardbench Eval Set - Length Bias"):
        #     with gr.Row():
        #         # backup
        #         rewardbench_table_len_hidden = gr.Dataframe(
        #             rewardbench_data_length.values,
        #             datatype=cols_rewardbench_data_length,
        #             headers=rewardbench_data_length.columns.tolist(),
        #             visible=False,
        #         )
        #         rewardbench_table_len = gr.Dataframe(
        #             regex_table(rewardbench_data_length.copy(), "", False).values,
        #             datatype=cols_rewardbench_data_length,
        #             headers=rewardbench_data_length.columns.tolist(),
        #             elem_id="rewardbench_dataframe_length",
        #             height=1000,
        #         )
        with gr.TabItem("Prior Test Sets"):
            with gr.Row():
                search_3 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
                model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers"], 
                                                 label="Model Types",
                                                 show_label=False, 
                                                #  info="Which model types to include.",
                                                 )
            with gr.Row():
                PREF_SET_TEXT = """
                For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). Only the subsets Anthropic Helpful, Anthropic HHH, Stanford SHP, and OpenAI's Summarize data are used in the leaderboard ranking.
                """
                gr.Markdown(PREF_SET_TEXT)
            with gr.Row():
                # backup
                pref_sets_table_hidden = gr.Dataframe(
                    prefs_data.values,
                    datatype=col_types_prefs,
                    headers=prefs_data.columns.tolist(),
                    visible=False,
                )
                pref_sets_table = gr.Dataframe(
                    regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]),
                    datatype=col_types_prefs,
                    headers=prefs_data.columns.tolist(),
                    elem_id="prefs_dataframe",
                    height=1000,
                )


        with gr.TabItem("About"):
            with gr.Row():
                gr.Markdown(ABOUT_TEXT)

        with gr.TabItem("Dataset Viewer"):
            with gr.Row():
                # loads one sample
                gr.Markdown("""## Random Dataset Sample Viewer
Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
                subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
                button = gr.Button("Show Random Sample")

            with gr.Row():
                sample_display = gr.Markdown("{sampled data loads here}")

            button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
        # removed plot because not pretty enough
        # with gr.TabItem("Model Correlation"):
        #     with gr.Row():
        #         plot = plot_avg_correlation(rewardbench_data_avg, prefs_data)
        #         gr.Plot(plot)

    search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
    search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
    # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len)
    search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)    
    
    model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
    model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
    model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)

    with gr.Row():
        with gr.Accordion("πŸ“š Citation", open=False):
            citation_button = gr.Textbox(
                value=r"""@misc{RewardBench,
    title={RewardBench: Evaluating Reward Models for Language Modeling},
    author={Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, LJ and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
    year={2024},
    howpublished={\url{https://huggingface.co/spaces/allenai/reward-bench}
}""",
                lines=7,
                label="Copy the following to cite these results.",
                elem_id="citation-button",
                show_copy_button=True,
            )
# Load data when app starts, TODO make this used somewhere...
# def load_data_on_start():
#     data_rewardbench = load_all_data(repo_dir_rewardbench)
#     rewardbench_table.update(data_rewardbench)

#     data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench)
#     rewardbench_table.update(data_rewardbench_avg)

#     data_prefs = load_all_data(repo_dir_prefs)
#     pref_sets_table.update(data_prefs)

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
scheduler.start()
app.launch(allowed_paths=['src/']) # had .queue() before launch before... not sure if that's necessary