File size: 9,918 Bytes
db7f350
3d87820
 
db7f350
 
 
 
3d87820
db7f350
 
 
 
 
 
3d87820
79f4b2b
db7f350
58e4674
db7f350
47c3ae2
 
741edbf
 
974b203
5f9d165
47c3ae2
db7f350
 
741edbf
 
3d87820
 
 
20e370a
 
5f9d165
 
 
974b203
39f3e26
5f9d165
 
 
 
 
0c95215
 
 
 
 
 
5f9d165
 
 
 
db7f350
3d87820
 
20e370a
741edbf
3d87820
 
db7f350
d0c2655
db7f350
5f9d165
db7f350
 
741edbf
db7f350
5f9d165
 
 
 
db7f350
 
 
 
 
 
3d87820
db7f350
 
 
 
974b203
3d87820
741edbf
 
 
3d87820
 
 
 
 
741edbf
3d87820
d0c2655
3d87820
 
 
627001e
741edbf
 
3d87820
 
627001e
 
 
 
 
3d87820
 
627001e
3d87820
 
5822c90
 
 
 
741edbf
 
3d87820
 
 
 
 
741edbf
 
3d87820
 
 
741edbf
 
 
 
3d87820
 
 
 
 
741edbf
3d87820
d0c2655
3d87820
db7f350
 
 
 
5f9d165
 
 
db7f350
741edbf
 
 
 
db7f350
741edbf
 
d0c2655
db7f350
974b203
 
 
 
 
 
 
 
 
 
ee803a2
db7f350
 
 
20e370a
5f9d165
 
741edbf
db7f350
3d87820
 
 
db7f350
 
3d87820
db7f350
 
 
 
 
5d94f6d
 
 
 
 
0c95215
db7f350
741edbf
 
5f9d165
2a643ca
3d87820
1bab0a6
 
 
 
 
db7f350
 
 
 
 
 
741edbf
 
db7f350
 
 
79f4b2b
 
8ce97f8
 
741edbf
39f3e26
5f9d165
 
 
8ce97f8
 
974b203
5f9d165
 
db7f350
 
 
 
 
 
3d87820
db7f350
5f9d165
 
 
3d87820
db7f350
 
 
 
 
 
 
 
 
0c95215
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
import os
import json
import datetime
from email.utils import parseaddr

import gradio as gr
import pandas as pd
import numpy as np

from datasets import load_dataset
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi

# InfoStrings
from scorer import question_scorer
from content import format_error, format_warning, format_log, TITLE, INTRODUCTION_TEXT, SUBMISSION_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, model_hyperlink

TOKEN = os.environ.get("TOKEN", None)

OWNER="gaia-benchmark"
DATA_DATASET = f"{OWNER}/GAIA"
INTERNAL_DATA_DATASET = f"{OWNER}/GAIA_internal"
SUBMISSION_DATASET = f"{OWNER}/submissions_internal"
CONTACT_DATASET = f"{OWNER}/contact_info"
RESULTS_DATASET = f"{OWNER}/results_public"
LEADERBOARD_PATH = f"{OWNER}/leaderboard"
api = HfApi()

YEAR_VERSION = "2023"

os.makedirs("scored", exist_ok=True)

# Display the results
eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", ignore_verifications=True, trust_remote_code=True)
contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", ignore_verifications=True, trust_remote_code=True)
def get_dataframe_from_results(eval_results, split):
    local_df = eval_results[split]
    local_df = local_df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
    local_df = local_df.remove_columns(["system_prompt", "url"])
    local_df = local_df.rename_column("model", "Agent name")
    local_df = local_df.rename_column("model_family", "Model family")
    local_df = local_df.rename_column("score", "Average score (%)")
    for i in [1, 2, 3]:
        local_df = local_df.rename_column(f"score_level{i}", f"Level {i} score (%)")
    df = pd.DataFrame(local_df)
    df = df.sort_values(by=["Average score (%)"], ascending=False)

    numeric_cols = [c for c in local_df.column_names if "score" in c]
    df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2)
    #df = df.style.format("{:.2%}", subset=numeric_cols)

    return df

eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="validation")
eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")

# Gold answers
gold_results = {}
gold_dataset = load_dataset(INTERNAL_DATA_DATASET, f"{YEAR_VERSION}_all", token=TOKEN, trust_remote_code=True)
gold_results = {split: {row["task_id"]: row for row in gold_dataset[split]} for split in ["test", "validation"]}


def restart_space():
    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)

TYPES = ["markdown", "number", "number", "number", "number", "str", "str"]

def add_new_eval(
    val_or_test: str,
    model: str,
    model_family: str,
    system_prompt: str,
    url: str,
    path_to_file: str,
    organisation: str,
    mail: str,
):
    # Very basic email parsing
    _, parsed_mail = parseaddr(mail)
    if not "@" in parsed_mail:
        return format_warning("Please provide a valid email adress.")

    print("Adding new eval")

    # Check if the combination model/org already exists and prints a warning message if yes
    if model.lower() in set([m.lower() for m in eval_results[val_or_test]["model"]]) and organisation.lower() in set([o.lower() for l in eval_results[val_or_test]["organisation"]]):
        return format_warning("This model has been already submitted.")
    
    if path_to_file is None:
        return format_warning("Please attach a file.")

    # Save submitted file
    api.upload_file(
        repo_id=SUBMISSION_DATASET, 
        path_or_fileobj=path_to_file.name, 
        path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_raw_{datetime.datetime.today()}.jsonl",
        repo_type="dataset", 
        token=TOKEN
    )

    # Compute score
    file_path = path_to_file.name        
    scores = {"all": 0, 1: 0, 2: 0, 3: 0}
    num_questions = {"all": 0, 1: 0, 2: 0, 3: 0}
    with open(f"scored/{organisation}_{model}.jsonl", "w") as scored_file:
        with open(file_path, 'r') as f:
            for ix, line in enumerate(f):
                try:
                    task = json.loads(line)
                except Exception:
                    return format_error(f"Line {ix} is incorrectly formatted. Please fix it and resubmit your file.")

                if "model_answer" not in task:
                    raise format_error(f"Line {ix} contains no model_answer key. Please fix it and resubmit your file.")
                answer = task["model_answer"]
                task_id = task["task_id"]
                try:
                    level = int(gold_results[val_or_test][task_id]["Level"])
                except KeyError:
                    return format_error(f"{task_id} not found in split {val_or_test}. Are you sure you submitted the correct file?")

                score = question_scorer(task['model_answer'], gold_results[val_or_test][task_id]["Final answer"])
                
                scored_file.write(
                    json.dumps({
                        "id": task_id,
                        "model_answer": answer,
                        "score": score,
                        "level": level
                    }) + "\n"
                )

                scores["all"] += score
                scores[level] += score
                num_questions["all"] += 1
                num_questions[level] += 1
    
    # Save scored file
    api.upload_file(
        repo_id=SUBMISSION_DATASET, 
        path_or_fileobj=f"scored/{organisation}_{model}.jsonl",
        path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_scored_{datetime.datetime.today()}.jsonl", 
        repo_type="dataset", 
        token=TOKEN
    )

    # Actual submission
    eval_entry = {
        "model": model,
        "model_family": model_family,
        "system_prompt": system_prompt,
        "url": url,
        "organisation": organisation,
        "score": scores["all"]/num_questions["all"],
        "score_level1": scores[1]/num_questions[1],
        "score_level2": scores[2]/num_questions[2],
        "score_level3": scores[3]/num_questions[3],
    }
    eval_results[val_or_test] = eval_results[val_or_test].add_item(eval_entry)
    print(eval_results)
    eval_results.push_to_hub(RESULTS_DATASET, config_name = YEAR_VERSION, token=TOKEN)

    contact_info = {
        "model": model,
        "model_family": model_family,
        "url": url,
        "organisation": organisation,
        "mail": mail,
    }
    contact_infos[val_or_test]= contact_infos[val_or_test].add_item(contact_info)
    contact_infos.push_to_hub(CONTACT_DATASET, config_name = YEAR_VERSION, token=TOKEN)

    return format_log(f"Model {model} submitted by {organisation} successfully.\nPlease wait a few hours and refresh the leaderboard to see your score displayed.")


def refresh():
    eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", ignore_verifications=True,trust_remote_code=True)
    eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="validation")
    eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
    return eval_dataframe_val, eval_dataframe_test

def upload_file(files):
    file_paths = [file.name for file in files]
    return file_paths


demo = gr.Blocks()
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                elem_id="citation-button",
            ) #.style(show_copy_button=True)

    with gr.Tab("Results: Test"):
        leaderboard_table_test = gr.components.Dataframe(
            value=eval_dataframe_test, datatype=TYPES, interactive=False,
            column_widths=["20%"] 
        )
    with gr.Tab("Results: Validation"):
        leaderboard_table_val = gr.components.Dataframe(
            value=eval_dataframe_val, datatype=TYPES, interactive=False,
            column_widths=["20%"] 
        )

    refresh_button = gr.Button("Refresh")
    refresh_button.click(
        refresh,
        inputs=[],
        outputs=[
            leaderboard_table_val,
            leaderboard_table_test,
        ],
    )
    with gr.Accordion("Submit a new model for evaluation"):
        with gr.Row():
            gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
        with gr.Row():
            with gr.Column():
                level_of_test = gr.Radio(["validation", "test"], value="validation", label="Split")
                model_name_textbox = gr.Textbox(label="Agent name")
                model_family_textbox = gr.Textbox(label="Model family")
                system_prompt_textbox = gr.Textbox(label="System prompt example")
                url_textbox = gr.Textbox(label="Url to model information")
            with gr.Column():
                organisation = gr.Textbox(label="Organisation")
                mail = gr.Textbox(label="Contact email (will be stored privately, & used if there is an issue with your submission)")
                file_output = gr.File()


        submit_button = gr.Button("Submit Eval")
        submission_result = gr.Markdown()
        submit_button.click(
            add_new_eval,
            [
                level_of_test,
                model_name_textbox,
                model_family_textbox,
                system_prompt_textbox,
                url_textbox,
                file_output,
                organisation,
                mail
            ],
            submission_result,
        )

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=3600)
scheduler.start()
demo.launch(debug=True)