File size: 12,215 Bytes
8e67ebe
 
4ade002
 
8e67ebe
a67391c
55fc7f4
8e67ebe
69b4d58
 
8e67ebe
6863798
8e67ebe
d6ca95d
 
 
d4a2bc9
 
 
 
 
 
61a2dd5
 
 
d4a2bc9
61a2dd5
d4a2bc9
 
 
1e6bb64
d4a2bc9
 
 
 
 
 
 
 
 
 
f14b394
d4a2bc9
 
 
 
 
8e67ebe
 
 
 
3c69767
8e67ebe
 
6863798
 
 
 
34ecb22
d6ca95d
 
4ade002
8e67ebe
d0e8be9
8e67ebe
 
d0e8be9
8e67ebe
 
 
 
69b4d58
d0e8be9
8e67ebe
d6ca95d
 
8e67ebe
69b4d58
 
8e67ebe
ce477d4
0d47bf5
d6ca95d
6863798
ce477d4
 
 
 
d0e8be9
ce477d4
d0e8be9
ce477d4
 
 
d0e8be9
ce477d4
 
 
 
 
 
 
8e67ebe
ce477d4
 
d0e8be9
6863798
d0e8be9
6863798
ce477d4
 
b2fa6ba
3f4d979
ce477d4
 
 
 
55fc7f4
d4a2bc9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61a2dd5
d0e8be9
61a2dd5
d0e8be9
d4a2bc9
 
 
 
 
f317a71
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69b4d58
 
 
55fc7f4
 
e7c6ec9
69b4d58
 
 
 
e133192
48fbcbf
69b4d58
 
 
 
 
d4a2bc9
d0e8be9
 
8e67ebe
 
d0e8be9
 
8e67ebe
 
40646ba
d0e8be9
ef23ff1
 
d0e8be9
137d615
 
 
 
 
5e23d17
 
d6ca95d
4dd39c5
4ade002
d6ca95d
31c09bf
d6ca95d
 
31c09bf
 
a0954a7
8fd12ea
a0954a7
4ade002
8fd12ea
14cd170
87e77f9
 
d6ca95d
 
87e77f9
 
 
 
 
 
6af7972
34ecb22
8eaf913
34ecb22
b19c539
34ecb22
8eaf913
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e348563
8eaf913
 
 
 
 
 
 
 
 
 
 
8e67ebe
 
d0e8be9
 
8e67ebe
8eaf913
14cd170
8e67ebe
d0e8be9
ce477d4
e0ca2ed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
import logging
import os
os.makedirs("tmp", exist_ok=True)
os.environ['TMP_DIR'] = "tmp"
import subprocess
import shutil
import glob
import gradio as gr
import numpy as np
from src.radial.radial import create_plot
from apscheduler.schedulers.background import BackgroundScheduler
from gradio_leaderboard import Leaderboard, SelectColumns
from gradio_space_ci import enable_space_ci
import json
from io import BytesIO

def handle_file_upload(file):
    file_path = file.name.split("/")[-1] if "/" in file.name else file.name
    logging.info("File uploaded: %s", file_path)
    with open(file.name, "r") as f:
        v = json.load(f)
    return v, file_path
def submit_file(v, file_path, mn, profile: gr.OAuthProfile | None):
    if profile is None:
        return "Hub Login Required"
    new_file = v['results']
    new_file['model'] = profile.username + "/" + mn
    new_file['moviesmc'] = new_file['moviemc']["acc,none"]
    new_file['musicmc'] = new_file['musicmc']["acc,none"]
    new_file['booksmc'] = new_file['bookmc']["acc,none"]
    new_file['mmluproru'] = new_file['mmluproru']["acc,none"]
    new_file['lawmc'] = new_file['lawmc']["acc,none"]
    new_file['model_dtype'] = v['config']["model_dtype"]
    new_file['ppl'] = 0
    new_file.pop('moviemc')
    new_file.pop('bookmc')

    buf = BytesIO()
    buf.write(json.dumps(new_file).encode('utf-8'))
    API.upload_file(
        path_or_fileobj=buf,
        path_in_repo="model_data/external/" + profile.username+mn + ".json",
        repo_id="Vikhrmodels/s-openbench-eval",
        repo_type="dataset",
    )
    os.environ[RESET_JUDGEMENT_ENV] = "1"
    return "Success!"

from src.display.about import (
    INTRODUCTION_TEXT,
    TITLE,
LLM_BENCHMARKS_TEXT
)
from src.display.css_html_js import custom_css
from src.display.utils import (
    AutoEvalColumn,
    fields,
)
from src.envs import API, H4_TOKEN, HF_HOME, REPO_ID, RESET_JUDGEMENT_ENV
from src.leaderboard.build_leaderboard import build_leadearboard_df, download_openbench, download_dataset
import huggingface_hub
# huggingface_hub.login(token=H4_TOKEN)

os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"

# Configure logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")

# Start ephemeral Spaces on PRs (see config in README.md)
enable_space_ci()

# download_openbench()

def restart_space():
    API.restart_space(repo_id=REPO_ID)
    download_openbench()

def update_plot(selected_models):
    return create_plot(selected_models)

def build_demo():
    download_openbench()
    demo = gr.Blocks(title="Small Shlepa", css=custom_css)
    leaderboard_df = build_leadearboard_df()
    with demo:
        gr.HTML(TITLE)
        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

        with gr.Tabs(elem_classes="tab-buttons"):
            with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
                Leaderboard(
                    value=leaderboard_df,
                    datatype=[c.type for c in fields(AutoEvalColumn)],
                    select_columns=SelectColumns(
                        default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
                        cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
                        label="Select Columns to Display:",
                    ),
                    search_columns=[
                        AutoEvalColumn.model.name,
                        # AutoEvalColumn.fullname.name,
                        # AutoEvalColumn.license.name
                    ],
                )

            # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
            #    gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
            # with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=2):
            #    gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")

            with gr.TabItem("🚀 Submit ", elem_id="llm-benchmark-tab-table", id=3):
                with gr.Row():
                    gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
                with gr.Row():
                    gr.Markdown("# ✨ Submit your model here!", elem_classes="markdown-text")

                with gr.Column():

                    # def upload_file(file,su,mn):
                    #     file_path = file.name.split("/")[-1] if "/" in file.name else file.name
                    #     logging.info("New submition: file saved to %s", file_path)
                    #     with open(file.name, "r") as f:
                    #         v=json.load(f)
                    #         new_file = v['results']
                    #         new_file['model'] = mn+"/"+su
                    #         new_file['moviesmc']=new_file['moviemc']["acc,none"]
                    #         new_file['musicmc']=new_file['musicmc']["acc,none"]
                    #         new_file['booksmc']=new_file['bookmc']["acc,none"]
                    #         new_file['lawmc']=new_file['lawmc']["acc,none"]
                    #         # name = v['config']["model_args"].split('=')[1].split(',')[0]
                    #         new_file['model_dtype'] = v['config']["model_dtype"]
                    #         new_file['ppl'] = 0
                    #         new_file.pop('moviemc')
                    #         new_file.pop('bookmc')
                    #     buf = BytesIO()
                    #     buf.write(json.dumps(new_file).encode('utf-8'))
                    #     API.upload_file(
                    #         path_or_fileobj=buf,
                    #         path_in_repo="model_data/external/" + su+mn + ".json",
                    #         repo_id="Vikhrmodels/s-openbench-eval",
                    #         repo_type="dataset",
                    #     )
                    #     os.environ[RESET_JUDGEMENT_ENV] = "1"
                    #     return file.name
                    # gr.LoginButton()
                    model_name_textbox = gr.Textbox(label="Model name")
                    # submitter_username = gr.Textbox(label="Username")

                    # def toggle_upload_button(model_name, username):
                    #     return bool(model_name) and bool(username)
                    file_output = gr.File(label="Drag and drop JSON file judgment here", type="filepath")
                    # upload_button = gr.Button("Click to Upload & Submit Answers", elem_id="upload_button",variant='primary')
                    uploaded_file = gr.State()
                    file_path = gr.State()
                    with gr.Row():
                        with gr.Column():
                            out = gr.Textbox("Статус отправки")
                        with gr.Column():
                             login_button = gr.LoginButton(elem_id="oauth-button")

                    submit_button = gr.Button("Submit File", elem_id="submit_button", variant='primary')

                    file_output.upload(
                        handle_file_upload,
                        file_output,
                        [uploaded_file, file_path]
                    )

                    submit_button.click(
                        submit_file,
                        [uploaded_file, file_path, model_name_textbox],
                        [out]
                    )

            with gr.TabItem("📊 Analytics", elem_id="llm-benchmark-tab-table", id=4):
                    with gr.Column():
                        model_dropdown = gr.Dropdown(
                            choices=leaderboard_df["model"].tolist(),
                            label="Models",
                            value=leaderboard_df["model"].tolist(),
                            multiselect=True,
                            info="Select models"
                        )
                    with gr.Column():
                        plot = gr.Plot(update_plot(model_dropdown.value))
                        # plot = gr.Plot()
                    model_dropdown.change(
                            fn=update_plot,
                            inputs=[model_dropdown],
                            outputs=[plot]
                    )
                    return demo


# print(os.system('cd src/gen && ../../.venv/bin/python gen_judgment.py'))
# print(os.system('cd src/gen/ && python show_result.py --output'))


def update_board():
    need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
    logging.info("Updating the judgement: %s", need_reset)
    if need_reset != "1":
        # return
        pass
    os.environ[RESET_JUDGEMENT_ENV] = "0"

    # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
    # recursively. In this specific context, it is used to delete the directory named "m_data" along
    # with all its files and subdirectories. This command helps in cleaning up the existing data in
    # the "m_data" directory before downloading new dataset files into it.
    # shutil.rmtree("./m_data")
    # shutil.rmtree("./data")
    download_dataset("Vikhrmodels/s-openbench-eval", "m_data")
    data_list = [{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0, 'mmluproru':0}]
    for file in glob.glob("./m_data/model_data/external/*.json"):
        with open(file) as f:
            try:
                data = json.load(f)
                data_list.append(data)
            except Exception as e:
                pass # data was badly formatted, should not fail
    print("DATALIST,", data_list)

    if len(data_list)>1:
        data_list.pop(0)

    if len(data_list)>4:
        with open("genned.json", "w") as f:
            json.dump(data_list, f)


        API.upload_file(
                path_or_fileobj="genned.json",
                path_in_repo="leaderboard.json",
                repo_id="Vikhrmodels/s-shlepa-metainfo",
                repo_type="dataset",
        )
    restart_space()


    # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py")
    # subprocess.run(["python3", gen_judgement_file], check=True)

def update_board_():
    need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
    logging.info("Updating the judgement: %s", need_reset)
    if need_reset != "1":
        # return
        pass
    os.environ[RESET_JUDGEMENT_ENV] = "0"

    # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
    # recursively. In this specific context, it is used to delete the directory named "m_data" along
    # with all its files and subdirectories. This command helps in cleaning up the existing data in
    # the "m_data" directory before downloading new dataset files into it.
    # shutil.rmtree("./m_data")
    # shutil.rmtree("./data")
    download_dataset("Vikhrmodels/s-openbench-eval", "m_data")
    data_list = [{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0, 'mmluproru':0}]
    for file in glob.glob("./m_data/model_data/external/*.json"):
        with open(file) as f:
            try:
                data = json.load(f)
                data_list.append(data)
            except Exception as e:
                pass # data was badly formatted, should not fail
    print("DATALIST,", data_list)

    if len(data_list)>1:
        data_list.pop(0)

    if len(data_list)>4:
        with open("genned.json", "w") as f:
            json.dump(data_list, f)


        API.upload_file(
                path_or_fileobj="genned.json",
                path_in_repo="leaderboard.json",
                repo_id="Vikhrmodels/s-shlepa-metainfo",
                repo_type="dataset",
        )

if __name__ == "__main__":
    os.environ[RESET_JUDGEMENT_ENV] = "1"

    scheduler = BackgroundScheduler()
    update_board_()
    scheduler.add_job(update_board, "interval", minutes=10)
    scheduler.start()

    demo_app = build_demo()
    demo_app.launch(debug=True,share=True)