File size: 4,428 Bytes
4b2522c
 
 
43c04ea
4b2522c
 
84b5dfa
 
4b2522c
 
 
 
 
84b5dfa
 
e5438c2
84b5dfa
 
 
 
 
 
 
 
 
 
 
67e855c
84b5dfa
 
e5438c2
 
84b5dfa
 
 
fd5a922
84b5dfa
 
 
4b2522c
 
 
 
84b5dfa
4b2522c
 
43c04ea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f161dff
43c04ea
 
b521c84
 
84b5dfa
4b2522c
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import gradio as gr
from apscheduler.schedulers.background import BackgroundScheduler
from src.static.env import API, REPO_ID, HF_TOKEN
from src.static.about import TITLE, INTRO, ABOUT, DOCUMENTATION, SUBMIT

from src.leaderboards.get_from_hub import get_leaderboard_info
from src.static.tag_info import *
from src.static.display import make_clickable


def restart_space():
    API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)

LEADERBOARDS_TO_INFO, INFO_TO_LEADERBOARDS = get_leaderboard_info()

def update_leaderboards(show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags):
    spaces_of_interest = []
    if show_all:
        spaces_of_interest = INFO_TO_LEADERBOARDS["all"]
    else:
        for tag in modality_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["modality"][tag.lower()])
        for tag in submission_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["submission"][tag.lower()])
        for tag in test_set_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["test"][tag.lower()])
        for tag in evaluation_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["eval"][tag.lower()])
        for tag in language_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["language"][tag.lower()])
        for tag in judge_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["judge"][tag.lower()])

    return "- " + "\n - ".join([
        make_clickable(space) + 
        f"*Tags: {', '.join(LEADERBOARDS_TO_INFO[space]) if len(LEADERBOARDS_TO_INFO[space]) > 0 else 'None. Please fill the tags!'}*" 
        for space in spaces_of_interest
        ])



demo = gr.Blocks()
with demo:
    gr.Markdown(TITLE)
    gr.Markdown(INTRO, elem_classes="markdown-text")

    with gr.Row():
        with gr.Column():
            modality_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in Modality],
                value=[],
                label="Modality of choice"
            )
            submission_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in SubmissionType],
                value=[],
                label="Submission type"
            )
            test_set_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in TestSetStatus],
                value=[],
                label="Test set status"
            )
            judge_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in Judge],
                value=[],
                label="Judge used for the evaluation"
            )
        with gr.Column():
            show_all = gr.Checkbox(
                value=False, 
                label="Show all leaderboards"
            )
            evaluation_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in EvaluationCategory],
                value=[],
                label="Specific evaluation categories"
            )
            language_tags = gr.CheckboxGroup(
                choices=[tag.capitalize() for tag in sorted(list(INFO_TO_LEADERBOARDS["language"].keys()))],
                value=[],
                label="Specific languages"
            ) 
    with gr.Row():
        leaderboards = gr.Markdown(
            value="",
        )

    for selector in [modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
        selector.change(
            lambda _: False,
            outputs=show_all
        )
    for selector in [show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
        selector.change(
            update_leaderboards,
            [show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags],
            leaderboards,
            queue=True,
        )

    with gr.Accordion("How to submit your leaderboard?", open=False):
        gr.Markdown(SUBMIT, elem_classes="markdown-text")

    with gr.Accordion("What do the tags mean?", open=False):
        gr.Markdown(ABOUT, elem_classes="markdown-text")

    #with gr.Accordion("How to build your own leaderboard?", open=False):
    #    gr.Markdown(DOCUMENTATION, elem_classes="markdown-text")

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
scheduler.start()

demo.queue(default_concurrency_limit=40).launch()