Clémentine
now intersection of leaderboards
4b43014
import gradio as gr
from apscheduler.schedulers.background import BackgroundScheduler
from src.static.env import API, REPO_ID, HF_TOKEN
from src.static.about import TITLE, INTRO, ABOUT, DOCUMENTATION, SUBMIT
from src.leaderboards.get_from_hub import get_leaderboard_info
from src.static.tag_info import *
from src.static.display import make_clickable
def restart_space():
API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
LEADERBOARDS_TO_INFO, INFO_TO_LEADERBOARDS = get_leaderboard_info()
def update_leaderboards(show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags):
spaces_of_interest = []
if show_all:
spaces_of_interest = INFO_TO_LEADERBOARDS["all"]
else:
number_of_tag_types = 0
for tag in modality_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["modality"][tag.lower()])
number_of_tag_types += 1
for tag in submission_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["submission"][tag.lower()])
number_of_tag_types += 1
for tag in test_set_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["test"][tag.lower()])
number_of_tag_types += 1
for tag in evaluation_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["eval"][tag.lower()])
number_of_tag_types += 1
for tag in language_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["language"][tag.lower()])
number_of_tag_types += 1
for tag in judge_tags:
spaces_of_interest.extend(INFO_TO_LEADERBOARDS["judge"][tag.lower()])
number_of_tag_types += 1
if number_of_tag_types > 1:
# Intersection of tag types
spaces_of_interest = [space for space in spaces_of_interest if spaces_of_interest.count(space) > 1]
spaces_of_interest = sorted(list(set(spaces_of_interest)))
return "\n".join(
[
make_clickable(space) +
f"*Tags: {', '.join(LEADERBOARDS_TO_INFO[space]) if len(LEADERBOARDS_TO_INFO[space]) > 0 else 'None. Please fill the tags!'}*"
for space in sorted(spaces_of_interest)
]
)
demo = gr.Blocks()
with demo:
gr.Markdown(TITLE)
gr.Markdown(INTRO, elem_classes="markdown-text")
with gr.Row():
with gr.Column():
modality_tags = gr.CheckboxGroup(
choices=[tag.name for tag in Modality],
value=[],
label="Modality of choice"
)
submission_tags = gr.CheckboxGroup(
choices=[tag.name for tag in SubmissionType],
value=[],
label="Submission type"
)
test_set_tags = gr.CheckboxGroup(
choices=[tag.name for tag in TestSetStatus],
value=[],
label="Test set status"
)
judge_tags = gr.CheckboxGroup(
choices=[tag.name for tag in Judge],
value=[],
label="Judge used for the evaluation"
)
with gr.Column():
show_all = gr.Checkbox(
value=False,
label="Show all leaderboards"
)
evaluation_tags = gr.CheckboxGroup(
choices=[tag.name for tag in EvaluationCategory],
value=[],
label="Specific evaluation categories"
)
language_tags = gr.CheckboxGroup(
choices=[tag.capitalize() for tag in sorted(list(INFO_TO_LEADERBOARDS["language"].keys()))],
value=[],
label="Specific languages"
)
with gr.Row():
leaderboards = gr.Markdown(
value="",
)
for selector in [modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
selector.change(
lambda _: False,
outputs=show_all
)
for selector in [show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
selector.change(
update_leaderboards,
[show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags],
leaderboards,
queue=True,
)
with gr.Accordion("How to submit your leaderboard?", open=False):
gr.Markdown(SUBMIT, elem_classes="markdown-text")
with gr.Accordion("What do the tags mean?", open=False):
gr.Markdown(ABOUT, elem_classes="markdown-text")
#with gr.Accordion("How to build your own leaderboard?", open=False):
# gr.Markdown(DOCUMENTATION, elem_classes="markdown-text")
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()