| import gradio as gr | |
| import core as core | |
| from style import CSS, T_SYMBOLS, TITLE | |
| demo = gr.Blocks(css=CSS) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown( | |
| "This is a (WIP) collection of multilingual evaluation results obtained using our fork of the LM-evaluation-harness (https://github.com/OpenGPTX/lm-evaluation-harness), based on https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard.\ | |
| Note that currently, not all benchmarks are available in all languages, results are averaged over those languages under the selected ones for which the benchmark is available.", | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.Column(): | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Row(): | |
| search_bar = gr.Textbox( | |
| label="Search models", | |
| placeholder=" 🔍 Separate multiple queries with ';' and press ENTER...", | |
| show_label=True, | |
| elem_id="search-bar", | |
| ) | |
| model_types = gr.CheckboxGroup( | |
| label="Select model type", | |
| choices=[ | |
| ( | |
| f"Pretrained {T_SYMBOLS['pretrained']}", | |
| T_SYMBOLS["pretrained"], | |
| ), | |
| (f"Chat {T_SYMBOLS['chat']}", T_SYMBOLS["chat"]), | |
| ], | |
| value=list(T_SYMBOLS.values()), | |
| ) | |
| with gr.Row(): | |
| langs_bar = gr.CheckboxGroup( | |
| choices=core.languages_list, | |
| value=core.languages_list, | |
| label="Select languages to average over", | |
| elem_id="column-select", | |
| interactive=True, | |
| scale=6, | |
| ) | |
| with gr.Column(scale=1): | |
| clear = gr.ClearButton( | |
| langs_bar, | |
| value="Deselect all languages", | |
| size="sm", | |
| scale=1, | |
| ) | |
| select = gr.Button( | |
| value="Select all languages", size="sm", scale=1 | |
| ) | |
| def update_bar(): | |
| langs_bar = gr.CheckboxGroup( | |
| choices=core.languages_list, | |
| value=core.languages_list, | |
| label="Select languages to average over", | |
| elem_id="column-select", | |
| interactive=True, | |
| ) | |
| return langs_bar | |
| select.click(update_bar, inputs=[], outputs=langs_bar) | |
| with gr.Row(): | |
| acc_task_group_names = core.task_groups_with_task_type("accuracy") | |
| shown_tasks = gr.CheckboxGroup( | |
| choices=acc_task_group_names, | |
| value=acc_task_group_names, | |
| label="Select tasks to show", | |
| elem_id="column-select", | |
| interactive=True, | |
| scale=50, | |
| ) | |
| fewshot = gr.Radio( | |
| choices=[("0-Shot", False), ("Few-shot", True)], | |
| value=True, | |
| label="Select evaluation type", | |
| interactive=True, | |
| scale=29, | |
| ) | |
| fewshot.change( | |
| core.fix_zeroshot, [shown_tasks, fewshot], shown_tasks | |
| ) | |
| clear = gr.ClearButton( | |
| shown_tasks, value="Deselect all tasks", size="sm", scale=21 | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem( | |
| "🏅 LLM accuracy benchmark", elem_id="llm-benchmark-tab-table-acc", id=0 | |
| ) as acc: | |
| leaderboard_table = gr.Dataframe() | |
| with gr.TabItem( | |
| "🌐 LLM translation benchmark", | |
| elem_id="llm-benchmark-tab-table-misc", | |
| id=1, | |
| ) as misc: | |
| leaderboard_table_misc = gr.Dataframe() | |
| with gr.TabItem("Plots", elem_id="llm-plot-tab", id=2) as plot: | |
| leaderboard_plot = gr.Plot(elem_id="plot") | |
| acc.select( | |
| lambda x: core.update_tab_tasks(0, x), | |
| inputs=fewshot, | |
| outputs=[shown_tasks, fewshot], | |
| ) | |
| misc.select( | |
| lambda x: core.update_tab_tasks(1, x), | |
| inputs=fewshot, | |
| outputs=[shown_tasks, fewshot], | |
| ) | |
| for comp, fn in [ | |
| (search_bar, "submit"), | |
| (langs_bar, "change"), | |
| (shown_tasks, "change"), | |
| (fewshot, "change"), | |
| (model_types, "change"), | |
| ]: | |
| getattr(comp, fn)( | |
| core.update_df, | |
| [shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| leaderboard_table, | |
| ) | |
| getattr(comp, fn)( | |
| core.update_df, | |
| [shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| leaderboard_table_misc, | |
| ) | |
| getattr(comp, fn)( | |
| core.update_plot, | |
| [shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| leaderboard_plot, | |
| ) | |
| gr.Blocks.load( | |
| block=demo, | |
| fn=core.update_df, | |
| inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| outputs=leaderboard_table, | |
| ) | |
| gr.Blocks.load( | |
| block=demo, | |
| fn=core.update_df, | |
| inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| outputs=leaderboard_table_misc, | |
| ) | |
| gr.Blocks.load( | |
| block=demo, | |
| fn=core.update_plot, | |
| inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot], | |
| outputs=leaderboard_plot, | |
| ) | |
| demo.launch() | |