Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
| import os | |
| import gradio as gr | |
| from apscheduler.schedulers.background import BackgroundScheduler | |
| from huggingface_hub import snapshot_download | |
| from src.about import BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE | |
| from src.benchmarks import LongDocBenchmarks, QABenchmarks | |
| from src.display.components import ( | |
| get_anonymous_checkbox, | |
| get_domain_dropdown, | |
| get_language_dropdown, | |
| get_leaderboard_table, | |
| get_metric_dropdown, | |
| get_noreranking_dropdown, | |
| get_reranking_dropdown, | |
| get_revision_and_ts_checkbox, | |
| get_search_bar, | |
| get_version_dropdown, | |
| ) | |
| from src.display.css_html_js import custom_css | |
| from src.envs import ( | |
| API, | |
| BENCHMARK_VERSION_LIST, | |
| BM25_LINK, | |
| COL_NAME_RERANKING_MODEL, | |
| COL_NAME_RETRIEVAL_MODEL, | |
| DEFAULT_METRIC_LONG_DOC, | |
| DEFAULT_METRIC_QA, | |
| EVAL_RESULTS_PATH, | |
| LATEST_BENCHMARK_VERSION, | |
| METRIC_LIST, | |
| REPO_ID, | |
| RESULTS_REPO, | |
| TOKEN, | |
| ) | |
| from src.loaders import load_eval_results | |
| from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file | |
| def restart_space(): | |
| API.restart_space(repo_id=REPO_ID) | |
| try: | |
| if not os.environ.get("LOCAL_MODE", False): | |
| print("Running in local mode") | |
| snapshot_download( | |
| repo_id=RESULTS_REPO, | |
| local_dir=EVAL_RESULTS_PATH, | |
| repo_type="dataset", | |
| tqdm_class=None, | |
| etag_timeout=30, | |
| token=TOKEN, | |
| ) | |
| except Exception: | |
| print("failed to download") | |
| restart_space() | |
| global data | |
| data = load_eval_results(EVAL_RESULTS_PATH) | |
| global datastore | |
| datastore = data[LATEST_BENCHMARK_VERSION] | |
| def update_metric_qa( | |
| metric: str, | |
| domains: list, | |
| langs: list, | |
| reranking_model: list, | |
| query: str, | |
| show_anonymous: bool, | |
| show_revision_and_timestamp: bool, | |
| ): | |
| return update_metric( | |
| datastore, | |
| "qa", | |
| metric, | |
| domains, | |
| langs, | |
| reranking_model, | |
| query, | |
| show_anonymous, | |
| show_revision_and_timestamp | |
| ) | |
| def update_metric_long_doc( | |
| metric: str, | |
| domains: list, | |
| langs: list, | |
| reranking_model: list, | |
| query: str, | |
| show_anonymous: bool, | |
| show_revision_and_timestamp, | |
| ): | |
| return update_metric( | |
| datastore, | |
| "long-doc", | |
| metric, | |
| domains, | |
| langs, | |
| reranking_model, | |
| query, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| def update_datastore(version): | |
| global datastore | |
| global data | |
| datastore = data[version] | |
| selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug]) | |
| selected_langs = get_language_dropdown(QABenchmarks[datastore.slug]) | |
| selected_rerankings = get_reranking_dropdown(datastore.reranking_models) | |
| leaderboard_table = get_leaderboard_table(datastore.formatted_df_qa, datastore.types_qa) | |
| hidden_leaderboard_table = \ | |
| get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False) | |
| return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table | |
| def update_datastore_long_doc(version): | |
| global datastore | |
| global data | |
| datastore = data[version] | |
| selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug]) | |
| selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug]) | |
| selected_rerankings = get_reranking_dropdown(datastore.reranking_models) | |
| leaderboard_table = get_leaderboard_table(datastore.formatted_df_ldoc, datastore.types_long_doc) | |
| hidden_leaderboard_table = \ | |
| get_leaderboard_table(datastore.raw_df_long_doc, datastore.types_long_doc, visible=False) | |
| return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem("Results", elem_id="results-tab-table"): | |
| with gr.Row(): | |
| selected_version = get_version_dropdown() | |
| with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0): | |
| with gr.Row(): | |
| with gr.Column(min_width=320): | |
| # select domain | |
| with gr.Row(): | |
| selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug]) | |
| # select language | |
| with gr.Row(): | |
| selected_langs = get_language_dropdown(QABenchmarks[datastore.slug]) | |
| with gr.Column(): | |
| # select the metric | |
| selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA) | |
| with gr.Row(): | |
| show_anonymous = get_anonymous_checkbox() | |
| with gr.Row(): | |
| show_revision_and_timestamp = get_revision_and_ts_checkbox() | |
| with gr.Tabs(elem_classes="tab-buttons") as sub_tabs: | |
| with gr.TabItem("Retrieval + Reranking", id=10): | |
| with gr.Row(): | |
| # search retrieval models | |
| with gr.Column(): | |
| search_bar = get_search_bar() | |
| # select reranking models | |
| with gr.Column(): | |
| selected_rerankings = get_reranking_dropdown(datastore.reranking_models) | |
| # shown_table | |
| lb_table = get_leaderboard_table(datastore.formatted_df_qa, datastore.types_qa) | |
| # Dummy leaderboard for handling the case when the user uses backspace key | |
| hidden_lb_table = get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False) | |
| selected_version.change( | |
| update_datastore, | |
| [ | |
| selected_version, | |
| ], | |
| [selected_domains, selected_langs, selected_rerankings, lb_table, hidden_lb_table], | |
| ) | |
| set_listeners( | |
| "qa", | |
| lb_table, | |
| hidden_lb_table, | |
| search_bar, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| # set metric listener | |
| selected_metric.change( | |
| update_metric_qa, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings, | |
| search_bar, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table, | |
| queue=True, | |
| ) | |
| with gr.TabItem("Retrieval Only", id=11): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| search_bar_retriever = get_search_bar() | |
| with gr.Column(scale=1): | |
| selected_noreranker = get_noreranking_dropdown() | |
| lb_df_retriever = datastore.formatted_df_qa[ | |
| datastore.formatted_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker" | |
| ] | |
| lb_df_retriever = reset_rank(lb_df_retriever) | |
| lb_table_retriever = get_leaderboard_table(lb_df_retriever, datastore.types_qa) | |
| # Dummy leaderboard for handling the case when the user uses backspace key | |
| hidden_lb_df_retriever = datastore.raw_df_qa[ | |
| datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker" | |
| ] | |
| hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever) | |
| hidden_lb_table_retriever = get_leaderboard_table( | |
| hidden_lb_df_retriever, datastore.types_qa, visible=False | |
| ) | |
| selected_version.change( | |
| update_datastore, | |
| [ | |
| selected_version, | |
| ], | |
| [ | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| lb_table_retriever, | |
| hidden_lb_table_retriever, | |
| ], | |
| ) | |
| set_listeners( | |
| "qa", | |
| lb_table_retriever, | |
| hidden_lb_table_retriever, | |
| search_bar_retriever, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| # set metric listener | |
| selected_metric.change( | |
| update_metric_qa, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| search_bar_retriever, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table_retriever, | |
| queue=True, | |
| ) | |
| with gr.TabItem("Reranking Only", id=12): | |
| lb_df_reranker = datastore.formatted_df_qa[ | |
| datastore.formatted_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK | |
| ] | |
| lb_df_reranker = reset_rank(lb_df_reranker) | |
| reranking_models_reranker = ( | |
| lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist() | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker) | |
| with gr.Column(scale=1): | |
| search_bar_reranker = gr.Textbox(show_label=False, visible=False) | |
| lb_table_reranker = get_leaderboard_table(lb_df_reranker, datastore.types_qa) | |
| hidden_lb_df_reranker = datastore.raw_df_qa[ | |
| datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK | |
| ] | |
| hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker) | |
| hidden_lb_table_reranker = get_leaderboard_table( | |
| hidden_lb_df_reranker, datastore.types_qa, visible=False | |
| ) | |
| selected_version.change( | |
| update_datastore, | |
| [ | |
| selected_version, | |
| ], | |
| [ | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker, | |
| lb_table_reranker, | |
| hidden_lb_table_reranker, | |
| ], | |
| ) | |
| set_listeners( | |
| "qa", | |
| lb_table_reranker, | |
| hidden_lb_table_reranker, | |
| search_bar_reranker, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| # set metric listener | |
| selected_metric.change( | |
| update_metric_qa, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker, | |
| search_bar_reranker, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table_reranker, | |
| queue=True, | |
| ) | |
| with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1): | |
| with gr.Row(): | |
| with gr.Column(min_width=320): | |
| # select domain | |
| with gr.Row(): | |
| selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug]) | |
| # select language | |
| with gr.Row(): | |
| selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug]) | |
| with gr.Column(): | |
| # select the metric | |
| with gr.Row(): | |
| selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC) | |
| with gr.Row(): | |
| show_anonymous = get_anonymous_checkbox() | |
| with gr.Row(): | |
| show_revision_and_timestamp = get_revision_and_ts_checkbox() | |
| with gr.Tabs(elem_classes="tab-buttons"): | |
| with gr.TabItem("Retrieval + Reranking", id=20): | |
| with gr.Row(): | |
| with gr.Column(): | |
| search_bar = get_search_bar() | |
| with gr.Column(): | |
| selected_rerankings = get_reranking_dropdown(datastore.reranking_models) | |
| lb_table_long_doc = get_leaderboard_table( | |
| datastore.formatted_df_ldoc, datastore.types_long_doc | |
| ) | |
| # Dummy leaderboard for handling the case when the user uses backspace key | |
| hidden_lb_table_long_doc = get_leaderboard_table( | |
| datastore.raw_df_long_doc, datastore.types_long_doc, visible=False | |
| ) | |
| selected_version.change( | |
| update_datastore_long_doc, | |
| [ | |
| selected_version, | |
| ], | |
| [ | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings, | |
| lb_table_long_doc, | |
| hidden_lb_table_long_doc, | |
| ], | |
| ) | |
| set_listeners( | |
| "long-doc", | |
| lb_table_long_doc, | |
| hidden_lb_table_long_doc, | |
| search_bar, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| # set metric listener | |
| selected_metric.change( | |
| update_metric_long_doc, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings, | |
| search_bar, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table_long_doc, | |
| queue=True, | |
| ) | |
| with gr.TabItem("Retrieval Only", id=21): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| search_bar_retriever = get_search_bar() | |
| with gr.Column(scale=1): | |
| selected_noreranker = get_noreranking_dropdown() | |
| lb_df_retriever_long_doc = datastore.formatted_df_ldoc[ | |
| datastore.formatted_df_ldoc[COL_NAME_RERANKING_MODEL] == "NoReranker" | |
| ] | |
| lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc) | |
| lb_table_retriever_long_doc = get_leaderboard_table( | |
| lb_df_retriever_long_doc, datastore.types_long_doc | |
| ) | |
| hidden_lb_df_retriever_long_doc = datastore.raw_df_long_doc[ | |
| datastore.raw_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker" | |
| ] | |
| hidden_lb_df_retriever_long_doc = reset_rank(hidden_lb_df_retriever_long_doc) | |
| hidden_lb_table_retriever_long_doc = get_leaderboard_table( | |
| hidden_lb_df_retriever_long_doc, datastore.types_long_doc, visible=False | |
| ) | |
| selected_version.change( | |
| update_datastore_long_doc, | |
| [ | |
| selected_version, | |
| ], | |
| [ | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| lb_table_retriever_long_doc, | |
| hidden_lb_table_retriever_long_doc, | |
| ], | |
| ) | |
| set_listeners( | |
| "long-doc", | |
| lb_table_retriever_long_doc, | |
| hidden_lb_table_retriever_long_doc, | |
| search_bar_retriever, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| selected_metric.change( | |
| update_metric_long_doc, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_noreranker, | |
| search_bar_retriever, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table_retriever_long_doc, | |
| queue=True, | |
| ) | |
| with gr.TabItem("Reranking Only", id=22): | |
| lb_df_reranker_ldoc = datastore.formatted_df_ldoc[ | |
| datastore.formatted_df_ldoc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK | |
| ] | |
| lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc) | |
| reranking_models_reranker_ldoc = ( | |
| lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist() | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| selected_rerankings_reranker_ldoc = get_reranking_dropdown( | |
| reranking_models_reranker_ldoc | |
| ) | |
| with gr.Column(scale=1): | |
| search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False) | |
| lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, datastore.types_long_doc) | |
| hidden_lb_df_reranker_ldoc = datastore.raw_df_long_doc[ | |
| datastore.raw_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK | |
| ] | |
| hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc) | |
| hidden_lb_table_reranker_ldoc = get_leaderboard_table( | |
| hidden_lb_df_reranker_ldoc, datastore.types_long_doc, visible=False | |
| ) | |
| selected_version.change( | |
| update_datastore_long_doc, | |
| [ | |
| selected_version, | |
| ], | |
| [ | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker_ldoc, | |
| lb_table_reranker_ldoc, | |
| hidden_lb_table_reranker_ldoc, | |
| ], | |
| ) | |
| set_listeners( | |
| "long-doc", | |
| lb_table_reranker_ldoc, | |
| hidden_lb_table_reranker_ldoc, | |
| search_bar_reranker_ldoc, | |
| selected_version, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker_ldoc, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ) | |
| selected_metric.change( | |
| update_metric_long_doc, | |
| [ | |
| selected_metric, | |
| selected_domains, | |
| selected_langs, | |
| selected_rerankings_reranker_ldoc, | |
| search_bar_reranker_ldoc, | |
| show_anonymous, | |
| show_revision_and_timestamp, | |
| ], | |
| lb_table_reranker_ldoc, | |
| queue=True, | |
| ) | |
| with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2): | |
| with gr.Column(): | |
| with gr.Row(): | |
| gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") | |
| with gr.Row(): | |
| gr.Markdown("## ✉️Submit your model here!", elem_classes="markdown-text") | |
| with gr.Row(): | |
| with gr.Column(): | |
| model_name = gr.Textbox(label="Retrieval Method name") | |
| with gr.Column(): | |
| model_url = gr.Textbox(label="Retrieval Method URL") | |
| with gr.Row(): | |
| with gr.Column(): | |
| reranking_model_name = gr.Textbox( | |
| label="Reranking Model name", info="Optional", value="NoReranker" | |
| ) | |
| with gr.Column(): | |
| reranking_model_url = gr.Textbox(label="Reranking Model URL", info="Optional", value="") | |
| with gr.Row(): | |
| with gr.Column(): | |
| benchmark_version = gr.Dropdown( | |
| BENCHMARK_VERSION_LIST, | |
| value=LATEST_BENCHMARK_VERSION, | |
| interactive=True, | |
| label="AIR-Bench Version", | |
| ) | |
| with gr.Row(): | |
| upload_button = gr.UploadButton("Click to upload search results", file_count="single") | |
| with gr.Row(): | |
| file_output = gr.File() | |
| with gr.Row(): | |
| is_anonymous = gr.Checkbox( | |
| label="Nope. I want to submit anonymously 🥷", | |
| value=False, | |
| info="Do you want to shown on the leaderboard by default?", | |
| ) | |
| with gr.Row(): | |
| submit_button = gr.Button("Submit") | |
| with gr.Row(): | |
| submission_result = gr.Markdown() | |
| upload_button.upload( | |
| upload_file, | |
| [ | |
| upload_button, | |
| ], | |
| file_output, | |
| ) | |
| submit_button.click( | |
| submit_results, | |
| [ | |
| file_output, | |
| model_name, | |
| model_url, | |
| reranking_model_name, | |
| reranking_model_url, | |
| benchmark_version, | |
| is_anonymous, | |
| ], | |
| submission_result, | |
| show_progress="hidden", | |
| ) | |
| with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3): | |
| gr.Markdown(BENCHMARKS_TEXT, elem_classes="markdown-text") | |
| if __name__ == "__main__": | |
| scheduler = BackgroundScheduler() | |
| scheduler.add_job(restart_space, "interval", seconds=1800) | |
| scheduler.start() | |
| demo.queue(default_concurrency_limit=40) | |
| demo.launch() | |