feat: adapt UI in app.py

app.py

@@ -18,28 +18,28 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_leaderboard_df
 from utils import update_table
+from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, metric_list
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception:
-    restart_space()
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+#         token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+#         token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
 
 raw_data_qa, original_df_qa = get_leaderboard_df(
     EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_1')
@@ -58,7 +58,7 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("QA", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -67,56 +67,49 @@ with demo:
                             show_label=False,
                             elem_id="search-bar",
                         )
+                    # select domain
                     with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumnQA)
-                                if not c.hidden and not c.never_hidden
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumnQA)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
+                        selected_domains = gr.CheckboxGroup(
+                            choices=DOMAIN_COLS_QA,
+                            value=DOMAIN_COLS_QA,
+                            label="Select the domains",
+                            elem_id="domain-column-select",
                             interactive=True,
                         )
+                    # select language
+                    with gr.Row():
+                        selected_langs = gr.CheckboxGroup(
+                            choices=LANG_COLS_QA,
+                            value=LANG_COLS_QA,
+                            label="Select the languages",
+                            elem_id="language-column-select",
+                            interactive=True
+                        )
+                    # select reranking models
+                    reranking_models = list(frozenset([eval_result.retrieval_model for eval_result in raw_data_qa]))
                     with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
+                        selected_rerankings = gr.CheckboxGroup(
+                            choices=reranking_models,
+                            value=reranking_models,
+                            label="Select the reranking models",
+                            elem_id="reranking-select",
+                            interactive=True
                         )
                 with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
+                    selected_metric = gr.Dropdown(
+                        choices=metric_list,
+                        value=metric_list,
+                        label="Select the metric",
                         interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
+                        elem_id="metric-select",
                     )
+            # update shown_columns when selected_langs and selected_domains are changed
+            shown_columns = leaderboard_df.columns
 
+            # reload the leaderboard_df and raw_data when selected_metric is changed
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumnQA) if c.never_hidden]
-                    + shown_columns.value
-                    ],
-                headers=[c.name for c in fields(AutoEvalColumnQA) if c.never_hidden] + shown_columns.value,
+                value=leaderboard_df,
+                # headers=shown_columns,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -124,41 +117,34 @@ with demo:
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df_qa[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size,
-                             deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
+            # hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            #     value=original_df_qa[COLS],
+            #     headers=COLS,
+            #     datatype=TYPES,
+            #     visible=False,
+            # )
+            # search_bar.submit(
+            #     update_table,
+            #     [
+            #         hidden_leaderboard_table_for_search,
+            #         shown_columns,
+            #         selected_rerankings,
+            #         search_bar,
+            #     ],
+            #     leaderboard_table,
+            # )
+            # for selector in [shown_columns, selected_rerankings, search_bar]:
+            #     selector.change(
+            #         update_table,
+            #         [
+            #             hidden_leaderboard_table_for_search,
+            #             shown_columns,
+            #             selected_rerankings,
+            #             search_bar,
+            #         ],
+            #         leaderboard_table,
+            #         queue=True,
+            #     )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/benchmarks.py

@@ -135,4 +135,7 @@ for task, domain_dict in dataset_dict.items():
 BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
 BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
-BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
+BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
+
+DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
+LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))

src/envs.py

@@ -17,9 +17,9 @@ RESULTS_REPO = f"{OWNER}/results"
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+EVAL_REQUESTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/tests/toydata/test_requests"  # os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/tests/toydata/test_results"  #os.path.join(CACHE_PATH, "eval-results")
+# EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+# EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)

src/populate.py

@@ -17,13 +17,15 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         all_data_json += v.to_dict(task=task, metric=metric)
 
     df = pd.DataFrame.from_records(all_data_json)
-    df[AutoEvalColumnQA.average.name] = df[benchmark_cols].mean(axis=1)
+    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
+    df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1)
     df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
     df.reset_index(inplace=True)
-    df = df[cols].round(decimals=2)
+    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
+    df = df[_cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    df = df[has_no_nan_values(df, _benchmark_cols)]
     return raw_data, df
 
 

tests/src/test_populate.py

@@ -9,9 +9,9 @@ def test_get_leaderboard_df():
     results_path = cur_fp.parents[1] / "toydata" / "test_results"
     cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'wiki_en', 'wiki_zh',]
     benchmark_cols = ['wiki_en', 'wiki_zh',]
-    raw_data, df = get_leaderboard_df(results_path, requests_path, cols, benchmark_cols)
+    raw_data, df = get_leaderboard_df(results_path, requests_path, cols, benchmark_cols, 'qa', 'ndcg_at_1')
     assert df.shape[0] == 2
-    # the results contains only one embedding model
+    # the results contain only one embedding model
     for i in range(2):
         assert df["Retrieval Model"][i] == "bge-m3"
     # the results contains only two reranking model