feat: implement anonymous displaying for qa

app.py

@@ -11,6 +11,7 @@ from src.about import (
 from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, METRIC_LIST, \
     DEFAULT_METRIC
 from src.display.css_html_js import custom_css
+from src.display.utils import COL_NAME_IS_ANONYMOUS
 from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
 from src.read_evals import get_raw_eval_results, get_leaderboard_df
 from src.utils import update_table, update_metric, update_table_long_doc, upload_file, get_default_cols, submit_results
@@ -20,13 +21,14 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-try:
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception:
-    restart_space()
+# try:
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+#         token=TOKEN
+#     )
+# except Exception as e:
+#     print(f'failed to download')
+#     restart_space()
 
 raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/AIR-Bench_24.04")
 
@@ -40,7 +42,7 @@ print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
 
 leaderboard_df_qa = original_df_qa.copy()
 shown_columns_qa, types_qa = get_default_cols('qa', leaderboard_df_qa.columns, add_fix_cols=True)
-leaderboard_df_qa = leaderboard_df_qa[shown_columns_qa]
+leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
 
 leaderboard_df_long_doc = original_df_long_doc.copy()
 shown_columns_long_doc, types_long_doc = get_default_cols('long-doc', leaderboard_df_long_doc.columns,
@@ -54,8 +56,9 @@ def update_metric_qa(
         langs: list,
         reranking_model: list,
         query: str,
+        show_anonymous: bool
 ):
-    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query)
+    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous)
 
 
 def update_metric_long_doc(
@@ -123,6 +126,12 @@ with demo:
                             multiselect=True,
                             interactive=True
                         )
+                    with gr.Row():
+                        show_anonymous = gr.Checkbox(
+                            label="Show anonymous submissions",
+                            value=False,
+                            info="The anonymous submissions might have invalid model information."
+                        )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df_qa,
@@ -134,10 +143,8 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=leaderboard_df_qa,
+                value=original_df_qa,
                 datatype=types_qa,
-                # headers=COLS,
-                # datatype=TYPES,
                 visible=False,
             )
 
@@ -150,13 +157,14 @@ with demo:
                     selected_langs,
                     selected_rerankings,
                     search_bar,
+                    show_anonymous,
                 ],
                 leaderboard_table,
             )
 
             # Set column-wise listener
             for selector in [
-                selected_domains, selected_langs, selected_rerankings
+                selected_domains, selected_langs, selected_rerankings, show_anonymous
             ]:
                 selector.change(
                     update_table,
@@ -166,6 +174,7 @@ with demo:
                         selected_langs,
                         selected_rerankings,
                         search_bar,
+                        show_anonymous,
                     ],
                     leaderboard_table,
                     queue=True,
@@ -180,6 +189,7 @@ with demo:
                     selected_langs,
                     selected_rerankings,
                     search_bar,
+                    show_anonymous,
                 ],
                 leaderboard_table,
                 queue=True

src/read_evals.py

@@ -15,6 +15,7 @@ from src.display.utils import (
     COL_NAME_RETRIEVAL_MODEL_LINK,
     COL_NAME_REVISION,
     COL_NAME_TIMESTAMP,
+    COL_NAME_IS_ANONYMOUS,
     COLS_QA,
     QA_BENCHMARK_COLS,
     COLS_LONG_DOC,
@@ -90,7 +91,7 @@ class FullEvalResult:
                 metric=config["metric"],
                 timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
                 revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e"),
-                is_anonymous=config.get("is_anonymous", False)
+                is_anonymous=config.get("is_anonymous", True)
             )
             result_list.append(eval_result)
         return cls(
@@ -124,6 +125,7 @@ class FullEvalResult:
             results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
             results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
             results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
+            results[eval_result.eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
 
             # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
@@ -183,11 +185,12 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     """
     Creates a dataframe from all the individual experiment results
     """
+    cols = [COL_NAME_IS_ANONYMOUS, ]
     if task == "qa":
-        cols = COLS_QA
+        cols += COLS_QA
         benchmark_cols = QA_BENCHMARK_COLS
     elif task == "long-doc":
-        cols = COLS_LONG_DOC
+        cols += COLS_LONG_DOC
         benchmark_cols = LONG_DOC_BENCHMARK_COLS
     else:
         raise NotImplemented
@@ -195,7 +198,7 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     for v in raw_data:
         all_data_json += v.to_dict(task=task, metric=metric)
     df = pd.DataFrame.from_records(all_data_json)
-    print(f'dataframe created: {df.shape}')
+    # print(f'dataframe created: {df.shape}')
 
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
 

src/utils.py

@@ -8,7 +8,7 @@ import pandas as pd
 from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
 from src.display.formatting import styled_message, styled_error
 from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
-    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, get_default_auto_eval_column_dict
+    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, get_default_auto_eval_column_dict
 from src.envs import API, SEARCH_RESULTS_REPO
 from src.read_evals import FullEvalResult, get_leaderboard_df
 
@@ -77,7 +77,7 @@ FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
 
 
 def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, task: str = "qa") -> pd.DataFrame:
-    cols = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
+    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
     selected_cols = []
     for c in cols:
         if task == "qa":
@@ -91,7 +91,7 @@ def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, t
         selected_cols.append(c)
     # We use COLS to maintain sorting
     filtered_df = df[FIXED_COLS + selected_cols]
-    filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].mean(axis=1).round(decimals=2)
+    filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].mean(axis=1, numeric_only=True).round(decimals=2)
     filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
     filtered_df.reset_index(inplace=True, drop=True)
     filtered_df[COL_NAME_RANK] = filtered_df[COL_NAME_AVG].rank(ascending=False, method="min")
@@ -105,8 +105,15 @@ def update_table(
         langs: list,
         reranking_query: list,
         query: str,
+        show_anonymous: bool
 ):
-    filtered_df = filter_models(hidden_df, reranking_query)
+    print(f"shown_anonymous: {show_anonymous}")
+    filtered_df = hidden_df
+    if not show_anonymous:
+        print(filtered_df[COL_NAME_IS_ANONYMOUS])
+        filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
+        print(f"filtered_df: {len(filtered_df)}")
+    filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, domains, langs)
     return df
@@ -118,10 +125,13 @@ def update_table_long_doc(
         langs: list,
         reranking_query: list,
         query: str,
+        # show_anonymous: bool
 ):
     filtered_df = filter_models(hidden_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, domains, langs, task='long_doc')
+    # if not show_anonymous:
+    #     df = df[~df[COL_NAME_IS_ANONYMOUS]]
     return df
 
 
@@ -133,6 +143,7 @@ def update_metric(
         langs: list,
         reranking_model: list,
         query: str,
+        show_anonymous: bool
 ) -> pd.DataFrame:
     if task == 'qa':
         leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
@@ -141,7 +152,8 @@ def update_metric(
             domains,
             langs,
             reranking_model,
-            query
+            query,
+            show_anonymous
         )
     elif task == "long-doc":
         leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
@@ -150,7 +162,8 @@ def update_metric(
             domains,
             langs,
             reranking_model,
-            query
+            query,
+            # show_anonymous
         )