feat: add metric selector

app.py

@@ -17,10 +17,12 @@ from src.display.utils import (
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_leaderboard_df
-from utils import update_table
+from utils import update_table, update_metric
 from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, metric_list
 
 
+from functools import partial
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
@@ -41,11 +43,21 @@ def restart_space():
 # except Exception:
 #     restart_space()
 
-raw_data_qa, original_df_qa = get_leaderboard_df(
-    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_3')
+from src.leaderboard.read_evals import get_raw_eval_results
+raw_data_qa = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
+original_df_qa = get_leaderboard_df(raw_data_qa, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_3')
 print(f'data loaded: {len(raw_data_qa)}, {original_df_qa.shape}')
 leaderboard_df = original_df_qa.copy()
 
+
+def update_metric_qa(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+):
+    return update_metric(raw_data_qa, metric, domains, langs, reranking_model, query)
 # (
 #     finished_eval_queue_df,
 #     running_eval_queue_df,
@@ -99,7 +111,7 @@ with demo:
                 with gr.Column(min_width=320):
                     selected_metric = gr.Dropdown(
                         choices=metric_list,
-                        value=metric_list[0],
+                        value=metric_list[1],
                         label="Select the metric",
                         interactive=True,
                         elem_id="metric-select",
@@ -117,11 +129,13 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df_qa,
+                value=leaderboard_df,
                 # headers=COLS,
                 # datatype=TYPES,
                 visible=False,
             )
+
+            # Set search_bar listener
             search_bar.submit(
                 update_table,
                 [
@@ -133,6 +147,8 @@ with demo:
                 ],
                 leaderboard_table,
             )
+
+            # Set column-wise listener
             for selector in [
                 selected_domains, selected_langs, selected_rerankings
             ]:
@@ -149,6 +165,20 @@ with demo:
                     queue=True,
                 )
 
+            # set metric listener
+            selected_metric.change(
+                update_metric_qa,
+                [
+                    selected_metric,
+                    selected_domains,
+                    selected_langs,
+                    selected_rerankings,
+                    search_bar,
+                ],
+                leaderboard_table,
+                queue=True
+            )
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

src/populate.py

@@ -5,31 +5,30 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumnQA, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
-from typing import Tuple
+from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, FullEvalResult
+from typing import Tuple, List
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, task: str, metric: str) -> Tuple[list[EvalResult], pd.DataFrame]:
+def get_leaderboard_df(raw_data: List[FullEvalResult], cols: list, benchmark_cols: list, task: str, metric: str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    print(f"raw_data loaded: {len(raw_data)}")
     all_data_json = []
     for v in raw_data:
         all_data_json += v.to_dict(task=task, metric=metric)
-
-    print(f'records loaded: {len(all_data_json)}')
     df = pd.DataFrame.from_records(all_data_json)
     print(f'dataframe created: {df.shape}')
+
+    # calculate the average score for selected benchmarks
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
-    df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1)
+    df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
     df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
     df.reset_index(inplace=True)
+
     _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
     df = df[_cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, _benchmark_cols)]
-    return raw_data, df
+    return df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:

tests/src/test_populate.py

@@ -1,4 +1,5 @@
 from src.populate import get_leaderboard_df
+from src.leaderboard.read_evals import get_raw_eval_results
 from pathlib import Path
 
 cur_fp = Path(__file__)
@@ -9,7 +10,8 @@ def test_get_leaderboard_df():
     results_path = cur_fp.parents[1] / "toydata" / "test_results"
     cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'wiki_en', 'wiki_zh',]
     benchmark_cols = ['wiki_en', 'wiki_zh',]
-    raw_data, df = get_leaderboard_df(results_path, requests_path, cols, benchmark_cols, 'qa', 'ndcg_at_1')
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    df = get_leaderboard_df(raw_data, cols, benchmark_cols, 'qa', 'ndcg_at_1')
     assert df.shape[0] == 2
     # the results contain only one embedding model
     for i in range(2):

utils.py

@@ -2,6 +2,10 @@ import pandas as pd
 
 from src.display.utils import AutoEvalColumnQA, COLS
 from src.benchmarks import BENCHMARK_COLS_QA, BenchmarksQA
+from src.leaderboard.read_evals import FullEvalResult
+from typing import List
+from src.populate import get_leaderboard_df
+from src.display.utils import COLS, QA_BENCHMARK_COLS
 
 
 def filter_models(df: pd.DataFrame, reranking_query: list) -> pd.DataFrame:
@@ -68,4 +72,23 @@ def update_table(
     filtered_df = filter_models(hidden_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, domains, langs)
-    return df
+    return df
+
+
+def update_metric(
+        raw_data: List[FullEvalResult],
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+) -> pd.DataFrame:
+    leaderboard_df = get_leaderboard_df(raw_data, COLS, QA_BENCHMARK_COLS, task='qa', metric=metric)
+    hidden_df = leaderboard_df
+    return update_table(
+        hidden_df,
+        domains,
+        langs,
+        reranking_model,
+        query
+    )