feat: add rank and language dropdown lists

app.py

@@ -12,7 +12,7 @@ from src.display.css_html_js import custom_css
 from src.leaderboard.read_evals import get_raw_eval_results, get_leaderboard_df
 
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
-from utils import update_table, update_metric, update_table_long_doc, upload_file
+from utils import update_table, update_metric, update_table_long_doc, upload_file, get_default_cols
 from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, metric_list
 
 
@@ -39,7 +39,12 @@ print(f'QA data loaded: {original_df_qa.shape}')
 print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
 
 leaderboard_df_qa = original_df_qa.copy()
+shown_columns_qa = get_default_cols('qa', leaderboard_df_qa.columns, add_fix_cols=True)
+leaderboard_df_qa = leaderboard_df_qa[shown_columns_qa]
+
 leaderboard_df_long_doc = original_df_long_doc.copy()
+shown_columns_long_doc = get_default_cols('long_doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
+leaderboard_df_long_doc = leaderboard_df_long_doc[shown_columns_long_doc]
 
 
 def update_metric_qa(
@@ -97,11 +102,12 @@ with demo:
                         )
                     # select language
                     with gr.Row():
-                        selected_langs = gr.CheckboxGroup(
+                        selected_langs = gr.Dropdown(
                             choices=LANG_COLS_QA,
                             value=LANG_COLS_QA,
                             label="Select the languages",
                             elem_id="language-column-select",
+                            multiselect=True,
                             interactive=True
                         )
                     # select reranking model
@@ -117,8 +123,6 @@ with demo:
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df_qa,
-                # headers=shown_columns,
-                # datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
@@ -205,11 +209,12 @@ with demo:
                         )
                     # select language
                     with gr.Row():
-                        selected_langs = gr.CheckboxGroup(
+                        selected_langs = gr.Dropdown(
                             choices=LANG_COLS_LONG_DOC,
                             value=LANG_COLS_LONG_DOC,
                             label="Select the languages",
                             elem_id="language-column-select-long-doc",
+                            multiselect=True,
                             interactive=True
                         )
                     # select reranking model
@@ -225,8 +230,6 @@ with demo:
 
             leaderboard_table_long_doc = gr.components.Dataframe(
                 value=leaderboard_df_long_doc,
-                # headers=shown_columns,
-                # datatype=TYPES,
                 elem_id="leaderboard-table-long-doc",
                 interactive=False,
                 visible=True,
@@ -235,8 +238,6 @@ with demo:
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
                 value=leaderboard_df_long_doc,
-                # headers=COLS,
-                # datatype=TYPES,
                 visible=False,
             )
 
@@ -293,7 +294,10 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
-                            ['AIR-Bench_24.04',], value=['AIR-Bench_24.04',], interactive=True, label="AIR-Bench Version")
+                            ["AIR-Bench_24.04",],
+                            value="AIR-Bench_24.04",
+                            interactive=True,
+                            label="AIR-Bench Version")
                     with gr.Column():
                         model_name_textbox = gr.Textbox(label="Model name")
                     with gr.Column():

src/display/utils.py

@@ -22,6 +22,7 @@ class ColumnContent:
 COL_NAME_AVG = "Average ⬆️"
 COL_NAME_RETRIEVAL_MODEL = "Retrieval Model"
 COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RANK = "Rank 🏆"
 
 def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     ## Leaderboard columns
@@ -36,6 +37,9 @@ def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     auto_eval_column_dict.append(
         ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
     )
+    auto_eval_column_dict.append(
+        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
+    )
     for benchmark in benchmarks:
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]

src/leaderboard/read_evals.py

@@ -9,8 +9,16 @@ import pandas as pd
 
 from src.benchmarks import get_safe_name
 from src.display.formatting import has_no_nan_values
-from src.display.utils import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COLS_QA, QA_BENCHMARK_COLS, \
-    COLS_LONG_DOC, LONG_DOC_BENCHMARK_COLS, COL_NAME_AVG
+from src.display.utils import (
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COLS_QA,
+    QA_BENCHMARK_COLS,
+    COLS_LONG_DOC,
+    LONG_DOC_BENCHMARK_COLS,
+    COL_NAME_AVG,
+    COL_NAME_RANK
+)
 
 
 @dataclass
@@ -158,6 +166,7 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     df[COL_NAME_AVG] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
     df = df.sort_values(by=[COL_NAME_AVG], ascending=False)
     df.reset_index(inplace=True)
+    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="dense")
 
     _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
     df = df[_cols].round(decimals=2)

utils.py

@@ -3,7 +3,7 @@ from typing import List
 import pandas as pd
 
 from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
-from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC
+from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
 from src.leaderboard.read_evals import FullEvalResult, get_leaderboard_df
 
 
@@ -37,25 +37,28 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumnQA.retrieval_model.name].str.contains(query, case=False))]
 
 
-def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, task: str = "qa") -> pd.DataFrame:
+def get_default_cols(task: str, columns: list, add_fix_cols: bool=True) -> list:
     if task == "qa":
-        always_here_cols = [
-            AutoEvalColumnQA.retrieval_model.name,
-            AutoEvalColumnQA.reranking_model.name,
-            AutoEvalColumnQA.average.name
-        ]
-        cols = list(frozenset(COLS_QA).intersection(frozenset(BENCHMARK_COLS_QA)))
+        cols = list(frozenset(COLS_QA).intersection(frozenset(BENCHMARK_COLS_QA)).intersection(frozenset(columns)))
     elif task == "long_doc":
-        always_here_cols = [
-            AutoEvalColumnLongDoc.retrieval_model.name,
-            AutoEvalColumnLongDoc.reranking_model.name,
-            AutoEvalColumnLongDoc.average.name
-        ]
-        cols = list(frozenset(COLS_LONG_DOC).intersection(frozenset(BENCHMARK_COLS_LONG_DOC)))
+        cols = list(frozenset(COLS_LONG_DOC).intersection(frozenset(BENCHMARK_COLS_LONG_DOC)).intersection(frozenset(columns)))
+    else:
+        raise NotImplemented
+    if add_fix_cols:
+        cols = FIXED_COLS + cols
+    return cols
+
+FIXED_COLS = [
+        COL_NAME_RANK,
+        COL_NAME_RETRIEVAL_MODEL,
+        COL_NAME_RERANKING_MODEL,
+        COL_NAME_AVG,
+    ]
+
+def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, task: str = "qa") -> pd.DataFrame:
+    cols = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
     selected_cols = []
     for c in cols:
-        if c not in df.columns:
-            continue
         if task == "qa":
             eval_col = BenchmarksQA[c].value
         elif task == "long_doc":
@@ -66,8 +69,10 @@ def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, t
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    filtered_df = df[always_here_cols + selected_cols]
-    filtered_df[always_here_cols[2]] = filtered_df[selected_cols].mean(axis=1).round(decimals=2)
+    filtered_df = df[FIXED_COLS + selected_cols]
+    filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].mean(axis=1).round(decimals=2)
+    filtered_df[COL_NAME_RANK] = filtered_df[COL_NAME_AVG].rank(ascending=False, method="dense")
+
     return filtered_df