add metadata and filters

app.py

@@ -173,7 +173,7 @@ with demo:
                         value=[t.to_str() for t in ModelType],
                         interactive=True,
                         elem_id="filter-columns-type",
-                        visible=False,
+                        visible=True,
                     )
                     filter_columns_precision = gr.CheckboxGroup(
                         label="Precision",
@@ -189,7 +189,7 @@ with demo:
                         value=list(NUMERIC_INTERVALS.keys()),
                         interactive=True,
                         elem_id="filter-columns-size",
-                        visible=False,
+                        visible=True,
                     )
                     filter_columns_nshot = gr.CheckboxGroup(
                         label="N-shot",
@@ -238,6 +238,7 @@ with demo:
                 interactive=False,
                 visible=True,
                 # column_widths=["2%", "33%"]
+                height=900
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key

src/about.py

@@ -1,35 +1,36 @@
 from dataclasses import dataclass
 from enum import Enum
 
-@dataclass
+@dataclass(frozen=True)
 class Task:
     benchmark: str
     metric: str
     col_name: str
+    type: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task2 = Task("belebele_pol_Latn", "acc,none", "belebele_pol_Latn")
-    task3 = Task("polemo2_in", "exact_match,score-first", "polemo2-in_g")
-    task4 = Task("polemo2_in_multiple_choice", "acc,none", "polemo2_in_mc")
-    task5 = Task("polemo2_out", "exact_match,score-first", "polemo2_out_g")
-    task6 = Task("polemo2_out_multiple_choice", "acc,none", "polemo2_out_mc")
-    task7 = Task("polish_8tags_multiple_choice", "acc,none", "8tags_mc")
-    task8 = Task("polish_8tags_regex", "exact_match,score-first", "8tags_g")
-    task9 = Task("polish_belebele_regex", "exact_match,score-first", "belebele_g")
-    task10 = Task("polish_dyk_multiple_choice", "f1,none", "dyk_mc")
-    task11 = Task("polish_dyk_regex", "f1,score-first", "dyk_g")
-    task12 = Task("polish_ppc_multiple_choice", "acc,none", "ppc_mc")
-    task13 = Task("polish_ppc_regex", "exact_match,score-first", "ppc_g")
-    task14 = Task("polish_psc_multiple_choice", "f1,none", "psc_mc")
-    task15 = Task("polish_psc_regex", "f1,score-first", "psc_g")
-    task16 = Task("polish_cbd_multiple_choice", "f1,none", "cbd_mc")
-    task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g")
-    task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc")
-    task19 = Task("polish_klej_ner_regex", "exact_match,score-first", "klej_ner_g")
+    task2 = Task("belebele_pol_Latn", "acc,none", "belebele_pol_Latn", "multiple_choice")
+    task3 = Task("polemo2_in", "exact_match,score-first", "polemo2-in_g", "generate_until")
+    task4 = Task("polemo2_in_multiple_choice", "acc,none", "polemo2-in_mc", "multiple_choice")
+    task5 = Task("polemo2_out", "exact_match,score-first", "polemo2-out_g", "generate_until")
+    task6 = Task("polemo2_out_multiple_choice", "acc,none", "polemo2-out_mc", "multiple_choice")
+    task7 = Task("polish_8tags_multiple_choice", "acc,none", "8tags_mc", "multiple_choice")
+    task8 = Task("polish_8tags_regex", "exact_match,score-first", "8tags_g", "generate_until")
+    task9 = Task("polish_belebele_regex", "exact_match,score-first", "belebele_g", "generate_until")
+    task10 = Task("polish_dyk_multiple_choice", "f1,none", "dyk_mc", "multiple_choice")
+    task11 = Task("polish_dyk_regex", "f1,score-first", "dyk_g", "generate_until")
+    task12 = Task("polish_ppc_multiple_choice", "acc,none", "ppc_mc", "multiple_choice")
+    task13 = Task("polish_ppc_regex", "exact_match,score-first", "ppc_g", "generate_until")
+    task14 = Task("polish_psc_multiple_choice", "f1,none", "psc_mc", "multiple_choice")
+    task15 = Task("polish_psc_regex", "f1,score-first", "psc_g", "generate_until")
+    task16 = Task("polish_cbd_multiple_choice", "f1,none", "cbd_mc", "multiple_choice")
+    task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g", "generate_until")
+    task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc", "multiple_choice")
+    task19 = Task("polish_klej_ner_regex", "exact_match,score-first", "klej_ner_g", "generate_until")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -29,15 +29,17 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("n_shot", "str", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average_g", ColumnContent, ColumnContent("Avg g", "number", True)])
+auto_eval_column_dict.append(["average_mc", ColumnContent, ColumnContent("Avg mc", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", True)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
@@ -67,9 +69,9 @@ class ModelDetails:
 
 class ModelType(Enum):
     PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    CPT = ModelDetails(name="continuously pretrained", symbol="🟩")
     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    RL = ModelDetails(name="RL-tuned", symbol="💬")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -77,11 +79,11 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
+        if "continuously pretrained" in type or "🟩" in type:
+            return ModelType.CPT
         if "pretrained" in type or "🟢" in type:
             return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
+        if "RL-tuned" in type or "💬" in type:
             return ModelType.RL
         if "instruction-tuned" in type or "⭕" in type:
             return ModelType.IFT

src/leaderboard/read_evals.py

@@ -106,8 +106,22 @@ class EvalResult:
             n_shot=NShotType.from_str(n_shot_num)
         )
 
+    def update_with_metadata(self, metadata):
+        #print('UPDATE', self.full_model, self.model, self.eval_name)
+        try:
+            meta=metadata[self.full_model]
+            self.model_type = ModelType.from_str(meta.get("type", "?"))
+            self.num_params = meta.get("params", 0)
+            self.license = meta.get("license", "?")
+            # self.lang = meta.get("lang", "?") #TODO
+            #TODO desc name
+        except KeyError:
+            print(f"Could not find metadata for {self.full_model}")
+
+
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
+        return
         request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
 
         try:
@@ -125,7 +139,13 @@ class EvalResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        print('average', average)
+        g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]
+        mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"]
+
+        average_g = sum([v for task,v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
+        average_mc = sum([v for task,v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
+
+
         data_dict={}
         # data_dict = {
         #     "eval_name": self.eval_name,  # not a column, just a save name,
@@ -202,6 +222,16 @@ class EvalResult:
         except KeyError:
             print(f"Could not find average")
 
+        try:
+            data_dict[AutoEvalColumn.average_g.name] = average_g
+        except KeyError:
+            print(f"Could not find average_g")
+
+        try:
+            data_dict[AutoEvalColumn.average_mc.name] = average_mc
+        except KeyError:
+            print(f"Could not find average_mc")
+
         try:
             data_dict[AutoEvalColumn.license.name] = self.license
         except KeyError:
@@ -267,7 +297,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
@@ -291,6 +321,9 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             # Creation of result
             eval_result = EvalResult.init_from_json_file(model_result_filepath, n_shot_num=n_shot)
             eval_result.update_with_request_file(requests_path)
+            #update with metadata
+            eval_result.update_with_metadata(metadata)
+
 
             # Store results of same eval together
             eval_name = f"{eval_result.eval_name}_{n_shot}-shot"

src/populate.py

@@ -9,7 +9,8 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    metadata=json.load(open(f"{requests_path}/metadata.json"))
+    raw_data = get_raw_eval_results(results_path, requests_path, metadata)
     all_data_json = [v.to_dict() for v in raw_data]
     print(all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
@@ -25,27 +26,27 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
+    # for entry in entries:
+    #     if ".json" in entry:
+    #         file_path = os.path.join(save_path, entry)
+    #         with open(file_path) as fp:
+    #             data = json.load(fp)
+    #
+    #         data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+    #         data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+    #
+    #         all_evals.append(data)
+    #     elif ".md" not in entry:
+    #         # this is a folder
+    #         sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+    #         for sub_entry in sub_entries:
+    #             file_path = os.path.join(save_path, entry, sub_entry)
+    #             with open(file_path) as fp:
+    #                 data = json.load(fp)
+    #
+    #             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+    #             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+    #             all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]