Update space

app.py

@@ -25,7 +25,7 @@ from src.display.utils import (
     Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_model_leaderboard_df
 from src.submission.submit import add_new_eval
 
 
@@ -97,6 +97,7 @@ def init_leaderboard(dataframe):
     )
 
 
+model_leaderboard_df = get_model_leaderboard_df()
 
 def overall_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -124,7 +125,7 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
 
         with gr.TabItem("🏅 Overview", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = overall_leaderboard(LEADERBOARD_DF)
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
             
 
         with gr.TabItem("🎯 Overall", elem_id="llm-benchmark-tab-table", id=1):

src/about.py

@@ -1,6 +1,21 @@
 from dataclasses import dataclass
 from enum import Enum
 
+@dataclass
+class Domain:
+    dimension: str
+    metric: str
+    col_name: str
+    
+    
+class Domains(Enum):
+    # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    dim0 = Domain("overall", "Avg Rank", "Overall")
+    # dim1 = Task("math", "acc", "Math")
+    # dim2 = Task("math_algebra", "acc", "Algebra")
+    
+    
+
 @dataclass
 class Task:
     benchmark: str
@@ -14,7 +29,7 @@ class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
-
+    
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -1,9 +1,9 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, Domains
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -22,26 +22,67 @@ class ColumnContent:
 
 ## Leaderboard columns
 auto_eval_column_dict = []
+# # Init
+# auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+
+# # new columns
+# for domain in Domains:
+#     auto_eval_column_dict.append([domain.name, ColumnContent, ColumnContent(domain.value.col_name, "number", True)])
+
+# auto_eval_column_dict.append(["organization", ColumnContent, ColumnContent("Organization", "str", False)])
+# auto_eval_column_dict.append(["knowledge_cutoff", ColumnContent, ColumnContent("Knowledge cutoff", "str", False)])
+
+
+# for task in Tasks:
+#     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, field(default_factory=lambda: ColumnContent("T", "str", True, never_hidden=True))])
+# #Scores
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# # Model information
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["model", ColumnContent, field(default_factory=lambda: ColumnContent("Model", "markdown", True, never_hidden=True))])
+auto_eval_column_dict.append(["license", ColumnContent, field(default_factory=lambda: ColumnContent("Hub License", "str", False))])
+
+# new columns
+for domain in Domains:
+    auto_eval_column_dict.append([domain.name, ColumnContent, field(default_factory=lambda: ColumnContent(domain.value.col_name, "number", True))])
+
+auto_eval_column_dict.append(["organization", ColumnContent, field(default_factory=lambda: ColumnContent("Organization", "str", False))])
+auto_eval_column_dict.append(["knowledge_cutoff", ColumnContent, field(default_factory=lambda: ColumnContent("Knowledge cutoff", "str", False))])
+
+
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda: ColumnContent(task.value.col_name, "number", True))])
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, field(default_factory=lambda: ColumnContent("T", "str", True, never_hidden=True))])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, field(default_factory=lambda: ColumnContent("Average ⬆️", "number", True))])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["model_type", ColumnContent, field(default_factory=lambda: ColumnContent("Type", "str", False))])
+auto_eval_column_dict.append(["architecture", ColumnContent, field(default_factory=lambda: ColumnContent("Architecture", "str", False))])
+auto_eval_column_dict.append(["weight_type", ColumnContent, field(default_factory=lambda: ColumnContent("Weight type", "str", False, True))])
+auto_eval_column_dict.append(["precision", ColumnContent, field(default_factory=lambda: ColumnContent("Precision", "str", False))])
+auto_eval_column_dict.append(["params", ColumnContent, field(default_factory=lambda: ColumnContent("#Params (B)", "number", False))])
+auto_eval_column_dict.append(["likes", ColumnContent, field(default_factory=lambda: ColumnContent("Hub ❤️", "number", False))])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, field(default_factory=lambda: ColumnContent("Available on the hub", "bool", False))])
+auto_eval_column_dict.append(["revision", ColumnContent, field(default_factory=lambda: ColumnContent("Model sha", "str", False, False))])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+# print all attributes of AutoEvalColumn
+print(AutoEvalColumn.__annotations__.keys())
+# preint precision attribute
+print(AutoEvalColumn.precision)
+
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, Domains
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -18,34 +18,57 @@ class ModelResult:
     """
     eval_name: str
     full_model: str
+    org: str 
+    model: str
+    results: dict
+    license: str = "?"
+    knowledge_cutoff: str = ""
     
     @classmethod
-    def init_from_jsonl_file(self, json_filepath):
-        try:
-            with open(json_filepath) as fp:
-                data = json.load(fp)
-        except:
-            data = eval(open(json_filepath).read()) # a list of dicts   
-        
+    def init_from_json_dict(self, data):
         
+        config = data.get("config")
+        # Get model and org
+        model = config.get("model_name")
+        org = config.get("org_name")
+        license = config.get("license")
+        knowledge_cutoff = config.get("knowledge_cutoff")
+                
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for domain in Domains:
+            domain = domain.value
+            results[domain.dimension] = data.get("results").get(domain.metric)
         
-
-        return
+        return self(
+            eval_name=f"{org}_{model}",
+            full_model=f"{org}/{model}",
+            org=org,
+            model=model,
+            results=results,
+            license=license,
+            knowledge_cutoff=knowledge_cutoff
+        )
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        
+        average = 1 / self.results[Domains.dim0.dimension] if self.results[Domains.dim0.dimension] != 0 else 0
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.organization.name: self.org,
+            AutoEvalColumn.knowledge_cutoff.name: self.knowledge_cutoff,
+
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
@@ -54,6 +77,9 @@ class ModelResult:
         for task in Tasks:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
+        for domain in Domains:
+            data_dict[domain.value.col_name] = self.results[domain.value.dimension]
+
         return data_dict
 
 @dataclass
@@ -154,6 +180,7 @@ class EvalResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        print(AutoEvalColumn.precision.name, self.precision.value.name)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -242,14 +269,19 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
 def get_raw_model_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = results_path
+
+    try:
+        with open(results_path) as fp:
+            data = json.load(fp)
+    except:
+        data = eval(open(results_path).read()) # a list of dicts   
 
     eval_results = {}
-    
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
 
+    for result in data:
+        # Creation of result
+        eval_result = ModelResult.init_from_json_dict(result)
+        
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():

src/populate.py

@@ -5,12 +5,28 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.leaderboard.read_evals import get_raw_eval_results, get_raw_model_results
 
 
+def get_model_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_model_results(results_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    # df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+    
+
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
+    # raw_data = get_raw_model_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)