change metrics, add model params

src/about.py

@@ -12,8 +12,8 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("metric1", "acc", "Metric1")
+    task1 = Task("metric2", "acc_norm", "Metric2")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/leaderboard/read_evals.py

@@ -43,6 +43,8 @@ class EvalResult:
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
+        self.num_params = config.get("model_params", 0)
+
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
         org_and_model = org_and_model.split("/", 1)
@@ -91,21 +93,21 @@ class EvalResult:
             architecture=architecture
         )
 
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+    # def update_with_request_file(self, requests_path):
+    #     """Finds the relevant request file for the current model and updates info with it"""
+    #     request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+    #     try:
+    #         with open(request_file, "r") as f:
+    #             request = json.load(f)
+    #         self.model_type = ModelType.from_str(request.get("model_type", ""))
+    #         self.weight_type = WeightType[request.get("weight_type", "Original")]
+    #         self.license = request.get("license", "?")
+    #         self.likes = request.get("likes", 0)
+    #         self.num_params = request.get("params", 0)
+    #         self.date = request.get("submitted_time", "")
+    #     except Exception:
+    #         print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -132,26 +134,26 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
+# def get_request_file_for_model(requests_path, model_name, precision):
+#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+#     request_files = os.path.join(
+#         requests_path,
+#         f"{model_name}_eval_request_*.json",
+#     )
+#     request_files = glob.glob(request_files)
+
+#     # Select correct request file (precision)
+#     request_file = ""
+#     request_files = sorted(request_files, reverse=True)
+#     for tmp_request_file in request_files:
+#         with open(tmp_request_file, "r") as f:
+#             req_content = json.load(f)
+#             if (
+#                 req_content["status"] in ["FINISHED"]
+#                 and req_content["precision"] == precision.split(".")[-1]
+#             ):
+#                 request_file = tmp_request_file
+#     return request_file
 
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
@@ -177,7 +179,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Creation of result
         print("Parsing", model_result_filepath)
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        # eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name