Spaces:
Running
Running
Tony Wu
commited on
Commit
•
bc3b144
1
Parent(s):
f0be090
refactor: simplify handling of the new vidore result format
Browse files- data/model_handler.py +3 -14
data/model_handler.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
import json
|
2 |
import os
|
3 |
-
from typing import Any, Dict
|
4 |
|
5 |
import pandas as pd
|
6 |
from huggingface_hub import HfApi, hf_hub_download, metadata_load
|
@@ -29,18 +29,6 @@ class ModelHandler:
|
|
29 |
def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
|
30 |
return "metadata" in results and "metrics" in results
|
31 |
|
32 |
-
@staticmethod
|
33 |
-
def convert_new_vidore_results_format(results: Dict[str, Any]) -> Tuple[str, str, Dict[str, Any]]:
|
34 |
-
if "metadata" not in results:
|
35 |
-
raise KeyError("results does not contain a 'metadata' key.")
|
36 |
-
if "metrics" not in results:
|
37 |
-
raise KeyError("results does not contain a 'metrics' key.")
|
38 |
-
|
39 |
-
metadata = results["metadata"]
|
40 |
-
metrics = results["metrics"]
|
41 |
-
|
42 |
-
return metadata["timestamp"], metadata["vidore_benchmark_version"], metrics
|
43 |
-
|
44 |
def get_vidore_data(self, metric="ndcg_at_5"):
|
45 |
models = self.api.list_models(filter="vidore")
|
46 |
repositories = [model.modelId for model in models] # type: ignore
|
@@ -71,7 +59,8 @@ class ModelHandler:
|
|
71 |
results = json.load(f)
|
72 |
|
73 |
if self._are_results_in_new_vidore_format(results):
|
74 |
-
|
|
|
75 |
|
76 |
self.model_infos[model_name] = {"meta": meta, "results": results}
|
77 |
except Exception as e:
|
|
|
1 |
import json
|
2 |
import os
|
3 |
+
from typing import Any, Dict
|
4 |
|
5 |
import pandas as pd
|
6 |
from huggingface_hub import HfApi, hf_hub_download, metadata_load
|
|
|
29 |
def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
|
30 |
return "metadata" in results and "metrics" in results
|
31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
def get_vidore_data(self, metric="ndcg_at_5"):
|
33 |
models = self.api.list_models(filter="vidore")
|
34 |
repositories = [model.modelId for model in models] # type: ignore
|
|
|
59 |
results = json.load(f)
|
60 |
|
61 |
if self._are_results_in_new_vidore_format(results):
|
62 |
+
metadata = results["metadata"]
|
63 |
+
results = results["metrics"]
|
64 |
|
65 |
self.model_infos[model_name] = {"meta": meta, "results": results}
|
66 |
except Exception as e:
|