Refactor evaluation logic

evaluation.py

@@ -1,3 +1,4 @@
+import copy
 from dataclasses import dataclass
 
 import streamlit as st
@@ -5,7 +6,7 @@ from huggingface_hub import DatasetFilter, HfApi
 from huggingface_hub.hf_api import DatasetInfo
 
 
-@dataclass(frozen=True, eq=True)
+@dataclass(frozen=True, eq=True, unsafe_hash=True)
 class EvaluationInfo:
     task: str
     model: str
@@ -15,30 +16,29 @@ class EvaluationInfo:
     metrics: set
 
 
-def compute_evaluation_id(dataset_info: DatasetInfo) -> int:
+def create_evaluation_info(dataset_info: DatasetInfo) -> int:
     if dataset_info.cardData is not None:
         metadata = dataset_info.cardData["eval_info"]
         metadata.pop("col_mapping", None)
         # TODO(lewtun): populate dataset cards with metric info
         if "metrics" not in metadata:
             metadata["metrics"] = frozenset()
-        metadata["metrics"] = frozenset(metadata["metrics"])
-        evaluation_info = EvaluationInfo(**metadata)
-        return hash(evaluation_info)
-    else:
-        return None
+        else:
+            metadata["metrics"] = frozenset(metadata["metrics"])
+        return EvaluationInfo(**metadata)
 
 
-def get_evaluation_ids():
+def get_evaluation_infos():
     filt = DatasetFilter(author="autoevaluate")
     evaluation_datasets = HfApi().list_datasets(filter=filt, full=True)
-    return [compute_evaluation_id(dset) for dset in evaluation_datasets]
+    return [create_evaluation_info(dset) for dset in evaluation_datasets]
 
 
 def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_split, metrics):
-    evaluation_ids = get_evaluation_ids()
+    evaluation_infos = get_evaluation_infos()
+    models_to_filter = copy.copy(models)
 
-    for idx, model in enumerate(models):
+    for model in models_to_filter:
         evaluation_info = EvaluationInfo(
             task=task,
             model=model,
@@ -47,12 +47,11 @@ def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_
             dataset_split=dataset_split,
             metrics=frozenset(metrics),
         )
-        candidate_id = hash(evaluation_info)
-        if candidate_id in evaluation_ids:
+        if evaluation_info in evaluation_infos:
             st.info(
-                f"Model `{model}` has already been evaluated on this configuration. \
+                f"Model [`{model}`](https://huggingface.co/{model}) has already been evaluated on this configuration. \
                     This model will be excluded from the evaluation job..."
             )
-            models.pop(idx)
+            models.remove(model)
 
     return models