update

src/display/utils.py

@@ -40,7 +40,7 @@ class Tasks(Enum):
     cnndm_b = Task("cnndm_v2", "bertscore_precision", "CNN-DM/BERT-P")
 
     race = Task("race", "acc", "RACE/Acc")
-    squadv2 = Task("squadv2", "exact", "SQUaDv2/EM")
+    # squadv2 = Task("squadv2", "exact_normalised", "SQUaDv2/EM")
 
     memotrap = Task("memo-trap_v2", "acc", "MemoTrap/Acc")
     ifeval = Task("ifeval", "prompt_level_strict_acc", "IFEval/Acc")

src/leaderboard/read_evals.py

@@ -11,6 +11,14 @@ from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, Weigh
 from src.submission.check_validity import is_model_on_hub
 
 
+def is_float(string):
+    try:
+        float(string)
+        return True
+    except ValueError:
+        return False
+
+
 @dataclass
 class EvalResult:
     # Also see src.display.utils.AutoEvalColumn for what will be displayed.
@@ -84,6 +92,12 @@ class EvalResult:
                     for k, v in entry_copy.items():
                         if "exact_match" in k:
                             results[task_name][k.replace("exact_match", "em")] = v
+                        if "squadv2" in task_name:
+                            value = results[task_name][k]
+                            if is_float(value) and 'normalised' not in k:
+                                results[task_name][f"{k}_normalised"] = value / 100.0
+                            else:
+                                del results[task_name][k]
 
                     entry_copy = results[task_name].copy()
 
@@ -99,6 +113,8 @@ class EvalResult:
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
+            # print(accs)
+
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc