update

app.py

@@ -28,8 +28,8 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-from src.submission.check_validity import already_submitted_models
-from src.tools.collections import update_collections
+# from src.submission.check_validity import already_submitted_models
+# from src.tools.collections import update_collections
 from src.tools.plots import (
     create_metric_plot_obj,
     create_plot_df,

beta-cli.py

@@ -10,7 +10,7 @@ snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="
 raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
 
 for entry in raw_data:
-    if '125' in entry.eval_name:
-        print(entry)
+    # if '125m' in entry.eval_name:
+    print(entry)
 
-# print(raw_data)
+# print(raw_data)

src/display/utils.py

@@ -18,9 +18,10 @@ class Tasks(Enum):
     hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
     mmlu = Task("hendrycksTest", "acc", "MMLU")
     truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
-    winogrande = Task("winogrande", "acc", "Winogrande")
-    gsm8k = Task("gsm8k", "acc", "GSM8K")
+    # winogrande = Task("winogrande", "acc", "Winogrande")
+    # gsm8k = Task("gsm8k", "acc", "GSM8K")
     drop = Task("drop", "f1", "DROP")
+    nqopen = Task("nq_open", "em", "NQ Open")
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
@@ -77,8 +78,8 @@ baseline_row = {
     AutoEvalColumn.hellaswag.name: 25.0,
     AutoEvalColumn.mmlu.name: 25.0,
     AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.winogrande.name: 50.0,
-    AutoEvalColumn.gsm8k.name: 0.21,
+    # AutoEvalColumn.winogrande.name: 50.0,
+    # AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.drop.name: 0.47,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
@@ -102,8 +103,8 @@ human_baseline_row = {
     AutoEvalColumn.hellaswag.name: 95.0,
     AutoEvalColumn.mmlu.name: 89.8,
     AutoEvalColumn.truthfulqa.name: 94.0,
-    AutoEvalColumn.winogrande.name: 94.0,
-    AutoEvalColumn.gsm8k.name: 100,
+    # AutoEvalColumn.winogrande.name: 94.0,
+    # AutoEvalColumn.gsm8k.name: 100,
     AutoEvalColumn.drop.name: 96.42,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",

src/leaderboard/read_evals.py

@@ -96,6 +96,10 @@ class EvalResult:
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
 
+        # XXX
+        if 'nq_open' not in results:
+            results['nq_open'] = 0.0
+
         return self(
             eval_name=result_key,
             full_model=full_model,