added 'forbidden models' submission, to allow orgs to request their models to not be submitted in case of contamination

app.py

@@ -10,7 +10,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
 from transformers import AutoConfig
 
-from src.auto_leaderboard.get_model_metadata import apply_metadata
+from src.auto_leaderboard.get_model_metadata import apply_metadata, DO_NOT_SUBMIT_MODELS
 from src.assets.text_content import *
 from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
 from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
@@ -227,9 +227,13 @@ def add_new_eval(
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
 
+    # Check if the model has been forbidden:
+    if out_path.split("eval-queue/")[1] in DO_NOT_SUBMIT_MODELS:
+        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
+
     # Check for duplicate submission
     if out_path.split("eval-queue/")[1].lower() in requested_models:
-        return styled_warning("This model has been already submitted.")
+        return styled_warning("This model has been already submitted.")    
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))

src/auto_leaderboard/get_model_metadata.py

@@ -8,7 +8,7 @@ from tqdm import tqdm
 
 from src.utils_display import AutoEvalColumn, model_hyperlink
 from src.auto_leaderboard.model_metadata_type import ModelType, model_type_from_str, MODEL_TYPE_METADATA
-from src.auto_leaderboard.model_metadata_flags import FLAGGED_MODELS
+from src.auto_leaderboard.model_metadata_flags import FLAGGED_MODELS, DO_NOT_SUBMIT_MODELS
 
 from huggingface_hub import HfApi
 import huggingface_hub
@@ -106,7 +106,18 @@ def flag_models(leaderboard_data:List[dict]):
             issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
             model_data[AutoEvalColumn.model.name] =  f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
 
+def remove_forbidden_models(leaderboard_data: List[dict]):
+    indices_to_remove = []
+    for ix, model in enumerate(leaderboard_data):
+        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+            indices_to_remove.append(ix)
+
+    for ix in reversed(indices_to_remove):
+        leaderboard_data.pop(ix)
+    return leaderboard_data
+
 def apply_metadata(leaderboard_data: List[dict]):
+    leaderboard_data = remove_forbidden_models(leaderboard_data)
     get_model_type(leaderboard_data)
     get_model_infos_from_hub(leaderboard_data)
     flag_models(leaderboard_data)

src/auto_leaderboard/model_metadata_flags.py

@@ -1,6 +1,12 @@
-# Model name to forum discussion id
+# Models which have been flagged by users as being problematic for a reason or another
+# (Model name to forum discussion link)
 FLAGGED_MODELS = {
     "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
     "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
     "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
-}
+}
+
+# Models which have been requested by orgs to not be submitted on the leaderboard
+DO_NOT_SUBMIT_MODELS = [
+    "Voicelab/trurl-2-13b", # trained on MMLU
+]