simplified some parts of the code + updated requirements

app.py

@@ -22,6 +22,8 @@ from src.display.utils import (
     AutoEvalColumn,
     ModelType,
     fields,
+    WeightType,
+    Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -37,25 +39,25 @@ from src.tools.plots import (
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
-
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
+if False:
+    try:
+        print(EVAL_REQUESTS_PATH)
+        snapshot_download(
+            repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+        )
+    except Exception:
+        restart_space()
+    try:
+        print(EVAL_RESULTS_PATH)
+        snapshot_download(
+            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+        )
+    except Exception:
+        restart_space()
 
 
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-update_collections(original_df.copy())
+# update_collections(original_df.copy())
 leaderboard_df = original_df.copy()
 
 plot_df = create_plot_df(create_scores_df(raw_data))
@@ -186,8 +188,8 @@ with demo:
                     )
                     filter_columns_precision = gr.CheckboxGroup(
                         label="Precision",
-                        choices=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
-                        value=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                        choices=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision],
                         interactive=True,
                         elem_id="filter-columns-precision",
                     )
@@ -317,7 +319,7 @@ with demo:
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType],
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
                         multiselect=False,
                         value=None,
@@ -326,14 +328,14 @@ with demo:
 
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=["float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ"],
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
                         label="Precision",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
-                        choices=["Original", "Delta", "Adapter"],
+                        choices=[i.value.name for i in WeightType],
                         label="Weights type",
                         multiselect=False,
                         value="Original",

requirements.txt

@@ -15,6 +15,5 @@ python-dateutil==2.8.2
 requests==2.28.2
 semantic-version==2.10.0
 tqdm==4.65.0
-git+https://github.com/clefourrier/transformers.git@req-fix#egg=transformers
-#transformers==4.35.1
+transformers==4.35.2
 tokenizers>=0.15.0

src/display/formatting.py

@@ -7,23 +7,6 @@ from huggingface_hub.hf_api import ModelInfo
 
 API = HfApi()
 
-LLAMAS = [
-    "huggingface/llama-7b",
-    "huggingface/llama-13b",
-    "huggingface/llama-30b",
-    "huggingface/llama-65b",
-]
-
-KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
-VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
-OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
-DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
-MODEL_PAGE = "https://huggingface.co/models"
-LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
-VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
-ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
-
-
 def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
@@ -31,44 +14,9 @@ def model_hyperlink(link, model_name):
 def make_clickable_model(model_name):
     link = f"https://huggingface.co/{model_name}"
 
-    if model_name in LLAMAS:
-        link = LLAMA_LINK
-        model_name = model_name.split("/")[1]
-    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
-        link = VICUNA_LINK
-        model_name = "stable-vicuna-13b"
-    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
-        link = ALPACA_LINK
-        model_name = "alpaca-13b"
-    if model_name == "dolly-12b":
-        link = DOLLY_LINK
-    elif model_name == "vicuna-13b":
-        link = VICUNA_LINK
-    elif model_name == "koala-13b":
-        link = KOALA_LINK
-    elif model_name == "oasst-12b":
-        link = OASST_LINK
-
     details_model_name = model_name.replace("/", "__")
     details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
 
-    if not bool(os.getenv("DEBUG", "False")):
-        # We only add these checks when not debugging, as they are extremely slow
-        print(f"details_link: {details_link}")
-        try:
-            check_path = list(
-                API.list_files_info(
-                    repo_id=f"open-llm-leaderboard/details_{details_model_name}",
-                    paths="README.md",
-                    repo_type="dataset",
-                )
-            )
-            print(f"check_path: {check_path}")
-        except Exception as err:
-            # No details repo for this model
-            print(f"No details repo for this model: {err}")
-            return model_hyperlink(link, model_name)
-
     return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
 
 

src/display/utils.py

@@ -1,8 +1,26 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
 import pandas as pd
 
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class Tasks(Enum):
+    arc = Task("arc:challenge", "acc_norm", "ARC")
+    hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
+    mmlu = Task("hendrycksTest", "acc", "MMLU")
+    truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
+    winogrande = Task("winogrande", "acc", "Winogrande")
+    gsm8k = Task("gsm8k", "acc", "GSM8K")
+    drop = Task("drop", "f1", "DROP")
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
@@ -16,36 +34,29 @@ class ColumnContent:
     never_hidden: bool = False
     dummy: bool = False
 
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:  # Auto evals column
-    model_type_symbol = ColumnContent("T", "str", True, never_hidden=True)
-    model = ColumnContent("Model", "markdown", True, never_hidden=True)
-    average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("ARC", "number", True)
-    hellaswag = ColumnContent("HellaSwag", "number", True)
-    mmlu = ColumnContent("MMLU", "number", True)
-    truthfulqa = ColumnContent("TruthfulQA", "number", True)
-    winogrande = ColumnContent("Winogrande", "number", True)
-    gsm8k = ColumnContent("GSM8K", "number", True)
-    drop = ColumnContent("DROP", "number", True)
-    model_type = ColumnContent("Type", "str", False)
-    architecture = ColumnContent("Architecture", "str", False)
-    weight_type = ColumnContent("Weight type", "str", False, True)
-    precision = ColumnContent("Precision", "str", False)  # , True)
-    license = ColumnContent("Hub License", "str", False)
-    params = ColumnContent("#Params (B)", "number", False)
-    likes = ColumnContent("Hub ❤️", "number", False)
-    still_on_hub = ColumnContent("Available on the hub", "bool", False)
-    revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent(
-        "model_name_for_query", "str", False, dummy=True
-    )  # dummy col to implement search bar (hidden by custom CSS)
-
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -99,17 +110,17 @@ human_baseline_row = {
 }
 
 @dataclass
-class ModelTypeDetails:
+class ModelDetails:
     name: str
-    symbol: str  # emoji
+    symbol: str = "" # emoji, only for the model type
 
 
 class ModelType(Enum):
-    PT = ModelTypeDetails(name="pretrained", symbol="🟢")
-    FT = ModelTypeDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelTypeDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelTypeDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelTypeDetails(name="", symbol="?")
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"
@@ -126,22 +137,33 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    qt_8bit = ModelDetails("8bit")
+    qt_4bit = ModelDetails("4bit")
+    qt_GPTQ = ModelDetails("GPTQ")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        if precision in ["8bit"]:
+            return Precision.qt_8bit
+        if precision in ["4bit"]:
+            return Precision.qt_4bit
+        if precision in ["GPTQ", "None"]:
+            return Precision.qt_GPTQ
+        return Precision.Unknown
+        
 
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-class Tasks(Enum):
-    arc = Task("arc:challenge", "acc_norm", AutoEvalColumn.arc.name)
-    hellaswag = Task("hellaswag", "acc_norm", AutoEvalColumn.hellaswag.name)
-    mmlu = Task("hendrycksTest", "acc", AutoEvalColumn.mmlu.name)
-    truthfulqa = Task("truthfulqa:mc", "mc2", AutoEvalColumn.truthfulqa.name)
-    winogrande = Task("winogrande", "acc", AutoEvalColumn.winogrande.name)
-    gsm8k = Task("gsm8k", "acc", AutoEvalColumn.gsm8k.name)
-    drop = Task("drop", "f1", AutoEvalColumn.drop.name)
 
 
 # Column selection

src/leaderboard/read_evals.py

@@ -10,7 +10,7 @@ from transformers import AutoConfig
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -23,9 +23,9 @@ class EvalResult:
     model: str
     revision: str # commit hash, "" if main
     results: dict
-    precision: str = ""
+    precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: str = "Original" # Original or Adapter
+    weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" # From config file
     license: str = "?"
     likes: int = 0
@@ -43,9 +43,7 @@ class EvalResult:
         config = data.get("config", data.get("config_general", None))
 
         # Precision
-        precision = config.get("model_dtype")
-        if precision == "None":
-            precision = "GPTQ"
+        precision = Precision.from_str(config.get("model_dtype"))
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
@@ -54,15 +52,15 @@ class EvalResult:
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision}"
+            result_key = f"{model}_{precision.value.name}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision}"
+            result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, error, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
         architecture = "?"
         if model_config is not None:
@@ -112,13 +110,13 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision)
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = request.get("weight_type", "?")
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
@@ -131,10 +129,10 @@ class EvalResult:
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision,
+            AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
@@ -167,7 +165,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
             if (
-                req_content["status"] in ["FINISHED", "PENDING_NEW_EVAL"]
+                req_content["status"] in ["FINISHED"]
                 and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file

src/submission/check_validity.py

@@ -87,8 +87,7 @@ def get_model_size(model_info: ModelInfo, precision: str):
 def get_model_arch(model_info: ModelInfo):
     return model_info.config.get("architectures", "Unknown")
 
-def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period, rate_limit_quota):
-    org_or_user, _ = submission_name.split("/")
+def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
     if org_or_user not in users_to_submission_dates:
         return True, ""
     submission_dates = sorted(users_to_submission_dates[org_or_user])

src/submission/submit.py

@@ -30,6 +30,11 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
 
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -38,11 +43,12 @@ def add_new_eval(
         return styled_error("Please select a model type.")
 
     # Is the user rate limited?
-    user_can_submit, error_msg = user_submission_permission(
-        model, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
-    )
-    if not user_can_submit:
-        return styled_error(error_msg)
+    if user_name != "":
+        user_can_submit, error_msg = user_submission_permission(
+            user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+        )
+        if not user_can_submit:
+            return styled_error(error_msg)
 
     # Did the model authors forbid its submission to the leaderboard?
     if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
@@ -99,12 +105,6 @@ def add_new_eval(
         "license": license,
     }
 
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
     # Check for duplicate submission
     if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")