updates

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

app.py

@@ -2,23 +2,31 @@ import json
 import os
 from datetime import datetime, timezone
 
-
 import gradio as gr
-import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
-from transformers import AutoConfig
-
-from src.auto_leaderboard.get_model_metadata import apply_metadata, DO_NOT_SUBMIT_MODELS
-from src.assets.text_content import *
-from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
-from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
 from src.assets.css_html_js import custom_css, get_window_url_params
-from src.utils_display import AutoEvalColumn, EvalQueueColumn, fields, styled_error, styled_warning, styled_message
-from src.init import get_all_requested_models, load_all_info_from_hub
-
-pd.set_option('display.precision', 1)
+from src.assets.text_content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+from src.display_models.get_model_metadata import DO_NOT_SUBMIT_MODELS, ModelType
+from src.display_models.utils import (
+    AutoEvalColumn,
+    EvalQueueColumn,
+    fields,
+    styled_error,
+    styled_message,
+    styled_warning,
+)
+from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+
+pd.set_option("display.precision", 1)
 
 # clone / pull the lmeh eval data
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
@@ -37,20 +45,16 @@ EVAL_RESULTS_PATH = "eval-results"
 EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
 EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
 
-api = HfApi()
+api = HfApi(token=H4_TOKEN)
+
 
 def restart_space():
     api.restart_space(
         repo_id="gsaivinay/open_llm_leaderboard", token=H4_TOKEN
     )
 
-eval_queue, requested_models, eval_results = load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH)
-
-if not IS_PUBLIC:
-    eval_queue_private, requested_models_private, eval_results_private = load_all_info_from_hub(PRIVATE_QUEUE_REPO, PRIVATE_RESULTS_REPO, EVAL_REQUESTS_PATH_PRIVATE, EVAL_RESULTS_PATH_PRIVATE)
-else:
-    eval_queue_private, eval_results_private = None, None
 
+# Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
@@ -63,116 +67,41 @@ if not IS_PUBLIC:
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)
-
-
-def get_leaderboard_df():
-    if eval_results:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results.git_pull()
-    if eval_results_private:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results_private.git_pull()
-
-    all_data = get_eval_results_dicts()
-
-    # if not IS_PUBLIC:
-    all_data.append(gpt4_values)
-    all_data.append(gpt35_values)
-
-    all_data.append(baseline)
-    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
-
-    df = pd.DataFrame.from_records(all_data)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[COLS].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
-    return df
+BENCHMARK_COLS = [
+    c.name
+    for c in [
+        AutoEvalColumn.arc,
+        AutoEvalColumn.hellaswag,
+        AutoEvalColumn.mmlu,
+        AutoEvalColumn.truthfulqa,
+    ]
+]
 
+## LOAD INFO FROM HUB
+eval_queue, requested_models, eval_results = load_all_info_from_hub(
+    QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
+)
 
-def get_evaluation_queue_df():
-    if eval_queue:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue.git_pull()
-    if eval_queue_private:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue_private.git_pull()
+if not IS_PUBLIC:
+    (eval_queue_private, requested_models_private, eval_results_private,) = load_all_info_from_hub(
+        PRIVATE_QUEUE_REPO,
+        PRIVATE_RESULTS_REPO,
+        EVAL_REQUESTS_PATH_PRIVATE,
+        EVAL_RESULTS_PATH_PRIVATE,
+    )
+else:
+    eval_queue_private, eval_results_private = None, None
 
-    entries = [
-        entry
-        for entry in os.listdir(EVAL_REQUESTS_PATH)
-        if not entry.startswith(".")
-    ]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data["# params"] = "unknown"
-            data["model"] = make_clickable_model(data["model"])
-            data["revision"] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [
-                e
-                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
-                if not e.startswith(".")
-            ]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                # data["# params"] = get_n_params(data["model"])
-                data["model"] = make_clickable_model(data["model"])
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)
-    df_running = pd.DataFrame.from_records(running_list, columns=EVAL_COLS)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=EVAL_COLS)
-    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
-
-
-
-original_df = get_leaderboard_df()
+original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df()
-
-def is_model_on_hub(model_name, revision) -> bool:
-    try:
-        AutoConfig.from_pretrained(model_name, revision=revision)
-        return True, None
-    
-    except ValueError as e:
-        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
-
-    except Exception as e:
-        print(f"Could not get the model config from the hub.: {e}")
-        return False, "was not found on hub!"
+) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
+## INTERACTION FUNCTIONS
 def add_new_eval(
     model: str,
     base_model: str,
@@ -196,13 +125,12 @@ def add_new_eval(
         base_model_on_hub, error = is_model_on_hub(base_model, revision)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
-        
 
     if not weight_type == "Adapter":
         model_on_hub, error = is_model_on_hub(model, revision)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
-    
+
     print("adding new eval")
 
     eval_entry = {
@@ -233,7 +161,7 @@ def add_new_eval(
 
     # Check for duplicate submission
     if out_path.split("eval-queue/")[1].lower() in requested_models:
-        return styled_warning("This model has been already submitted.")    
+        return styled_warning("This model has been already submitted.")
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -242,7 +170,6 @@ def add_new_eval(
         path_or_fileobj=out_path,
         path_in_repo=out_path.split("eval-queue/")[1],
         repo_id=QUEUE_REPO,
-        token=H4_TOKEN,
         repo_type="dataset",
         commit_message=f"Add {model} to eval queue",
     )
@@ -250,16 +177,19 @@ def add_new_eval(
     # remove the local file
     os.remove(out_path)
 
-    return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list.")
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )
 
 
-def refresh():
-    leaderboard_df = get_leaderboard_df()
+# Basics
+def refresh() -> list[pd.DataFrame]:
+    leaderboard_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
     (
         finished_eval_queue_df,
         running_eval_queue_df,
         pending_eval_queue_df,
-    ) = get_evaluation_queue_df()
+    ) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, COLS)
     return (
         leaderboard_df,
         finished_eval_queue_df,
@@ -268,74 +198,72 @@ def refresh():
     )
 
 
-def search_table(df, leaderboard_table, query):
-    if AutoEvalColumn.model_type.name in leaderboard_table.columns:
+def change_tab(query_param: str):
+    query_param = query_param.replace("'", '"')
+    query_param = json.loads(query_param)
+
+    if isinstance(query_param, dict) and "tab" in query_param and query_param["tab"] == "evaluation":
+        return gr.Tabs.update(selected=1)
+    else:
+        return gr.Tabs.update(selected=0)
+
+
+# Searching and filtering
+def search_table(df: pd.DataFrame, current_columns_df: pd.DataFrame, query: str) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+    if AutoEvalColumn.model_type.name in current_columns:
         filtered_df = df[
             (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))
             | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))
-            ]
+        ]
     else:
         filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-    return filtered_df[leaderboard_table.columns]
+    return filtered_df[current_columns]
 
 
-def select_columns(df, columns):
-    always_here_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
-    # We use COLS to maintain sorting 
-    filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]]
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+    ]
     return filtered_df
 
-#TODO allow this to filter by values of any columns
-def filter_items(df, leaderboard_table, query):
-    if query == "all":
-        return df[leaderboard_table.columns]
-    else:
-        query = query[0] #take only the emoji character
-    if AutoEvalColumn.model_type_symbol.name in leaderboard_table.columns:
-        filtered_df = df[(df[AutoEvalColumn.model_type_symbol.name] == query)]
-    else:
-        return filtered_df[leaderboard_table.columns]
-    return filtered_df[leaderboard_table.columns]
-
-def filter_items_size(df, leaderboard_table, query):
-    numeric_intervals = {
-        "all": None,
-        "< 1B": (0, 1),
-        "~3B": (1, 5),
-        "~7B": (6, 11),
-        "~13B": (12, 15),
-        "~35B": (16, 55),
-        "60B+": (55, 1000)
-    }
-
-    if query == "all":
-        return df[leaderboard_table.columns]
-
-    numeric_interval = numeric_intervals[query]
-
-    if AutoEvalColumn.params.name in leaderboard_table.columns:
-        params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors='coerce')
-        filtered_df = df[params_column.between(*numeric_interval)]
-    else:
-        return filtered_df[leaderboard_table.columns]
-    return filtered_df[leaderboard_table.columns]
 
-def change_tab(query_param):
-    query_param = query_param.replace("'", '"')
-    query_param = json.loads(query_param)
-
-    if (
-        isinstance(query_param, dict)
-        and "tab" in query_param
-        and query_param["tab"] == "evaluation"
-    ):
-        return gr.Tabs.update(selected=1)
-    else:
-        return gr.Tabs.update(selected=0)
+def filter_models(
+    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: str, size_query: str, show_deleted: bool
+) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+
+    # Show all models
+    if show_deleted:
+        filtered_df = df[current_columns]
+    else:  # Show only still on the hub models
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True][current_columns]
+
+    if type_query != "all":
+        type_emoji = type_query[0]
+        filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name] == type_emoji]
+
+    if size_query != "all":
+        numeric_intervals = {
+            "all": (0, 10000),
+            "< 1B": (0, 1),
+            "~3B": (1, 5),
+            "~7B": (6, 11),
+            "~13B": (12, 15),
+            "~35B": (16, 55),
+            "60B+": (55, 10000),
+        }
+        numeric_interval = numeric_intervals[size_query]
+        params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+
+        filtered_df = filtered_df[params_column.between(*numeric_interval)]
 
-def update_filter_type(input_type, shown_columns):
-    shown_columns.append(AutoEvalColumn.params.name)
-    return gr.update(visible=(input_type == 'types')), gr.update(visible=(input_type == 'sizes')), shown_columns
+    return filtered_df
 
 
 demo = gr.Blocks(css=custom_css)
@@ -346,13 +274,39 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
-                shown_columns = gr.CheckboxGroup(
-                    choices = [c for c in COLS if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]], 
-                    value = [c for c in COLS_LITE if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]],
-                    label="Select columns to show", 
-                    elem_id="column-select", 
-                    interactive=True,
-                )
+                with gr.Column():
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c
+                                for c in COLS
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            value=[
+                                c
+                                for c in COLS_LITE
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    with gr.Row():
+                        deleted_models_visibility = gr.Checkbox(
+                            value=True, label="Show models removed from the hub", interactive=True
+                        )
                 with gr.Column(min_width=320):
                     search_bar = gr.Textbox(
                         placeholder="🔍 Search for your model and press ENTER...",
@@ -360,46 +314,47 @@ with demo:
                         elem_id="search-bar",
                     )
                     with gr.Box(elem_id="box-filter"):
-                        filter_type = gr.Dropdown(
-                                label="⏚ Filter model",
-                                choices=["types", "sizes"], value="types",
-                                interactive=True,
-                                elem_id="filter_type"
-                        )
-                        filter_columns = gr.Radio(
+                        filter_columns_type = gr.Radio(
                             label="⏚ Filter model types",
-                            show_label=False,
-                            choices = [
-                                "all", 
+                            choices=[
+                                "all",
                                 ModelType.PT.to_str(),
                                 ModelType.FT.to_str(),
                                 ModelType.IFT.to_str(),
-                                ModelType.RL.to_str(), 
+                                ModelType.RL.to_str(),
                             ],
                             value="all",
-                            elem_id="filter-columns"
+                            interactive=True,
+                            elem_id="filter-columns-type",
                         )
                         filter_columns_size = gr.Radio(
                             label="⏚ Filter model sizes",
-                            show_label=False,
-                            choices = [
+                            choices=[
                                 "all",
                                 "< 1B",
                                 "~3B",
                                 "~7B",
                                 "~13B",
                                 "~35B",
-                                "60B+"
+                                "60B+",
                             ],
                             value="all",
-                            visible=False,
                             interactive=True,
-                            elem_id="filter-columns-size"
+                            elem_id="filter-columns-size",
                         )
-            
+
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name]],
-                headers=[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name],
+                value=leaderboard_df[
+                    [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+                    + shown_columns.value
+                    + [AutoEvalColumn.dummy.name]
+                ],
+                headers=[
+                    AutoEvalColumn.model_type_symbol.name,
+                    AutoEvalColumn.model.name,
+                ]
+                + shown_columns.value
+                + [AutoEvalColumn.dummy.name],
                 datatype=TYPES,
                 max_rows=None,
                 elem_id="leaderboard-table",
@@ -417,14 +372,55 @@ with demo:
             )
             search_bar.submit(
                 search_table,
-                [hidden_leaderboard_table_for_search, leaderboard_table, search_bar],
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            shown_columns.change(
+                select_columns,
+                [hidden_leaderboard_table_for_search, shown_columns],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_type.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_size.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+            deleted_models_visibility.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
                 leaderboard_table,
+                queue=False,
             )
-            
-            filter_type.change(update_filter_type,inputs=[filter_type, shown_columns],outputs=[filter_columns, filter_columns_size, shown_columns],queue=False).then(select_columns, [hidden_leaderboard_table_for_search, shown_columns], leaderboard_table, queue=False)
-            shown_columns.change(select_columns, [hidden_leaderboard_table_for_search, shown_columns], leaderboard_table, queue=False)
-            filter_columns.change(filter_items, [hidden_leaderboard_table_for_search, leaderboard_table, filter_columns], leaderboard_table, queue=False)
-            filter_columns_size.change(filter_items_size, [hidden_leaderboard_table_for_search, leaderboard_table, filter_columns_size], leaderboard_table, queue=False)
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
@@ -434,7 +430,10 @@ with demo:
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
-                    with gr.Accordion(f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             finished_eval_table = gr.components.Dataframe(
                                 value=finished_eval_queue_df,
@@ -442,7 +441,10 @@ with demo:
                                 datatype=EVAL_TYPES,
                                 max_rows=5,
                             )
-                    with gr.Accordion(f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             running_eval_table = gr.components.Dataframe(
                                 value=running_eval_queue_df,
@@ -451,7 +453,10 @@ with demo:
                                 max_rows=5,
                             )
 
-                    with gr.Accordion(f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             pending_eval_table = gr.components.Dataframe(
                                 value=pending_eval_queue_df,

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -24,7 +24,7 @@ gradio_client==0.1.3
 h11==0.14.0
 httpcore==0.17.0
 httpx==0.24.0
-huggingface-hub==0.13.4
+huggingface-hub==0.16.4
 idna==3.4
 Jinja2==3.1.2
 jsonschema==4.17.3
@@ -59,7 +59,7 @@ sniffio==1.3.0
 starlette==0.26.1
 toolz==0.12.0
 tqdm==4.65.0
-transformers==4.28.1
+transformers==4.32.0
 typing_extensions==4.5.0
 tzdata==2023.3
 tzlocal==4.3

src/assets/css_html_js.py

@@ -89,13 +89,13 @@ table th:first-child {
 #filter_type label > .wrap .wrap-inner input{
     width: 1px
 }
-#filter-columns{
+#filter-columns-type{
     border:0;
-    padding:0;
+    padding:0.5;
 }
 #filter-columns-size{
     border:0;
-    padding:0;
+    padding:0.5;
 }
 #box-filter > .form{
     border: 0
@@ -108,4 +108,4 @@ get_window_url_params = """
         url_params = Object.fromEntries(params);
         return url_params;
     }
-    """
+    """

src/assets/hardcoded_evals.py

@@ -1,4 +1,4 @@
-from src.utils_display import AutoEvalColumn, model_hyperlink
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
 
 gpt4_values = {
     AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
@@ -6,9 +6,9 @@ gpt4_values = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 84.3,
     AutoEvalColumn.arc.name: 96.3,
-    AutoEvalColumn.hellaswag.name:  95.3,
-    AutoEvalColumn.mmlu.name:  86.4,
-    AutoEvalColumn.truthfulqa.name:  59.0,
+    AutoEvalColumn.hellaswag.name: 95.3,
+    AutoEvalColumn.mmlu.name: 86.4,
+    AutoEvalColumn.truthfulqa.name: 59.0,
     AutoEvalColumn.dummy.name: "GPT-4",
     AutoEvalColumn.model_type.name: "",
 }
@@ -19,9 +19,9 @@ gpt35_values = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 71.9,
     AutoEvalColumn.arc.name: 85.2,
-    AutoEvalColumn.hellaswag.name:  85.5,
-    AutoEvalColumn.mmlu.name:  70.0,
-    AutoEvalColumn.truthfulqa.name:  47.0,
+    AutoEvalColumn.hellaswag.name: 85.5,
+    AutoEvalColumn.mmlu.name: 70.0,
+    AutoEvalColumn.truthfulqa.name: 47.0,
     AutoEvalColumn.dummy.name: "GPT-3.5",
     AutoEvalColumn.model_type.name: "",
 }
@@ -32,10 +32,9 @@ baseline = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 25.0,
     AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name:  25.0,
-    AutoEvalColumn.mmlu.name:  25.0,
-    AutoEvalColumn.truthfulqa.name:  25.0,
+    AutoEvalColumn.hellaswag.name: 25.0,
+    AutoEvalColumn.mmlu.name: 25.0,
+    AutoEvalColumn.truthfulqa.name: 25.0,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
 }
-

src/assets/text_content.py

@@ -1,17 +1,17 @@
-from ..auto_leaderboard.model_metadata_type import ModelType
+from src.display_models.model_metadata_type import ModelType
 
 TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
 
-INTRODUCTION_TEXT = f"""
+INTRODUCTION_TEXT = """
 📐 The 🤗 Open LLM Leaderboard aims to track, rank and evaluate open LLMs and chatbots.
 
-🤗 Submit a model for automated evaluation on the 🤗 GPU cluster on the "Submit" page! 
+🤗 Submit a model for automated evaluation on the 🤗 GPU cluster on the "Submit" page!
 The leaderboard's backend runs the great [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) - read more details in the "About" page!
 """
 
 LLM_BENCHMARKS_TEXT = f"""
 # Context
-With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art. 
+With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art.
 
 ## Icons
 {ModelType.PT.to_str(" : ")} model
@@ -25,14 +25,14 @@ If there is no icon, we have not uploaded the information on the model yet, feel
 
 ## How it works
 
-📈 We evaluate models on 4 key benchmarks using the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks. 
+📈 We evaluate models on 4 key benchmarks using the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks.
 
 - <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
 - <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
 - <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
 - <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model’s propensity to reproduce falsehoods commonly found online. Note: TruthfulQA in the Harness is actually a minima a 6-shots task, as it is prepended by 6 examples systematically, even when launched using 0 for the number of few-shot examples.
 
-For all these evaluations, a higher score is a better score. 
+For all these evaluations, a higher score is a better score.
 We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
 
 ## Details and logs
@@ -46,7 +46,7 @@ To reproduce our results, here is the commands you can run, using [this version]
 `python main.py --model=hf-causal --model_args="pretrained=<your_model>,use_accelerate=True,revision=<your_model_revision>"`
 ` --tasks=<task_list> --num_fewshot=<n_few_shot> --batch_size=2 --output_path=<output_path>`
 
-The total batch size we get for models which fit on one A100 node is 16 (8 GPUs * 2). If you don't use parallelism, adapt your batch size to fit. 
+The total batch size we get for models which fit on one A100 node is 16 (8 GPUs * 2). If you don't use parallelism, adapt your batch size to fit.
 *You can expect results to vary slightly for different batch sizes because of padding.*
 
 The tasks and few shots parameters are:
@@ -65,7 +65,7 @@ If you still have questions, you can check our FAQ [here](https://huggingface.co
 We also gather cool resources from the community, other teams, and other labs [here](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/174)!
 """
 
-EVALUATION_QUEUE_TEXT = f"""
+EVALUATION_QUEUE_TEXT = """
 # Evaluation Queue for the 🤗 Open LLM Leaderboard
 
 Models added here will be automatically evaluated on the 🤗 cluster.
@@ -79,7 +79,7 @@ config = AutoConfig.from_pretrained("your model name", revision=revision)
 model = AutoModel.from_pretrained("your model name", revision=revision)
 tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. 
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
 Note: make sure your model is public!
 Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
@@ -94,8 +94,8 @@ This is a leaderboard for Open LLMs, and we'd love for as many people as possibl
 When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 
 ## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped. 
-Make sure you have followed the above steps first. 
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
 If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 
@@ -135,7 +135,7 @@ CITATION_BUTTON_TEXT = r"""
   url          = {https://doi.org/10.5281/zenodo.5371628}
 }
 @misc{clark2018think,
-      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge}, 
+      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
       author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
       year={2018},
       eprint={1803.05457},
@@ -143,7 +143,7 @@ CITATION_BUTTON_TEXT = r"""
       primaryClass={cs.AI}
 }
 @misc{zellers2019hellaswag,
-      title={HellaSwag: Can a Machine Really Finish Your Sentence?}, 
+      title={HellaSwag: Can a Machine Really Finish Your Sentence?},
       author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
       year={2019},
       eprint={1905.07830},
@@ -151,7 +151,7 @@ CITATION_BUTTON_TEXT = r"""
       primaryClass={cs.CL}
 }
 @misc{hendrycks2021measuring,
-      title={Measuring Massive Multitask Language Understanding}, 
+      title={Measuring Massive Multitask Language Understanding},
       author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
       year={2021},
       eprint={2009.03300},
@@ -159,7 +159,7 @@ CITATION_BUTTON_TEXT = r"""
       primaryClass={cs.CY}
 }
 @misc{lin2022truthfulqa,
-      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
+      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
       author={Stephanie Lin and Jacob Hilton and Owain Evans},
       year={2022},
       eprint={2109.07958},

src/auto_leaderboard/model_metadata_type.py

@@ -1,551 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Dict
-
-
-@dataclass
-class ModelInfo:
-    name: str
-    symbol: str # emoji
-
-
-class ModelType(Enum):
-    PT = ModelInfo(name="pretrained", symbol="🟢")
-    FT = ModelInfo(name="fine-tuned", symbol="🔶")
-    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
-    RL = ModelInfo(name="RL-tuned", symbol="🟦")
-    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
-
-    def to_str(self, separator = " "):
-        return f"{self.value.symbol}{separator}{self.value.name}" 
-
-
-MODEL_TYPE_METADATA: Dict[str, ModelType] = {
-    'notstoic/PygmalionCoT-7b': ModelType.IFT,
-    'aisquared/dlite-v1-355m': ModelType.IFT,
-    'aisquared/dlite-v1-1_5b': ModelType.IFT,
-    'aisquared/dlite-v1-774m': ModelType.IFT,
-    'aisquared/dlite-v1-124m': ModelType.IFT,
-    'aisquared/chopt-2_7b': ModelType.IFT,
-    'aisquared/dlite-v2-124m': ModelType.IFT,
-    'aisquared/dlite-v2-774m': ModelType.IFT,
-    'aisquared/dlite-v2-1_5b': ModelType.IFT,
-    'aisquared/chopt-1_3b': ModelType.IFT,
-    'aisquared/dlite-v2-355m': ModelType.IFT,
-    'augtoma/qCammel-13': ModelType.IFT,
-    'Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload': ModelType.IFT,
-    'Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload': ModelType.IFT,
-    'TheBloke/alpaca-lora-65B-HF': ModelType.FT,
-    'TheBloke/tulu-7B-fp16': ModelType.IFT,
-    'TheBloke/guanaco-7B-HF': ModelType.FT,
-    'TheBloke/koala-7B-HF': ModelType.FT,
-    'TheBloke/wizardLM-7B-HF': ModelType.IFT,
-    'TheBloke/airoboros-13B-HF': ModelType.IFT,
-    'TheBloke/koala-13B-HF': ModelType.FT,
-    'TheBloke/Wizard-Vicuna-7B-Uncensored-HF': ModelType.FT,
-    'TheBloke/dromedary-65b-lora-HF': ModelType.IFT,
-    'TheBloke/wizardLM-13B-1.0-fp16': ModelType.IFT,
-    'TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16': ModelType.FT,
-    'TheBloke/Wizard-Vicuna-30B-Uncensored-fp16': ModelType.FT,
-    'TheBloke/wizard-vicuna-13B-HF': ModelType.IFT,
-    'TheBloke/UltraLM-13B-fp16': ModelType.IFT,
-    'TheBloke/OpenAssistant-FT-7-Llama-30B-HF': ModelType.FT,
-    'TheBloke/vicuna-13B-1.1-HF': ModelType.IFT,
-    'TheBloke/guanaco-13B-HF': ModelType.FT,
-    'TheBloke/guanaco-65B-HF': ModelType.FT,
-    'TheBloke/airoboros-7b-gpt4-fp16': ModelType.IFT,
-    'TheBloke/llama-30b-supercot-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Llama-2-13B-fp16': ModelType.PT,
-    'TheBloke/llama-2-70b-Guanaco-QLoRA-fp16': ModelType.FT,
-    'TheBloke/landmark-attention-llama7b-fp16': ModelType.IFT,
-    'TheBloke/Planner-7B-fp16': ModelType.IFT,
-    'TheBloke/Wizard-Vicuna-13B-Uncensored-HF': ModelType.FT,
-    'TheBloke/gpt4-alpaca-lora-13B-HF': ModelType.IFT,
-    'TheBloke/gpt4-x-vicuna-13B-HF': ModelType.IFT,
-    'TheBloke/gpt4-alpaca-lora_mlp-65B-HF': ModelType.IFT,
-    'TheBloke/tulu-13B-fp16': ModelType.IFT,
-    'TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16': ModelType.IFT,
-    'TheBloke/Llama-2-70B-fp16': ModelType.IFT,
-    'TheBloke/WizardLM-30B-fp16': ModelType.IFT,
-    'TheBloke/robin-13B-v2-fp16': ModelType.FT,
-    'TheBloke/robin-33B-v2-fp16': ModelType.FT,
-    'TheBloke/Vicuna-13B-CoT-fp16': ModelType.IFT,
-    'TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16': ModelType.FT,
-    'TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/GPlatty-30B-SuperHOT-8K-fp16': ModelType.FT,
-    'TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16': ModelType.IFT,
-    'jphme/orca_mini_v2_ger_7b': ModelType.IFT,
-    'Ejafa/vicuna_7B_vanilla_1.1': ModelType.FT,
-    'kevinpro/Vicuna-13B-CoT': ModelType.IFT,
-    'AlekseyKorshuk/pygmalion-6b-vicuna-chatml': ModelType.FT,
-    'AlekseyKorshuk/chatml-pyg-v1': ModelType.FT,
-    'concedo/Vicuzard-30B-Uncensored': ModelType.FT,
-    'concedo/OPT-19M-ChatSalad': ModelType.FT,
-    'concedo/Pythia-70M-ChatSalad': ModelType.FT,
-    'digitous/13B-HyperMantis': ModelType.IFT,
-    'digitous/Adventien-GPTJ': ModelType.FT,
-    'digitous/Alpacino13b': ModelType.IFT,
-    'digitous/GPT-R': ModelType.IFT,
-    'digitous/Javelin-R': ModelType.IFT,
-    'digitous/Javalion-GPTJ': ModelType.IFT,
-    'digitous/Javalion-R': ModelType.IFT,
-    'digitous/Skegma-GPTJ': ModelType.FT,
-    'digitous/Alpacino30b': ModelType.IFT,
-    'digitous/Janin-GPTJ': ModelType.FT,
-    'digitous/Janin-R': ModelType.FT,
-    'digitous/Javelin-GPTJ': ModelType.FT,
-    'SaylorTwift/gpt2_test': ModelType.PT,
-    'anton-l/gpt-j-tiny-random': ModelType.FT,
-    'Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca': ModelType.FT,
-    'Lazycuber/pyg-instruct-wizardlm': ModelType.FT,
-    'Lazycuber/Janemalion-6B': ModelType.FT,
-    'IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1': ModelType.FT,
-    'IDEA-CCNL/Ziya-LLaMA-13B-v1': ModelType.IFT,
-    'dsvv-cair/alpaca-cleaned-llama-30b-bf16': ModelType.FT,
-    'gpt2-medium': ModelType.PT,
-    'camel-ai/CAMEL-13B-Combined-Data': ModelType.IFT,
-    'camel-ai/CAMEL-13B-Role-Playing-Data': ModelType.FT,
-    'camel-ai/CAMEL-33B-Combined-Data': ModelType.IFT,
-    'PygmalionAI/pygmalion-6b': ModelType.FT,
-    'PygmalionAI/metharme-1.3b': ModelType.IFT,
-    'PygmalionAI/pygmalion-1.3b': ModelType.FT,
-    'PygmalionAI/pygmalion-350m': ModelType.FT,
-    'PygmalionAI/pygmalion-2.7b': ModelType.FT,
-    'medalpaca/medalpaca-7b': ModelType.FT,
-    'lilloukas/Platypus-30B': ModelType.IFT,
-    'lilloukas/GPlatty-30B': ModelType.FT,
-    'mncai/chatdoctor': ModelType.FT,
-    'chaoyi-wu/MedLLaMA_13B': ModelType.FT,
-    'LoupGarou/WizardCoder-Guanaco-15B-V1.0': ModelType.IFT,
-    'LoupGarou/WizardCoder-Guanaco-15B-V1.1': ModelType.FT,
-    'hakurei/instruct-12b': ModelType.IFT,
-    'hakurei/lotus-12B': ModelType.FT,
-    'shibing624/chinese-llama-plus-13b-hf': ModelType.IFT,
-    'shibing624/chinese-alpaca-plus-7b-hf': ModelType.IFT,
-    'shibing624/chinese-alpaca-plus-13b-hf': ModelType.IFT,
-    'mosaicml/mpt-7b-instruct': ModelType.IFT,
-    'mosaicml/mpt-30b-chat': ModelType.IFT,
-    'mosaicml/mpt-7b-storywriter': ModelType.FT,
-    'mosaicml/mpt-30b-instruct': ModelType.IFT,
-    'mosaicml/mpt-7b-chat': ModelType.IFT,
-    'mosaicml/mpt-30b': ModelType.PT,
-    'Corianas/111m': ModelType.IFT,
-    'Corianas/Quokka_1.3b': ModelType.IFT,
-    'Corianas/256_5epoch': ModelType.FT,
-    'Corianas/Quokka_256m': ModelType.IFT,
-    'Corianas/Quokka_590m': ModelType.IFT,
-    'Corianas/gpt-j-6B-Dolly': ModelType.FT,
-    'Corianas/Quokka_2.7b': ModelType.IFT,
-    'cyberagent/open-calm-7b': ModelType.FT,
-    'Aspik101/Nous-Hermes-13b-pl-lora_unload': ModelType.IFT,
-    'THUDM/chatglm2-6b': ModelType.IFT,
-    'MetaIX/GPT4-X-Alpasta-30b': ModelType.IFT,
-    'NYTK/PULI-GPTrio': ModelType.PT,
-    'EleutherAI/pythia-1.3b': ModelType.PT,
-    'EleutherAI/pythia-2.8b-deduped': ModelType.PT,
-    'EleutherAI/gpt-neo-125m': ModelType.PT,
-    'EleutherAI/pythia-160m': ModelType.PT,
-    'EleutherAI/gpt-neo-2.7B': ModelType.PT,
-    'EleutherAI/pythia-1b-deduped': ModelType.PT,
-    'EleutherAI/pythia-6.7b': ModelType.PT,
-    'EleutherAI/pythia-70m-deduped': ModelType.PT,
-    'EleutherAI/gpt-neox-20b': ModelType.PT,
-    'EleutherAI/pythia-1.4b-deduped': ModelType.PT,
-    'EleutherAI/pythia-2.7b': ModelType.PT,
-    'EleutherAI/pythia-6.9b-deduped': ModelType.PT,
-    'EleutherAI/pythia-70m': ModelType.PT,
-    'EleutherAI/gpt-j-6b': ModelType.PT,
-    'EleutherAI/pythia-12b-deduped': ModelType.PT,
-    'EleutherAI/gpt-neo-1.3B': ModelType.PT,
-    'EleutherAI/pythia-410m-deduped': ModelType.PT,
-    'EleutherAI/pythia-160m-deduped': ModelType.PT,
-    'EleutherAI/polyglot-ko-12.8b': ModelType.PT,
-    'EleutherAI/pythia-12b': ModelType.PT,
-    'roneneldan/TinyStories-33M': ModelType.PT,
-    'roneneldan/TinyStories-28M': ModelType.PT,
-    'roneneldan/TinyStories-1M': ModelType.PT,
-    'roneneldan/TinyStories-8M': ModelType.PT,
-    'roneneldan/TinyStories-3M': ModelType.PT,
-    'jerryjalapeno/nart-100k-7b': ModelType.FT,
-    'lmsys/vicuna-13b-v1.3': ModelType.IFT,
-    'lmsys/vicuna-7b-v1.3': ModelType.IFT,
-    'lmsys/vicuna-13b-v1.1': ModelType.IFT,
-    'lmsys/vicuna-13b-delta-v1.1': ModelType.IFT,
-    'lmsys/vicuna-7b-delta-v1.1': ModelType.IFT,
-    'abhiramtirumala/DialoGPT-sarcastic-medium': ModelType.FT,
-    'haonan-li/bactrian-x-llama-13b-merged': ModelType.IFT,
-    'Gryphe/MythoLogic-13b': ModelType.IFT,
-    'Gryphe/MythoBoros-13b': ModelType.IFT,
-    'pillowtalks-ai/delta13b': ModelType.FT,
-    'wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard': ModelType.FT,
-    'bigscience/bloom-7b1': ModelType.PT,
-    'bigcode/tiny_starcoder_py': ModelType.PT,
-    'bigcode/starcoderplus': ModelType.FT,
-    'bigcode/gpt_bigcode-santacoder': ModelType.PT,
-    'bigcode/starcoder': ModelType.PT,
-    'Open-Orca/OpenOrca-Preview1-13B': ModelType.IFT,
-    'microsoft/DialoGPT-large': ModelType.FT,
-    'microsoft/DialoGPT-small': ModelType.FT,
-    'microsoft/DialoGPT-medium': ModelType.FT,
-    'microsoft/CodeGPT-small-py': ModelType.FT,
-    'Tincando/fiction_story_generator': ModelType.FT,
-    'Pirr/pythia-13b-deduped-green_devil': ModelType.FT,
-    'Aeala/GPT4-x-AlpacaDente2-30b': ModelType.FT,
-    'Aeala/GPT4-x-AlpacaDente-30b': ModelType.FT,
-    'Aeala/GPT4-x-Alpasta-13b': ModelType.FT,
-    'Aeala/VicUnlocked-alpaca-30b': ModelType.IFT,
-    'Tap-M/Luna-AI-Llama2-Uncensored': ModelType.FT,
-    'illuin/test-custom-llama': ModelType.FT,
-    'dvruette/oasst-llama-13b-2-epochs': ModelType.FT,
-    'dvruette/oasst-gpt-neox-20b-1000-steps': ModelType.FT,
-    'dvruette/llama-13b-pretrained-dropout': ModelType.PT,
-    'dvruette/llama-13b-pretrained': ModelType.PT,
-    'dvruette/llama-13b-pretrained-sft-epoch-1': ModelType.FT,
-    'dvruette/llama-13b-pretrained-sft-do2': ModelType.FT,
-    'dvruette/oasst-gpt-neox-20b-3000-steps': ModelType.FT,
-    'dvruette/oasst-pythia-12b-pretrained-sft': ModelType.FT,
-    'dvruette/oasst-pythia-6.9b-4000-steps': ModelType.FT,
-    'dvruette/gpt-neox-20b-full-precision': ModelType.FT,
-    'dvruette/oasst-llama-13b-1000-steps': ModelType.FT,
-    'openlm-research/open_llama_7b_700bt_preview': ModelType.PT,
-    'openlm-research/open_llama_7b': ModelType.PT,
-    'openlm-research/open_llama_7b_v2': ModelType.PT,
-    'openlm-research/open_llama_3b': ModelType.PT,
-    'openlm-research/open_llama_13b': ModelType.PT,
-    'openlm-research/open_llama_3b_v2': ModelType.PT,
-    'PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged': ModelType.IFT,
-    'GeorgiaTechResearchInstitute/galpaca-30b': ModelType.IFT,
-    'GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct': ModelType.IFT,
-    'databricks/dolly-v2-7b': ModelType.IFT,
-    'databricks/dolly-v2-3b': ModelType.IFT,
-    'databricks/dolly-v2-12b': ModelType.IFT,
-    'Rachneet/gpt2-xl-alpaca': ModelType.FT,
-    'Locutusque/gpt2-conversational-or-qa': ModelType.FT,
-    'psyche/kogpt': ModelType.FT,
-    'NbAiLab/nb-gpt-j-6B-alpaca': ModelType.IFT,
-    'Mikael110/llama-2-7b-guanaco-fp16': ModelType.FT,
-    'Mikael110/llama-2-13b-guanaco-fp16': ModelType.FT,
-    'Fredithefish/CrimsonPajama': ModelType.IFT,
-    'Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K': ModelType.FT,
-    'Fredithefish/ScarletPajama-3B-HF': ModelType.FT,
-    'Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4': ModelType.IFT,
-    'acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1': ModelType.IFT,
-    'eachadea/vicuna-13b-1.1': ModelType.FT,
-    'eachadea/vicuna-7b-1.1': ModelType.FT,
-    'eachadea/vicuna-13b': ModelType.FT,
-    'openaccess-ai-collective/wizard-mega-13b': ModelType.IFT,
-    'openaccess-ai-collective/manticore-13b': ModelType.IFT,
-    'openaccess-ai-collective/manticore-30b-chat-pyg-alpha': ModelType.IFT,
-    'openaccess-ai-collective/minotaur-13b': ModelType.IFT,
-    'openaccess-ai-collective/minotaur-13b-fixed': ModelType.IFT,
-    'openaccess-ai-collective/hippogriff-30b-chat': ModelType.IFT,
-    'openaccess-ai-collective/manticore-13b-chat-pyg': ModelType.IFT,
-    'pythainlp/wangchanglm-7.5B-sft-enth': ModelType.IFT,
-    'pythainlp/wangchanglm-7.5B-sft-en-sharded': ModelType.IFT,
-    'euclaise/gpt-neox-122m-minipile-digits': ModelType.FT,
-    'stabilityai/StableBeluga1-Delta': ModelType.IFT,
-    'stabilityai/stablelm-tuned-alpha-7b': ModelType.IFT,
-    'stabilityai/StableBeluga2': ModelType.IFT,
-    'stabilityai/StableBeluga-13B': ModelType.IFT,
-    'stabilityai/StableBeluga-7B': ModelType.IFT,
-    'stabilityai/stablelm-base-alpha-7b': ModelType.PT,
-    'stabilityai/stablelm-base-alpha-3b': ModelType.PT,
-    'stabilityai/stablelm-tuned-alpha-3b': ModelType.IFT,
-    'alibidaran/medical_transcription_generator': ModelType.FT,
-    'CalderaAI/30B-Lazarus': ModelType.IFT,
-    'CalderaAI/13B-BlueMethod': ModelType.IFT,
-    'CalderaAI/13B-Ouroboros': ModelType.IFT,
-    'KoboldAI/OPT-13B-Erebus': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Janeway': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Shinen': ModelType.FT,
-    'KoboldAI/fairseq-dense-2.7B': ModelType.PT,
-    'KoboldAI/OPT-6B-nerys-v2': ModelType.FT,
-    'KoboldAI/GPT-NeoX-20B-Skein': ModelType.FT,
-    'KoboldAI/PPO_Pygway-6b-Mix': ModelType.FT,
-    'KoboldAI/fairseq-dense-6.7B': ModelType.PT,
-    'KoboldAI/fairseq-dense-125M': ModelType.PT,
-    'KoboldAI/OPT-13B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Erebus': ModelType.FT,
-    'KoboldAI/OPT-350M-Nerys-v2': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Nerys-v2': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/OPT-13B-Nerys-v2': ModelType.FT,
-    'KoboldAI/GPT-NeoX-20B-Erebus': ModelType.FT,
-    'KoboldAI/OPT-6.7B-Erebus': ModelType.FT,
-    'KoboldAI/fairseq-dense-355M': ModelType.PT,
-    'KoboldAI/OPT-6.7B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Adventure': ModelType.FT,
-    'KoboldAI/OPT-350M-Erebus': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Skein': ModelType.FT,
-    'KoboldAI/OPT-30B-Erebus': ModelType.FT,
-    'klosax/pythia-160m-deduped-step92k-193bt': ModelType.PT,
-    'klosax/open_llama_3b_350bt_preview': ModelType.PT,
-    'klosax/openllama-3b-350bt': ModelType.PT,
-    'klosax/pythia-70m-deduped-step44k-92bt': ModelType.PT,
-    'klosax/open_llama_13b_600bt_preview': ModelType.PT,
-    'klosax/open_llama_7b_400bt_preview': ModelType.PT,
-    'kfkas/Llama-2-ko-7b-Chat': ModelType.IFT,
-    'WeOpenML/Alpaca-7B-v1': ModelType.IFT,
-    'WeOpenML/PandaLM-Alpaca-7B-v1': ModelType.IFT,
-    'TFLai/gpt2-turkish-uncased': ModelType.FT,
-    'ehartford/WizardLM-13B-Uncensored': ModelType.IFT,
-    'ehartford/dolphin-llama-13b': ModelType.IFT,
-    'ehartford/Wizard-Vicuna-30B-Uncensored': ModelType.FT,
-    'ehartford/WizardLM-30B-Uncensored': ModelType.IFT,
-    'ehartford/Wizard-Vicuna-13B-Uncensored': ModelType.FT,
-    'ehartford/WizardLM-7B-Uncensored': ModelType.IFT,
-    'ehartford/based-30b': ModelType.FT,
-    'ehartford/Wizard-Vicuna-7B-Uncensored': ModelType.FT,
-    'wahaha1987/llama_7b_sharegpt94k_fastchat': ModelType.FT,
-    'wahaha1987/llama_13b_sharegpt94k_fastchat': ModelType.FT,
-    'OpenAssistant/oasst-sft-1-pythia-12b': ModelType.FT,
-    'OpenAssistant/stablelm-7b-sft-v7-epoch-3': ModelType.IFT,
-    'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5': ModelType.FT,
-    'OpenAssistant/pythia-12b-sft-v8-2.5k-steps': ModelType.IFT,
-    'OpenAssistant/pythia-12b-sft-v8-7k-steps': ModelType.IFT,
-    'OpenAssistant/pythia-12b-pre-v8-12.5k-steps': ModelType.IFT,
-    'OpenAssistant/llama2-13b-orca-8k-3319': ModelType.IFT,
-    'junelee/wizard-vicuna-13b': ModelType.FT,
-    'BreadAi/gpt-YA-1-1_160M': ModelType.PT,
-    'BreadAi/MuseCan': ModelType.PT,
-    'BreadAi/MusePy-1-2': ModelType.PT,
-    'BreadAi/DiscordPy': ModelType.PT,
-    'BreadAi/PM_modelV2': ModelType.PT,
-    'BreadAi/gpt-Youtube': ModelType.PT,
-    'BreadAi/StoryPy': ModelType.FT,
-    'julianweng/Llama-2-7b-chat-orcah': ModelType.FT,
-    'AGI-inc/lora_moe_7b_baseline': ModelType.FT,
-    'AGI-inc/lora_moe_7b': ModelType.FT,
-    'togethercomputer/GPT-NeoXT-Chat-Base-20B': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Chat-7B-v0.1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-7B-Base': ModelType.PT,
-    'togethercomputer/RedPajama-INCITE-7B-Instruct': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Base-3B-v1': ModelType.PT,
-    'togethercomputer/Pythia-Chat-Base-7B': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Base-7B-v0.1': ModelType.PT,
-    'togethercomputer/GPT-JT-6B-v1': ModelType.IFT,
-    'togethercomputer/GPT-JT-6B-v0': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Chat-3B-v1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-7B-Chat': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Instruct-3B-v1': ModelType.IFT,
-    'Writer/camel-5b-hf': ModelType.IFT,
-    'Writer/palmyra-base': ModelType.PT,
-    'MBZUAI/LaMini-GPT-1.5B': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-111m': ModelType.IFT,
-    'MBZUAI/lamini-neo-1.3b': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-1.3b': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-256m': ModelType.IFT,
-    'MBZUAI/LaMini-GPT-124M': ModelType.IFT,
-    'MBZUAI/lamini-neo-125m': ModelType.IFT,
-    'TehVenom/DiffMerge-DollyGPT-Pygmalion': ModelType.FT,
-    'TehVenom/PPO_Shygmalion-6b': ModelType.FT,
-    'TehVenom/Dolly_Shygmalion-6b-Dev_V8P2': ModelType.FT,
-    'TehVenom/Pygmalion_AlpacaLora-7b': ModelType.FT,
-    'TehVenom/PPO_Pygway-V8p4_Dev-6b': ModelType.FT,
-    'TehVenom/Dolly_Malion-6b': ModelType.FT,
-    'TehVenom/PPO_Shygmalion-V8p4_Dev-6b': ModelType.FT,
-    'TehVenom/ChanMalion': ModelType.FT,
-    'TehVenom/GPT-J-Pyg_PPO-6B': ModelType.IFT,
-    'TehVenom/Pygmalion-13b-Merged': ModelType.FT,
-    'TehVenom/Metharme-13b-Merged': ModelType.IFT,
-    'TehVenom/Dolly_Shygmalion-6b': ModelType.FT,
-    'TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4': ModelType.IFT,
-    'georgesung/llama2_7b_chat_uncensored': ModelType.FT,
-    'vicgalle/gpt2-alpaca': ModelType.IFT,
-    'vicgalle/alpaca-7b': ModelType.FT,
-    'vicgalle/gpt2-alpaca-gpt4': ModelType.IFT,
-    'facebook/opt-350m': ModelType.PT,
-    'facebook/opt-125m': ModelType.PT,
-    'facebook/xglm-4.5B': ModelType.PT,
-    'facebook/opt-2.7b': ModelType.PT,
-    'facebook/opt-6.7b': ModelType.PT,
-    'facebook/galactica-30b': ModelType.PT,
-    'facebook/opt-13b': ModelType.PT,
-    'facebook/opt-66b': ModelType.PT,
-    'facebook/xglm-7.5B': ModelType.PT,
-    'facebook/xglm-564M': ModelType.PT,
-    'facebook/opt-30b': ModelType.PT,
-    'golaxy/gogpt-7b': ModelType.FT,
-    'golaxy/gogpt2-7b': ModelType.FT,
-    'golaxy/gogpt-7b-bloom': ModelType.FT,
-    'golaxy/gogpt-3b-bloom': ModelType.FT,
-    'psmathur/orca_mini_v2_7b': ModelType.IFT,
-    'psmathur/orca_mini_7b': ModelType.IFT,
-    'psmathur/orca_mini_3b': ModelType.IFT,
-    'psmathur/orca_mini_v2_13b': ModelType.IFT,
-    'gpt2-xl': ModelType.PT,
-    'lxe/Cerebras-GPT-2.7B-Alpaca-SP': ModelType.FT,
-    'Monero/Manticore-13b-Chat-Pyg-Guanaco': ModelType.FT,
-    'Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b': ModelType.IFT,
-    'Monero/WizardLM-13b-OpenAssistant-Uncensored': ModelType.IFT,
-    'Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b': ModelType.IFT,
-    'jzjiao/opt-1.3b-rlhf': ModelType.FT,
-    'HuggingFaceH4/starchat-beta': ModelType.IFT,
-    'KnutJaegersberg/gpt-2-xl-EvolInstruct': ModelType.IFT,
-    'KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct': ModelType.IFT,
-    'KnutJaegersberg/galactica-orca-wizardlm-1.3b': ModelType.IFT,
-    'openchat/openchat_8192': ModelType.IFT,
-    'openchat/openchat_v2': ModelType.IFT,
-    'openchat/openchat_v2_w': ModelType.IFT,
-    'ausboss/llama-13b-supercot': ModelType.IFT,
-    'ausboss/llama-30b-supercot': ModelType.IFT,
-    'Neko-Institute-of-Science/metharme-7b': ModelType.IFT,
-    'Neko-Institute-of-Science/pygmalion-7b': ModelType.FT,
-    'SebastianSchramm/Cerebras-GPT-111M-instruction': ModelType.IFT,
-    'victor123/WizardLM-13B-1.0': ModelType.IFT,
-    'OpenBuddy/openbuddy-openllama-13b-v7-fp16': ModelType.FT,
-    'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16': ModelType.FT,
-    'OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16': ModelType.FT,
-    'baichuan-inc/Baichuan-7B': ModelType.PT,
-    'tiiuae/falcon-40b-instruct': ModelType.IFT,
-    'tiiuae/falcon-40b': ModelType.PT,
-    'tiiuae/falcon-7b': ModelType.PT,
-    'YeungNLP/firefly-llama-13b': ModelType.FT,
-    'YeungNLP/firefly-llama-13b-v1.2': ModelType.FT,
-    'YeungNLP/firefly-llama2-13b': ModelType.FT,
-    'YeungNLP/firefly-ziya-13b': ModelType.FT,
-    'shaohang/Sparse0.5_OPT-1.3': ModelType.FT,
-    'xzuyn/Alpacino-SuperCOT-13B': ModelType.IFT,
-    'xzuyn/MedicWizard-7B': ModelType.FT,
-    'xDAN-AI/xDAN_13b_l2_lora': ModelType.FT,
-    'beomi/KoAlpaca-Polyglot-5.8B': ModelType.FT,
-    'beomi/llama-2-ko-7b': ModelType.IFT,
-    'Salesforce/codegen-6B-multi': ModelType.PT,
-    'Salesforce/codegen-16B-nl': ModelType.PT,
-    'Salesforce/codegen-6B-nl': ModelType.PT,
-    'ai-forever/rugpt3large_based_on_gpt2': ModelType.FT,
-    'gpt2-large': ModelType.PT,
-    'frank098/orca_mini_3b_juniper': ModelType.FT,
-    'frank098/WizardLM_13B_juniper': ModelType.FT,
-    'FPHam/Free_Sydney_13b_HF': ModelType.FT,
-    'huggingface/llama-13b': ModelType.PT,
-    'huggingface/llama-7b': ModelType.PT,
-    'huggingface/llama-65b': ModelType.PT,
-    'huggingface/llama-30b': ModelType.PT,
-    'Henk717/chronoboros-33B': ModelType.IFT,
-    'jondurbin/airoboros-13b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-7b': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.1': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.2': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.3': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-l2-7b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-l2-13b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-l2-70b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-13b': ModelType.IFT,
-    'jondurbin/airoboros-33b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-33b-gpt4-1.2': ModelType.IFT,
-    'jondurbin/airoboros-65b-gpt4-1.2': ModelType.IFT,
-    'ariellee/SuperPlatty-30B': ModelType.IFT,
-    'danielhanchen/open_llama_3b_600bt_preview': ModelType.FT,
-    'cerebras/Cerebras-GPT-256M': ModelType.PT,
-    'cerebras/Cerebras-GPT-1.3B': ModelType.PT,
-    'cerebras/Cerebras-GPT-13B': ModelType.PT,
-    'cerebras/Cerebras-GPT-2.7B': ModelType.PT,
-    'cerebras/Cerebras-GPT-111M': ModelType.PT,
-    'cerebras/Cerebras-GPT-6.7B': ModelType.PT,
-    'Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf': ModelType.RL,
-    'Yhyu13/llama-30B-hf-openassitant': ModelType.FT,
-    'NousResearch/Nous-Hermes-Llama2-13b': ModelType.IFT,
-    'NousResearch/Nous-Hermes-llama-2-7b': ModelType.IFT,
-    'NousResearch/Redmond-Puffin-13B': ModelType.IFT,
-    'NousResearch/Nous-Hermes-13b': ModelType.IFT,
-    'project-baize/baize-v2-7b': ModelType.IFT,
-    'project-baize/baize-v2-13b': ModelType.IFT,
-    'LLMs/WizardLM-13B-V1.0': ModelType.FT,
-    'LLMs/AlpacaGPT4-7B-elina': ModelType.FT,
-    'wenge-research/yayi-7b': ModelType.FT,
-    'wenge-research/yayi-7b-llama2': ModelType.FT,
-    'wenge-research/yayi-13b-llama2': ModelType.FT,
-    'yhyhy3/open_llama_7b_v2_med_instruct': ModelType.IFT,
-    'llama-anon/instruct-13b': ModelType.IFT,
-    'huggingtweets/jerma985': ModelType.FT,
-    'huggingtweets/gladosystem': ModelType.FT,
-    'huggingtweets/bladeecity-jerma985': ModelType.FT,
-    'huggyllama/llama-13b': ModelType.PT,
-    'huggyllama/llama-65b': ModelType.PT,
-    'FabbriSimo01/Facebook_opt_1.3b_Quantized': ModelType.PT,
-    'upstage/Llama-2-70b-instruct': ModelType.IFT,
-    'upstage/Llama-2-70b-instruct-1024': ModelType.IFT,
-    'upstage/llama-65b-instruct': ModelType.IFT,
-    'upstage/llama-30b-instruct-2048': ModelType.IFT,
-    'upstage/llama-30b-instruct': ModelType.IFT,
-    'WizardLM/WizardLM-13B-1.0': ModelType.IFT,
-    'WizardLM/WizardLM-13B-V1.1': ModelType.IFT,
-    'WizardLM/WizardLM-13B-V1.2': ModelType.IFT,
-    'WizardLM/WizardLM-30B-V1.0': ModelType.IFT,
-    'WizardLM/WizardCoder-15B-V1.0': ModelType.IFT,
-    'gpt2': ModelType.PT,
-    'keyfan/vicuna-chinese-replication-v1.1': ModelType.IFT,
-    'nthngdy/pythia-owt2-70m-100k': ModelType.FT,
-    'nthngdy/pythia-owt2-70m-50k': ModelType.FT,
-    'quantumaikr/KoreanLM-hf': ModelType.FT,
-    'quantumaikr/open_llama_7b_hf': ModelType.FT,
-    'quantumaikr/QuantumLM-70B-hf': ModelType.IFT,
-    'MayaPH/FinOPT-Lincoln': ModelType.FT,
-    'MayaPH/FinOPT-Franklin': ModelType.FT,
-    'MayaPH/GodziLLa-30B': ModelType.IFT,
-    'MayaPH/GodziLLa-30B-plus': ModelType.IFT,
-    'MayaPH/FinOPT-Washington': ModelType.FT,
-    'ogimgio/gpt-neo-125m-neurallinguisticpioneers': ModelType.FT,
-    'layoric/llama-2-13b-code-alpaca': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b-v2': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b-v3': ModelType.FT,
-    'timdettmers/guanaco-33b-merged': ModelType.FT,
-    'elinas/chronos-33b': ModelType.IFT,
-    'heegyu/RedTulu-Uncensored-3B-0719': ModelType.IFT,
-    'heegyu/WizardVicuna-Uncensored-3B-0719': ModelType.IFT,
-    'heegyu/WizardVicuna-3B-0719': ModelType.IFT,
-    'meta-llama/Llama-2-7b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-7b-hf': ModelType.PT,
-    'meta-llama/Llama-2-13b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-13b-hf': ModelType.PT,
-    'meta-llama/Llama-2-70b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-70b-hf': ModelType.PT,
-    'xhyi/PT_GPTNEO350_ATG': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-20b': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt': ModelType.FT,
-    'h2oai/h2ogpt-oig-oasst1-512-6_9b': ModelType.IFT,
-    'h2oai/h2ogpt-oasst1-512-12b': ModelType.IFT,
-    'h2oai/h2ogpt-oig-oasst1-256-6_9b': ModelType.IFT,
-    'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt': ModelType.FT,
-    'h2oai/h2ogpt-oasst1-512-20b': ModelType.IFT,
-    'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-12b': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-multilang-1024-20b': ModelType.FT,
-    'bofenghuang/vigogne-13b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-13b-chat': ModelType.FT,
-    'bofenghuang/vigogne-2-7b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-7b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-7b-chat': ModelType.FT,
-    'Vmware/open-llama-7b-v2-open-instruct': ModelType.IFT,
-    'VMware/open-llama-0.7T-7B-open-instruct-v1.1': ModelType.IFT,
-    'ewof/koishi-instruct-3b': ModelType.IFT,
-    'gywy/llama2-13b-chinese-v1': ModelType.FT,
-    'GOAT-AI/GOAT-7B-Community': ModelType.FT,
-    'psyche/kollama2-7b': ModelType.FT,
-    'TheTravellingEngineer/llama2-7b-hf-guanaco': ModelType.FT,
-    'beaugogh/pythia-1.4b-deduped-sharegpt': ModelType.FT,
-    'augtoma/qCammel-70-x': ModelType.IFT,
-    'Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload': ModelType.IFT,
-    'anhnv125/pygmalion-6b-roleplay': ModelType.FT,
-    '64bits/LexPodLM-13B': ModelType.FT,
-}
-
-
-def model_type_from_str(type):
-    if "fine-tuned" in type or "🔶" in type:
-        return ModelType.FT
-    if "pretrained" in type or "🟢" in type:
-        return ModelType.PT
-    if "RL-tuned" in type or "🟦" in type:
-        return ModelType.RL
-    if "instruction-tuned" in type or "⭕" in type:
-        return ModelType.IFT
-    return ModelType.Unknown
-

src/{auto_leaderboard → display_models}/get_model_metadata.py

@@ -1,17 +1,17 @@
-import re
-import os
 import glob
 import json
 import os
+import re
 from typing import List
+
+import huggingface_hub
+from huggingface_hub import HfApi
 from tqdm import tqdm
 
-from src.utils_display import AutoEvalColumn, model_hyperlink
-from src.auto_leaderboard.model_metadata_type import ModelType, model_type_from_str, MODEL_TYPE_METADATA
-from src.auto_leaderboard.model_metadata_flags import FLAGGED_MODELS, DO_NOT_SUBMIT_MODELS
+from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
+from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
 
-from huggingface_hub import HfApi
-import huggingface_hub
 api = HfApi(token=os.environ.get("H4_TOKEN", None))
 
 
@@ -38,15 +38,18 @@ def get_model_license(model_info):
     except Exception:
         return None
 
+
 def get_model_likes(model_info):
     return model_info.likes
 
+
 size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
 
+
 def get_model_size(model_name, model_info):
     # In billions
     try:
-        return round(model_info.safetensors["total"] / 1e9, 3) 
+        return round(model_info.safetensors["total"] / 1e9, 3)
     except AttributeError:
         try:
             size_match = re.search(size_pattern, model_name.lower())
@@ -58,7 +61,10 @@ def get_model_size(model_name, model_info):
 
 def get_model_type(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
-        request_files = os.path.join("eval-queue", model_data["model_name_for_query"] + "_eval_request_*" + ".json")
+        request_files = os.path.join(
+            "eval-queue",
+            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
+        )
         request_files = glob.glob(request_files)
 
         # Select correct request file (precision)
@@ -70,9 +76,12 @@ def get_model_type(leaderboard_data: List[dict]):
             for tmp_request_file in request_files:
                 with open(tmp_request_file, "r") as f:
                     req_content = json.load(f)
-                    if req_content["status"] == "FINISHED" and req_content["precision"] == model_data["Precision"].split(".")[-1]: 
+                    if (
+                        req_content["status"] == "FINISHED"
+                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
+                    ):
                         request_file = tmp_request_file
-        
+
         if request_file == "":
             model_data[AutoEvalColumn.model_type.name] = ""
             model_data[AutoEvalColumn.model_type_symbol.name] = ""
@@ -81,30 +90,41 @@ def get_model_type(leaderboard_data: List[dict]):
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            is_delta = request["weight_type"] != "Original"
+            request["weight_type"] != "Original"
         except Exception:
-            is_delta = False
+            pass
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
             model_type = model_type_from_str(request["model_type"])
             model_data[AutoEvalColumn.model_type.name] = model_type.value.name
-            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol #+ ("🔺" if is_delta else "")
+            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
         except KeyError:
             if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[model_data["model_name_for_query"]].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[model_data["model_name_for_query"]].value.symbol #+ ("🔺" if is_delta else "")
+                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.symbol  # + ("🔺" if is_delta else "")
             else:
                 model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
                 model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
 
-def flag_models(leaderboard_data:List[dict]):
+
+def flag_models(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
         if model_data["model_name_for_query"] in FLAGGED_MODELS:
             issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
-            issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
-            model_data[AutoEvalColumn.model.name] =  f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+            issue_link = model_hyperlink(
+                FLAGGED_MODELS[model_data["model_name_for_query"]],
+                f"See discussion #{issue_num}",
+            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+
 
 def remove_forbidden_models(leaderboard_data: List[dict]):
     indices_to_remove = []
@@ -116,6 +136,7 @@ def remove_forbidden_models(leaderboard_data: List[dict]):
         leaderboard_data.pop(ix)
     return leaderboard_data
 
+
 def apply_metadata(leaderboard_data: List[dict]):
     leaderboard_data = remove_forbidden_models(leaderboard_data)
     get_model_type(leaderboard_data)

src/{auto_leaderboard → display_models}/model_metadata_flags.py

@@ -8,5 +8,5 @@ FLAGGED_MODELS = {
 
 # Models which have been requested by orgs to not be submitted on the leaderboard
 DO_NOT_SUBMIT_MODELS = [
-    "Voicelab/trurl-2-13b", # trained on MMLU
-]
+    "Voicelab/trurl-2-13b",  # trained on MMLU
+]

src/display_models/model_metadata_type.py

@@ -0,0 +1,550 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict
+
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelInfo(name="pretrained", symbol="🟢")
+    FT = ModelInfo(name="fine-tuned", symbol="🔶")
+    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
+    RL = ModelInfo(name="RL-tuned", symbol="🟦")
+    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+
+MODEL_TYPE_METADATA: Dict[str, ModelType] = {
+    "notstoic/PygmalionCoT-7b": ModelType.IFT,
+    "aisquared/dlite-v1-355m": ModelType.IFT,
+    "aisquared/dlite-v1-1_5b": ModelType.IFT,
+    "aisquared/dlite-v1-774m": ModelType.IFT,
+    "aisquared/dlite-v1-124m": ModelType.IFT,
+    "aisquared/chopt-2_7b": ModelType.IFT,
+    "aisquared/dlite-v2-124m": ModelType.IFT,
+    "aisquared/dlite-v2-774m": ModelType.IFT,
+    "aisquared/dlite-v2-1_5b": ModelType.IFT,
+    "aisquared/chopt-1_3b": ModelType.IFT,
+    "aisquared/dlite-v2-355m": ModelType.IFT,
+    "augtoma/qCammel-13": ModelType.IFT,
+    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload": ModelType.IFT,
+    "TheBloke/alpaca-lora-65B-HF": ModelType.FT,
+    "TheBloke/tulu-7B-fp16": ModelType.IFT,
+    "TheBloke/guanaco-7B-HF": ModelType.FT,
+    "TheBloke/koala-7B-HF": ModelType.FT,
+    "TheBloke/wizardLM-7B-HF": ModelType.IFT,
+    "TheBloke/airoboros-13B-HF": ModelType.IFT,
+    "TheBloke/koala-13B-HF": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF": ModelType.FT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.IFT,
+    "TheBloke/wizardLM-13B-1.0-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16": ModelType.FT,
+    "TheBloke/wizard-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/UltraLM-13B-fp16": ModelType.IFT,
+    "TheBloke/OpenAssistant-FT-7-Llama-30B-HF": ModelType.FT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.IFT,
+    "TheBloke/guanaco-13B-HF": ModelType.FT,
+    "TheBloke/guanaco-65B-HF": ModelType.FT,
+    "TheBloke/airoboros-7b-gpt4-fp16": ModelType.IFT,
+    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-13B-fp16": ModelType.PT,
+    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16": ModelType.FT,
+    "TheBloke/landmark-attention-llama7b-fp16": ModelType.IFT,
+    "TheBloke/Planner-7B-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.FT,
+    "TheBloke/gpt4-alpaca-lora-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-x-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF": ModelType.IFT,
+    "TheBloke/tulu-13B-fp16": ModelType.IFT,
+    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-70B-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-30B-fp16": ModelType.IFT,
+    "TheBloke/robin-13B-v2-fp16": ModelType.FT,
+    "TheBloke/robin-33B-v2-fp16": ModelType.FT,
+    "TheBloke/Vicuna-13B-CoT-fp16": ModelType.IFT,
+    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16": ModelType.FT,
+    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16": ModelType.IFT,
+    "jphme/orca_mini_v2_ger_7b": ModelType.IFT,
+    "Ejafa/vicuna_7B_vanilla_1.1": ModelType.FT,
+    "kevinpro/Vicuna-13B-CoT": ModelType.IFT,
+    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml": ModelType.FT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.FT,
+    "concedo/Vicuzard-30B-Uncensored": ModelType.FT,
+    "concedo/OPT-19M-ChatSalad": ModelType.FT,
+    "concedo/Pythia-70M-ChatSalad": ModelType.FT,
+    "digitous/13B-HyperMantis": ModelType.IFT,
+    "digitous/Adventien-GPTJ": ModelType.FT,
+    "digitous/Alpacino13b": ModelType.IFT,
+    "digitous/GPT-R": ModelType.IFT,
+    "digitous/Javelin-R": ModelType.IFT,
+    "digitous/Javalion-GPTJ": ModelType.IFT,
+    "digitous/Javalion-R": ModelType.IFT,
+    "digitous/Skegma-GPTJ": ModelType.FT,
+    "digitous/Alpacino30b": ModelType.IFT,
+    "digitous/Janin-GPTJ": ModelType.FT,
+    "digitous/Janin-R": ModelType.FT,
+    "digitous/Javelin-GPTJ": ModelType.FT,
+    "SaylorTwift/gpt2_test": ModelType.PT,
+    "anton-l/gpt-j-tiny-random": ModelType.FT,
+    "Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca": ModelType.FT,
+    "Lazycuber/pyg-instruct-wizardlm": ModelType.FT,
+    "Lazycuber/Janemalion-6B": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-v1": ModelType.IFT,
+    "dsvv-cair/alpaca-cleaned-llama-30b-bf16": ModelType.FT,
+    "gpt2-medium": ModelType.PT,
+    "camel-ai/CAMEL-13B-Combined-Data": ModelType.IFT,
+    "camel-ai/CAMEL-13B-Role-Playing-Data": ModelType.FT,
+    "camel-ai/CAMEL-33B-Combined-Data": ModelType.IFT,
+    "PygmalionAI/pygmalion-6b": ModelType.FT,
+    "PygmalionAI/metharme-1.3b": ModelType.IFT,
+    "PygmalionAI/pygmalion-1.3b": ModelType.FT,
+    "PygmalionAI/pygmalion-350m": ModelType.FT,
+    "PygmalionAI/pygmalion-2.7b": ModelType.FT,
+    "medalpaca/medalpaca-7b": ModelType.FT,
+    "lilloukas/Platypus-30B": ModelType.IFT,
+    "lilloukas/GPlatty-30B": ModelType.FT,
+    "mncai/chatdoctor": ModelType.FT,
+    "chaoyi-wu/MedLLaMA_13B": ModelType.FT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.0": ModelType.IFT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.1": ModelType.FT,
+    "hakurei/instruct-12b": ModelType.IFT,
+    "hakurei/lotus-12B": ModelType.FT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-7b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-13b-hf": ModelType.IFT,
+    "mosaicml/mpt-7b-instruct": ModelType.IFT,
+    "mosaicml/mpt-30b-chat": ModelType.IFT,
+    "mosaicml/mpt-7b-storywriter": ModelType.FT,
+    "mosaicml/mpt-30b-instruct": ModelType.IFT,
+    "mosaicml/mpt-7b-chat": ModelType.IFT,
+    "mosaicml/mpt-30b": ModelType.PT,
+    "Corianas/111m": ModelType.IFT,
+    "Corianas/Quokka_1.3b": ModelType.IFT,
+    "Corianas/256_5epoch": ModelType.FT,
+    "Corianas/Quokka_256m": ModelType.IFT,
+    "Corianas/Quokka_590m": ModelType.IFT,
+    "Corianas/gpt-j-6B-Dolly": ModelType.FT,
+    "Corianas/Quokka_2.7b": ModelType.IFT,
+    "cyberagent/open-calm-7b": ModelType.FT,
+    "Aspik101/Nous-Hermes-13b-pl-lora_unload": ModelType.IFT,
+    "THUDM/chatglm2-6b": ModelType.IFT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.IFT,
+    "NYTK/PULI-GPTrio": ModelType.PT,
+    "EleutherAI/pythia-1.3b": ModelType.PT,
+    "EleutherAI/pythia-2.8b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-125m": ModelType.PT,
+    "EleutherAI/pythia-160m": ModelType.PT,
+    "EleutherAI/gpt-neo-2.7B": ModelType.PT,
+    "EleutherAI/pythia-1b-deduped": ModelType.PT,
+    "EleutherAI/pythia-6.7b": ModelType.PT,
+    "EleutherAI/pythia-70m-deduped": ModelType.PT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/pythia-1.4b-deduped": ModelType.PT,
+    "EleutherAI/pythia-2.7b": ModelType.PT,
+    "EleutherAI/pythia-6.9b-deduped": ModelType.PT,
+    "EleutherAI/pythia-70m": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "EleutherAI/pythia-12b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-1.3B": ModelType.PT,
+    "EleutherAI/pythia-410m-deduped": ModelType.PT,
+    "EleutherAI/pythia-160m-deduped": ModelType.PT,
+    "EleutherAI/polyglot-ko-12.8b": ModelType.PT,
+    "EleutherAI/pythia-12b": ModelType.PT,
+    "roneneldan/TinyStories-33M": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.PT,
+    "roneneldan/TinyStories-1M": ModelType.PT,
+    "roneneldan/TinyStories-8M": ModelType.PT,
+    "roneneldan/TinyStories-3M": ModelType.PT,
+    "jerryjalapeno/nart-100k-7b": ModelType.FT,
+    "lmsys/vicuna-13b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-7b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-13b-v1.1": ModelType.IFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.IFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.IFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.FT,
+    "haonan-li/bactrian-x-llama-13b-merged": ModelType.IFT,
+    "Gryphe/MythoLogic-13b": ModelType.IFT,
+    "Gryphe/MythoBoros-13b": ModelType.IFT,
+    "pillowtalks-ai/delta13b": ModelType.FT,
+    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard": ModelType.FT,
+    "bigscience/bloom-7b1": ModelType.PT,
+    "bigcode/tiny_starcoder_py": ModelType.PT,
+    "bigcode/starcoderplus": ModelType.FT,
+    "bigcode/gpt_bigcode-santacoder": ModelType.PT,
+    "bigcode/starcoder": ModelType.PT,
+    "Open-Orca/OpenOrca-Preview1-13B": ModelType.IFT,
+    "microsoft/DialoGPT-large": ModelType.FT,
+    "microsoft/DialoGPT-small": ModelType.FT,
+    "microsoft/DialoGPT-medium": ModelType.FT,
+    "microsoft/CodeGPT-small-py": ModelType.FT,
+    "Tincando/fiction_story_generator": ModelType.FT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente-30b": ModelType.FT,
+    "Aeala/GPT4-x-Alpasta-13b": ModelType.FT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.IFT,
+    "Tap-M/Luna-AI-Llama2-Uncensored": ModelType.FT,
+    "illuin/test-custom-llama": ModelType.FT,
+    "dvruette/oasst-llama-13b-2-epochs": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.FT,
+    "dvruette/llama-13b-pretrained-dropout": ModelType.PT,
+    "dvruette/llama-13b-pretrained": ModelType.PT,
+    "dvruette/llama-13b-pretrained-sft-epoch-1": ModelType.FT,
+    "dvruette/llama-13b-pretrained-sft-do2": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-3000-steps": ModelType.FT,
+    "dvruette/oasst-pythia-12b-pretrained-sft": ModelType.FT,
+    "dvruette/oasst-pythia-6.9b-4000-steps": ModelType.FT,
+    "dvruette/gpt-neox-20b-full-precision": ModelType.FT,
+    "dvruette/oasst-llama-13b-1000-steps": ModelType.FT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_7b_v2": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_13b": ModelType.PT,
+    "openlm-research/open_llama_3b_v2": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/galpaca-30b": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct": ModelType.IFT,
+    "databricks/dolly-v2-7b": ModelType.IFT,
+    "databricks/dolly-v2-3b": ModelType.IFT,
+    "databricks/dolly-v2-12b": ModelType.IFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.FT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.FT,
+    "psyche/kogpt": ModelType.FT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.IFT,
+    "Mikael110/llama-2-7b-guanaco-fp16": ModelType.FT,
+    "Mikael110/llama-2-13b-guanaco-fp16": ModelType.FT,
+    "Fredithefish/CrimsonPajama": ModelType.IFT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K": ModelType.FT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.FT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4": ModelType.IFT,
+    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1": ModelType.IFT,
+    "eachadea/vicuna-13b-1.1": ModelType.FT,
+    "eachadea/vicuna-7b-1.1": ModelType.FT,
+    "eachadea/vicuna-13b": ModelType.FT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b-fixed": ModelType.IFT,
+    "openaccess-ai-collective/hippogriff-30b-chat": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b-chat-pyg": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.IFT,
+    "euclaise/gpt-neox-122m-minipile-digits": ModelType.FT,
+    "stabilityai/StableBeluga1-Delta": ModelType.IFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.IFT,
+    "stabilityai/StableBeluga2": ModelType.IFT,
+    "stabilityai/StableBeluga-13B": ModelType.IFT,
+    "stabilityai/StableBeluga-7B": ModelType.IFT,
+    "stabilityai/stablelm-base-alpha-7b": ModelType.PT,
+    "stabilityai/stablelm-base-alpha-3b": ModelType.PT,
+    "stabilityai/stablelm-tuned-alpha-3b": ModelType.IFT,
+    "alibidaran/medical_transcription_generator": ModelType.FT,
+    "CalderaAI/30B-Lazarus": ModelType.IFT,
+    "CalderaAI/13B-BlueMethod": ModelType.IFT,
+    "CalderaAI/13B-Ouroboros": ModelType.IFT,
+    "KoboldAI/OPT-13B-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Janeway": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Shinen": ModelType.FT,
+    "KoboldAI/fairseq-dense-2.7B": ModelType.PT,
+    "KoboldAI/OPT-6B-nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Skein": ModelType.FT,
+    "KoboldAI/PPO_Pygway-6b-Mix": ModelType.FT,
+    "KoboldAI/fairseq-dense-6.7B": ModelType.PT,
+    "KoboldAI/fairseq-dense-125M": ModelType.PT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-350M-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-13B-Nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-6.7B-Erebus": ModelType.FT,
+    "KoboldAI/fairseq-dense-355M": ModelType.PT,
+    "KoboldAI/OPT-6.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Adventure": ModelType.FT,
+    "KoboldAI/OPT-350M-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Skein": ModelType.FT,
+    "KoboldAI/OPT-30B-Erebus": ModelType.FT,
+    "klosax/pythia-160m-deduped-step92k-193bt": ModelType.PT,
+    "klosax/open_llama_3b_350bt_preview": ModelType.PT,
+    "klosax/openllama-3b-350bt": ModelType.PT,
+    "klosax/pythia-70m-deduped-step44k-92bt": ModelType.PT,
+    "klosax/open_llama_13b_600bt_preview": ModelType.PT,
+    "klosax/open_llama_7b_400bt_preview": ModelType.PT,
+    "kfkas/Llama-2-ko-7b-Chat": ModelType.IFT,
+    "WeOpenML/Alpaca-7B-v1": ModelType.IFT,
+    "WeOpenML/PandaLM-Alpaca-7B-v1": ModelType.IFT,
+    "TFLai/gpt2-turkish-uncased": ModelType.FT,
+    "ehartford/WizardLM-13B-Uncensored": ModelType.IFT,
+    "ehartford/dolphin-llama-13b": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-30B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-30B-Uncensored": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-13B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-7B-Uncensored": ModelType.IFT,
+    "ehartford/based-30b": ModelType.FT,
+    "ehartford/Wizard-Vicuna-7B-Uncensored": ModelType.FT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.FT,
+    "wahaha1987/llama_13b_sharegpt94k_fastchat": ModelType.FT,
+    "OpenAssistant/oasst-sft-1-pythia-12b": ModelType.FT,
+    "OpenAssistant/stablelm-7b-sft-v7-epoch-3": ModelType.IFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.FT,
+    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-sft-v8-7k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps": ModelType.IFT,
+    "OpenAssistant/llama2-13b-orca-8k-3319": ModelType.IFT,
+    "junelee/wizard-vicuna-13b": ModelType.FT,
+    "BreadAi/gpt-YA-1-1_160M": ModelType.PT,
+    "BreadAi/MuseCan": ModelType.PT,
+    "BreadAi/MusePy-1-2": ModelType.PT,
+    "BreadAi/DiscordPy": ModelType.PT,
+    "BreadAi/PM_modelV2": ModelType.PT,
+    "BreadAi/gpt-Youtube": ModelType.PT,
+    "BreadAi/StoryPy": ModelType.FT,
+    "julianweng/Llama-2-7b-chat-orcah": ModelType.FT,
+    "AGI-inc/lora_moe_7b_baseline": ModelType.FT,
+    "AGI-inc/lora_moe_7b": ModelType.FT,
+    "togethercomputer/GPT-NeoXT-Chat-Base-20B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Base": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-7B-Instruct": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/Pythia-Chat-Base-7B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "togethercomputer/GPT-JT-6B-v1": ModelType.IFT,
+    "togethercomputer/GPT-JT-6B-v0": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Chat": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1": ModelType.IFT,
+    "Writer/camel-5b-hf": ModelType.IFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/LaMini-GPT-1.5B": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-111m": ModelType.IFT,
+    "MBZUAI/lamini-neo-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-256m": ModelType.IFT,
+    "MBZUAI/LaMini-GPT-124M": ModelType.IFT,
+    "MBZUAI/lamini-neo-125m": ModelType.IFT,
+    "TehVenom/DiffMerge-DollyGPT-Pygmalion": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-6b": ModelType.FT,
+    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2": ModelType.FT,
+    "TehVenom/Pygmalion_AlpacaLora-7b": ModelType.FT,
+    "TehVenom/PPO_Pygway-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/Dolly_Malion-6b": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/ChanMalion": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B": ModelType.IFT,
+    "TehVenom/Pygmalion-13b-Merged": ModelType.FT,
+    "TehVenom/Metharme-13b-Merged": ModelType.IFT,
+    "TehVenom/Dolly_Shygmalion-6b": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4": ModelType.IFT,
+    "georgesung/llama2_7b_chat_uncensored": ModelType.FT,
+    "vicgalle/gpt2-alpaca": ModelType.IFT,
+    "vicgalle/alpaca-7b": ModelType.FT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.IFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/xglm-4.5B": ModelType.PT,
+    "facebook/opt-2.7b": ModelType.PT,
+    "facebook/opt-6.7b": ModelType.PT,
+    "facebook/galactica-30b": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/xglm-7.5B": ModelType.PT,
+    "facebook/xglm-564M": ModelType.PT,
+    "facebook/opt-30b": ModelType.PT,
+    "golaxy/gogpt-7b": ModelType.FT,
+    "golaxy/gogpt2-7b": ModelType.FT,
+    "golaxy/gogpt-7b-bloom": ModelType.FT,
+    "golaxy/gogpt-3b-bloom": ModelType.FT,
+    "psmathur/orca_mini_v2_7b": ModelType.IFT,
+    "psmathur/orca_mini_7b": ModelType.IFT,
+    "psmathur/orca_mini_3b": ModelType.IFT,
+    "psmathur/orca_mini_v2_13b": ModelType.IFT,
+    "gpt2-xl": ModelType.PT,
+    "lxe/Cerebras-GPT-2.7B-Alpaca-SP": ModelType.FT,
+    "Monero/Manticore-13b-Chat-Pyg-Guanaco": ModelType.FT,
+    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b": ModelType.IFT,
+    "Monero/WizardLM-13b-OpenAssistant-Uncensored": ModelType.IFT,
+    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b": ModelType.IFT,
+    "jzjiao/opt-1.3b-rlhf": ModelType.FT,
+    "HuggingFaceH4/starchat-beta": ModelType.IFT,
+    "KnutJaegersberg/gpt-2-xl-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/galactica-orca-wizardlm-1.3b": ModelType.IFT,
+    "openchat/openchat_8192": ModelType.IFT,
+    "openchat/openchat_v2": ModelType.IFT,
+    "openchat/openchat_v2_w": ModelType.IFT,
+    "ausboss/llama-13b-supercot": ModelType.IFT,
+    "ausboss/llama-30b-supercot": ModelType.IFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.IFT,
+    "Neko-Institute-of-Science/pygmalion-7b": ModelType.FT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.IFT,
+    "victor123/WizardLM-13B-1.0": ModelType.IFT,
+    "OpenBuddy/openbuddy-openllama-13b-v7-fp16": ModelType.FT,
+    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16": ModelType.FT,
+    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16": ModelType.FT,
+    "baichuan-inc/Baichuan-7B": ModelType.PT,
+    "tiiuae/falcon-40b-instruct": ModelType.IFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "YeungNLP/firefly-llama-13b": ModelType.FT,
+    "YeungNLP/firefly-llama-13b-v1.2": ModelType.FT,
+    "YeungNLP/firefly-llama2-13b": ModelType.FT,
+    "YeungNLP/firefly-ziya-13b": ModelType.FT,
+    "shaohang/Sparse0.5_OPT-1.3": ModelType.FT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.IFT,
+    "xzuyn/MedicWizard-7B": ModelType.FT,
+    "xDAN-AI/xDAN_13b_l2_lora": ModelType.FT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.FT,
+    "beomi/llama-2-ko-7b": ModelType.IFT,
+    "Salesforce/codegen-6B-multi": ModelType.PT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-6B-nl": ModelType.PT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.FT,
+    "gpt2-large": ModelType.PT,
+    "frank098/orca_mini_3b_juniper": ModelType.FT,
+    "frank098/WizardLM_13B_juniper": ModelType.FT,
+    "FPHam/Free_Sydney_13b_HF": ModelType.FT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "Henk717/chronoboros-33B": ModelType.IFT,
+    "jondurbin/airoboros-13b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-7b": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.1": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.3": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-l2-7b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-13b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-70b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-13b": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-65b-gpt4-1.2": ModelType.IFT,
+    "ariellee/SuperPlatty-30B": ModelType.IFT,
+    "danielhanchen/open_llama_3b_600bt_preview": ModelType.FT,
+    "cerebras/Cerebras-GPT-256M": ModelType.PT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-13B": ModelType.PT,
+    "cerebras/Cerebras-GPT-2.7B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "cerebras/Cerebras-GPT-6.7B": ModelType.PT,
+    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf": ModelType.RL,
+    "Yhyu13/llama-30B-hf-openassitant": ModelType.FT,
+    "NousResearch/Nous-Hermes-Llama2-13b": ModelType.IFT,
+    "NousResearch/Nous-Hermes-llama-2-7b": ModelType.IFT,
+    "NousResearch/Redmond-Puffin-13B": ModelType.IFT,
+    "NousResearch/Nous-Hermes-13b": ModelType.IFT,
+    "project-baize/baize-v2-7b": ModelType.IFT,
+    "project-baize/baize-v2-13b": ModelType.IFT,
+    "LLMs/WizardLM-13B-V1.0": ModelType.FT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.FT,
+    "wenge-research/yayi-7b": ModelType.FT,
+    "wenge-research/yayi-7b-llama2": ModelType.FT,
+    "wenge-research/yayi-13b-llama2": ModelType.FT,
+    "yhyhy3/open_llama_7b_v2_med_instruct": ModelType.IFT,
+    "llama-anon/instruct-13b": ModelType.IFT,
+    "huggingtweets/jerma985": ModelType.FT,
+    "huggingtweets/gladosystem": ModelType.FT,
+    "huggingtweets/bladeecity-jerma985": ModelType.FT,
+    "huggyllama/llama-13b": ModelType.PT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "FabbriSimo01/Facebook_opt_1.3b_Quantized": ModelType.PT,
+    "upstage/Llama-2-70b-instruct": ModelType.IFT,
+    "upstage/Llama-2-70b-instruct-1024": ModelType.IFT,
+    "upstage/llama-65b-instruct": ModelType.IFT,
+    "upstage/llama-30b-instruct-2048": ModelType.IFT,
+    "upstage/llama-30b-instruct": ModelType.IFT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.1": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.2": ModelType.IFT,
+    "WizardLM/WizardLM-30B-V1.0": ModelType.IFT,
+    "WizardLM/WizardCoder-15B-V1.0": ModelType.IFT,
+    "gpt2": ModelType.PT,
+    "keyfan/vicuna-chinese-replication-v1.1": ModelType.IFT,
+    "nthngdy/pythia-owt2-70m-100k": ModelType.FT,
+    "nthngdy/pythia-owt2-70m-50k": ModelType.FT,
+    "quantumaikr/KoreanLM-hf": ModelType.FT,
+    "quantumaikr/open_llama_7b_hf": ModelType.FT,
+    "quantumaikr/QuantumLM-70B-hf": ModelType.IFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.FT,
+    "MayaPH/FinOPT-Franklin": ModelType.FT,
+    "MayaPH/GodziLLa-30B": ModelType.IFT,
+    "MayaPH/GodziLLa-30B-plus": ModelType.IFT,
+    "MayaPH/FinOPT-Washington": ModelType.FT,
+    "ogimgio/gpt-neo-125m-neurallinguisticpioneers": ModelType.FT,
+    "layoric/llama-2-13b-code-alpaca": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v2": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v3": ModelType.FT,
+    "timdettmers/guanaco-33b-merged": ModelType.FT,
+    "elinas/chronos-33b": ModelType.IFT,
+    "heegyu/RedTulu-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-3B-0719": ModelType.IFT,
+    "meta-llama/Llama-2-7b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-7b-hf": ModelType.PT,
+    "meta-llama/Llama-2-13b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-13b-hf": ModelType.PT,
+    "meta-llama/Llama-2-70b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-70b-hf": ModelType.PT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-20b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt": ModelType.FT,
+    "h2oai/h2ogpt-oig-oasst1-512-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-oasst1-512-12b": ModelType.IFT,
+    "h2oai/h2ogpt-oig-oasst1-256-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt": ModelType.FT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b": ModelType.FT,
+    "bofenghuang/vigogne-13b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-13b-chat": ModelType.FT,
+    "bofenghuang/vigogne-2-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-chat": ModelType.FT,
+    "Vmware/open-llama-7b-v2-open-instruct": ModelType.IFT,
+    "VMware/open-llama-0.7T-7B-open-instruct-v1.1": ModelType.IFT,
+    "ewof/koishi-instruct-3b": ModelType.IFT,
+    "gywy/llama2-13b-chinese-v1": ModelType.FT,
+    "GOAT-AI/GOAT-7B-Community": ModelType.FT,
+    "psyche/kollama2-7b": ModelType.FT,
+    "TheTravellingEngineer/llama2-7b-hf-guanaco": ModelType.FT,
+    "beaugogh/pythia-1.4b-deduped-sharegpt": ModelType.FT,
+    "augtoma/qCammel-70-x": ModelType.IFT,
+    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "anhnv125/pygmalion-6b-roleplay": ModelType.FT,
+    "64bits/LexPodLM-13B": ModelType.FT,
+}
+
+
+def model_type_from_str(type):
+    if "fine-tuned" in type or "🔶" in type:
+        return ModelType.FT
+    if "pretrained" in type or "🟢" in type:
+        return ModelType.PT
+    if "RL-tuned" in type or "🟦" in type:
+        return ModelType.RL
+    if "instruction-tuned" in type or "⭕" in type:
+        return ModelType.IFT
+    return ModelType.Unknown

src/{auto_leaderboard/load_results.py → display_models/read_results.py}

@@ -1,14 +1,13 @@
-from dataclasses import dataclass
-
-import glob
 import json
 import os
+from dataclasses import dataclass
 from typing import Dict, List, Tuple
-import dateutil
 
-from src.utils_display import AutoEvalColumn, make_clickable_model
+import dateutil
 import numpy as np
 
+from src.display_models.utils import AutoEvalColumn, make_clickable_model
+
 METRICS = ["acc_norm", "acc_norm", "acc", "mc2"]
 BENCHMARKS = ["arc:challenge", "hellaswag", "hendrycksTest", "truthfulqa:mc"]
 BENCH_TO_NAME = {
@@ -31,13 +30,15 @@ class EvalResult:
     weight_type: str = ""
 
     def to_dict(self):
+        from src.load_from_hub import is_model_on_hub
+
         if self.org is not None:
             base_model = f"{self.org}/{self.model}"
         else:
             base_model = f"{self.model}"
         data_dict = {}
 
-        data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict["eval_name"] = self.eval_name  # not a column, just a save name
         data_dict["weight_type"] = self.weight_type  # not a column, just a save name
         data_dict[AutoEvalColumn.precision.name] = self.precision
         data_dict[AutoEvalColumn.model_type.name] = self.model_type
@@ -45,6 +46,9 @@ class EvalResult:
         data_dict[AutoEvalColumn.dummy.name] = base_model
         data_dict[AutoEvalColumn.revision.name] = self.revision
         data_dict[AutoEvalColumn.average.name] = sum([v for k, v in self.results.items()]) / 4.0
+        data_dict[AutoEvalColumn.still_on_hub.name] = (
+            is_model_on_hub(base_model, self.revision)[0] or base_model == "baseline"
+        )
 
         for benchmark in BENCHMARKS:
             if benchmark not in self.results.keys():
@@ -60,10 +64,9 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     with open(json_filepath) as fp:
         data = json.load(fp)
 
-    
     for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
         if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
-            return None, [] # we skip models with the wrong version 
+            return None, []  # we skip models with the wrong version
 
     try:
         config = data["config"]
@@ -87,17 +90,24 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     else:
         org = model_split[0]
         model = model_split[1]
-        result_key =  f"{org}_{model}_{model_sha}_{precision}"
+        result_key = f"{org}_{model}_{model_sha}_{precision}"
 
     eval_results = []
     for benchmark, metric in zip(BENCHMARKS, METRICS):
-        accs = np.array([v.get(metric, 0) for k, v in data["results"].items() if benchmark in k])
-        if accs.size == 0:
+        accs = np.array([v.get(metric, None) for k, v in data["results"].items() if benchmark in k])
+        if accs.size == 0 or any([acc is None for acc in accs]):
             continue
         mean_acc = np.mean(accs) * 100.0
-        eval_results.append(EvalResult(
-            eval_name=result_key, org=org, model=model, revision=model_sha, results={benchmark: mean_acc}, precision=precision, #todo model_type=, weight_type=
-        ))
+        eval_results.append(
+            EvalResult(
+                eval_name=result_key,
+                org=org,
+                model=model,
+                revision=model_sha,
+                results={benchmark: mean_acc},
+                precision=precision,  # todo model_type=, weight_type=
+            )
+        )
 
     return result_key, eval_results
 
@@ -113,11 +123,11 @@ def get_eval_results() -> List[EvalResult]:
         # Sort the files by date
         # store results by precision maybe?
         try:
-            files.sort(key=lambda x:  dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
+            files.sort(key=lambda x: dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
-        #up_to_date = files[-1]
+        # up_to_date = files[-1]
         for file in files:
             json_filepaths.append(os.path.join(root, file))
 

src/{utils_display.py → display_models/utils.py}

@@ -1,24 +1,27 @@
 import os
 from dataclasses import dataclass
+
 from huggingface_hub import HfApi
 
 API = HfApi()
 
 
-# These classes are for user facing column names, to avoid having to change them 
-# all around the code when a modif is needed 
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modif is needed
 @dataclass
 class ColumnContent:
     name: str
-    type: str 
-    displayed_by_default: bool 
+    type: str
+    displayed_by_default: bool
     hidden: bool = False
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
+
 @dataclass(frozen=True)
-class AutoEvalColumn: # Auto evals column
+class AutoEvalColumn:  # Auto evals column
     model_type_symbol = ColumnContent("T", "str", True)
     model = ColumnContent("Model", "markdown", True)
     average = ColumnContent("Average ⬆️", "number", True)
@@ -27,15 +30,19 @@ class AutoEvalColumn: # Auto evals column
     mmlu = ColumnContent("MMLU", "number", True)
     truthfulqa = ColumnContent("TruthfulQA", "number", True)
     model_type = ColumnContent("Type", "str", False)
-    precision = ColumnContent("Precision", "str", False) #, True)
+    precision = ColumnContent("Precision", "str", False)  # , True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)
     likes = ColumnContent("Hub ❤️", "number", False)
+    still_on_hub = ColumnContent("Available on the hub", "bool", False)
     revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+    dummy = ColumnContent(
+        "model_name_for_query", "str", True
+    )  # dummy col to implement search bar (hidden by custom CSS)
+
 
 @dataclass(frozen=True)
-class EloEvalColumn: # Elo evals column
+class EloEvalColumn:  # Elo evals column
     model = ColumnContent("Model", "markdown", True)
     gpt4 = ColumnContent("GPT-4 (all)", "number", True)
     human_all = ColumnContent("Human (all)", "number", True)
@@ -44,7 +51,7 @@ class EloEvalColumn: # Elo evals column
 
 
 @dataclass(frozen=True)
-class EvalQueueColumn: # Queue column
+class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
@@ -52,7 +59,13 @@ class EvalQueueColumn: # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+LLAMAS = [
+    "huggingface/llama-7b",
+    "huggingface/llama-13b",
+    "huggingface/llama-30b",
+    "huggingface/llama-65b",
+]
 
 
 KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
@@ -90,29 +103,44 @@ def make_clickable_model(model_name):
     elif model_name == "oasst-12b":
         link = OASST_LINK
 
-    details_model_name = model_name.replace('/', '__')
+    details_model_name = model_name.replace("/", "__")
     details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
 
     if not bool(os.getenv("DEBUG", "False")):
         # We only add these checks when not debugging, as they are extremely slow
         print(f"details_link: {details_link}")
         try:
-            check_path = list(API.list_files_info(repo_id=f"open-llm-leaderboard/details_{details_model_name}",
-                                                paths="README.md",
-                                                repo_type="dataset"))
+            check_path = list(
+                API.list_files_info(
+                    repo_id=f"open-llm-leaderboard/details_{details_model_name}",
+                    paths="README.md",
+                    repo_type="dataset",
+                )
+            )
             print(f"check_path: {check_path}")
         except Exception as err:
             # No details repo for this model
             print(f"No details repo for this model: {err}")
             return model_hyperlink(link, model_name)
 
-    return model_hyperlink(link, model_name) + '  ' + model_hyperlink(details_link, "📑")
+    return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
+
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 
+
 def styled_warning(warn):
     return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
 
+
 def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/init.py

@@ -1,51 +0,0 @@
-import os 
-from huggingface_hub import Repository
-
-
-def get_all_requested_models(requested_models_dir):
-    depth = 1
-    file_names = []
-
-    for root, dirs, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            file_names.extend([os.path.join(root, file) for file in files])
-
-    return set([file_name.lower().split("eval-queue/")[1] for file_name in file_names])
-
-def load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, QUEUE_PATH, RESULTS_PATH):
-    eval_queue_repo = None
-    eval_results_repo = None
-    requested_models = None
-
-    print("Pulling evaluation requests and results.")
-
-    eval_queue_repo = Repository(
-        local_dir=QUEUE_PATH,
-        clone_from=QUEUE_REPO,
-        repo_type="dataset",
-    )
-    eval_queue_repo.git_pull()
-
-    eval_results_repo = Repository(
-        local_dir=RESULTS_PATH,
-        clone_from=RESULTS_REPO,
-        repo_type="dataset",
-    )
-    eval_results_repo.git_pull()
-
-    requested_models = get_all_requested_models("eval-queue")
-
-    return eval_queue_repo, requested_models, eval_results_repo
-
-
-#def load_results(model, benchmark, metric):
-#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
-#    if not os.path.exists(file_path):
-#        return 0.0, None
-
-#    with open(file_path) as fp:
-#        data = json.load(fp)
-#    accs = np.array([v[metric] for k, v in data["results"].items()])
-#    mean_acc = np.mean(accs)
-#    return mean_acc, data["config"]["model_args"]

src/load_from_hub.py

@@ -0,0 +1,139 @@
+import json
+import os
+
+import pandas as pd
+from huggingface_hub import Repository
+from transformers import AutoConfig
+
+from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
+from src.display_models.get_model_metadata import apply_metadata
+from src.display_models.read_results import get_eval_results_dicts, make_clickable_model
+from src.display_models.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+
+def get_all_requested_models(requested_models_dir: str) -> set[str]:
+    depth = 1
+    file_names = []
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            file_names.extend([os.path.join(root, file) for file in files])
+
+    return set([file_name.lower().split("eval-queue/")[1] for file_name in file_names])
+
+
+def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
+    eval_queue_repo = None
+    eval_results_repo = None
+    requested_models = None
+
+    print("Pulling evaluation requests and results.")
+
+    eval_queue_repo = Repository(
+        local_dir=QUEUE_PATH,
+        clone_from=QUEUE_REPO,
+        repo_type="dataset",
+    )
+    # eval_queue_repo.git_pull()
+
+    eval_results_repo = Repository(
+        local_dir=RESULTS_PATH,
+        clone_from=RESULTS_REPO,
+        repo_type="dataset",
+    )
+    # eval_results_repo.git_pull()
+
+    requested_models = get_all_requested_models("eval-queue")
+
+    return eval_queue_repo, requested_models, eval_results_repo
+
+
+def get_leaderboard_df(
+    eval_results: Repository, eval_results_private: Repository, cols: list, benchmark_cols: list
+) -> pd.DataFrame:
+    if eval_results:
+        print("Pulling evaluation results for the leaderboard.")
+        # eval_results.git_pull()
+    if eval_results_private:
+        print("Pulling evaluation results for the leaderboard.")
+        # eval_results_private.git_pull()
+
+    all_data = get_eval_results_dicts()
+
+    # if not IS_PUBLIC:
+    all_data.append(gpt4_values)
+    all_data.append(gpt35_values)
+
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(
+    eval_queue: Repository, eval_queue_private: Repository, save_path: str, cols: list
+) -> list[pd.DataFrame]:
+    if eval_queue:
+        print("Pulling changes for the evaluation queue.")
+        # eval_queue.git_pull()
+    if eval_queue_private:
+        print("Pulling changes for the evaluation queue.")
+        # eval_queue_private.git_pull()
+
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+def is_model_on_hub(model_name: str, revision: str) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
+        return True, None
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+        )
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"