Update space

README.md

@@ -1,20 +1,32 @@
 ---
-title: HumanLikeness
+title: Humanlike Evaluation Leaderboard
 emoji: 🥇
-colorFrom: green
+colorFrom: blue
 colorTo: indigo
 sdk: gradio
+sdk_version: 4.37.1
 app_file: app.py
 pinned: true
-license: mit
+license: apache-2.0
+tags:
+- leaderboard
+models:
+- google/gemma-2-9b
 ---
 
-# Start the configuration
 
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+python>3.10
+pip spacy
+python -m spacy download en_core_web_sm
+pip install google.generativeai
+python -m spacy download en_core_web_trf
 
-Results files should have the following format and be stored as json files:
-```json
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+Most of the variables to change for a default leaderboard are in env (replace the path for your leaderboard) and src/display/about.
+
+Results files should have the following format:
+```
 {
     "config": {
         "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
@@ -32,13 +44,4 @@ Results files should have the following format and be stored as json files:
 }
 ```
 
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+Request files are created automatically by this tool.

app.py

@@ -1,110 +1,234 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
+import src.display.about as about
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+import src.display.utils as utils
+import src.envs as envs
+import src.populate as populate
+import src.submission.submit as submit
 
 
 def restart_space():
-    API.restart_space(repo_id=REPO_ID)
+    envs.API.restart_space(repo_id=envs.REPO_ID, token=envs.TOKEN)
 
-### Space initialisation
 try:
-    print(EVAL_REQUESTS_PATH)
+    print(envs.EVAL_REQUESTS_PATH)
     snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
     )
 except Exception:
     restart_space()
 try:
-    print(EVAL_RESULTS_PATH)
+    print(envs.EVAL_RESULTS_PATH)
     snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=envs.RESULTS_REPO, local_dir=envs.EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
     )
 except Exception:
     restart_space()
 
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+raw_data, original_df = populate.get_leaderboard_df(envs.EVAL_RESULTS_PATH, envs.EVAL_REQUESTS_PATH, utils.COLS, utils.BENCHMARK_COLS)
+leaderboard_df = original_df.copy()
 
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
+) = populate.get_evaluation_queue_df(envs.EVAL_REQUESTS_PATH, utils.EVAL_COLS)
+
+
+# Searching and filtering
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    type_query: list,
+    precision_query: str,
+    size_query: list,
+    show_deleted: bool,
+    query: str,
+):
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_queries(query, filtered_df)
+    df = select_columns(filtered_df, columns)
+    return df
+
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[utils.AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        utils.AutoEvalColumn.model_type_symbol.name,
+        utils.AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in utils.COLS if c in df.columns and c in columns] + [utils.AutoEvalColumn.dummy.name]
+    ]
+    return filtered_df
+
+
+def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[utils.AutoEvalColumn.model.name, utils.AutoEvalColumn.precision.name, utils.AutoEvalColumn.revision.name]
+            )
+
+    return filtered_df
+
+
+def filter_models(
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+) -> pd.DataFrame:
+    # Show all models
+    # if show_deleted:
+    #   filtered_df = df
+    # else:  # Show only still on the hub models
+        # filtered_df = df[df[utils.AutoEvalColumn.still_on_hub.name]]
+
+    filtered_df = df
+    
+    type_emoji = [t[0] for t in type_query]
+    filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+
+    numeric_interval = pd.IntervalIndex(sorted([utils.NUMERIC_INTERVALS[s] for s in size_query]))
+    params_column = pd.to_numeric(df[utils.AutoEvalColumn.params.name], errors="coerce")
+    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    filtered_df = filtered_df.loc[mask]
+
+    return filtered_df
 
 
 demo = gr.Blocks(css=custom_css)
 with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.HTML(about.TITLE)
+    gr.Markdown(about.INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c.name
+                                for c in utils.fields(utils.AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden and not c.dummy
+                            ],
+                            value=[
+                                c.name
+                                for c in utils.fields(utils.AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    with gr.Row():
+                        deleted_models_visibility = gr.Checkbox(
+                            value=False, label="Show gated/private/deleted models", interactive=True
+                        )
+                with gr.Column(min_width=320):
+                    #with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model types",
+                        choices=[t.to_str() for t in utils.ModelType],
+                        value=[t.to_str() for t in utils.ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_precision = gr.CheckboxGroup(
+                        label="Precision",
+                        choices=[i.value.name for i in utils.Precision],
+                        value=[i.value.name for i in utils.Precision],
+                        interactive=True,
+                        elem_id="filter-columns-precision",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(utils.NUMERIC_INTERVALS.keys()),
+                        value=list(utils.NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df[
+                    [c.name for c in utils.fields(utils.AutoEvalColumn) if c.never_hidden]
+                    + shown_columns.value
+                    + [utils.AutoEvalColumn.dummy.name]
+                ],
+                headers=[c.name for c in utils.fields(utils.AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=utils.TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                column_widths=["2%", "33%"]
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[utils.COLS],
+                headers=utils.COLS,
+                datatype=utils.TYPES,
+                visible=False,
+            )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_columns_type,
+                        filter_columns_precision,
+                        filter_columns_size,
+                        deleted_models_visibility,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+            gr.Markdown(about.LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+                    gr.Markdown(about.EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
                     with gr.Accordion(
@@ -114,8 +238,8 @@ with demo:
                         with gr.Row():
                             finished_eval_table = gr.components.Dataframe(
                                 value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
+                                headers=utils.EVAL_COLS,
+                                datatype=utils.EVAL_TYPES,
                                 row_count=5,
                             )
                     with gr.Accordion(
@@ -125,8 +249,8 @@ with demo:
                         with gr.Row():
                             running_eval_table = gr.components.Dataframe(
                                 value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
+                                headers=utils.EVAL_COLS,
+                                datatype=utils.EVAL_TYPES,
                                 row_count=5,
                             )
 
@@ -137,8 +261,8 @@ with demo:
                         with gr.Row():
                             pending_eval_table = gr.components.Dataframe(
                                 value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
+                                headers=utils.EVAL_COLS,
+                                datatype=utils.EVAL_TYPES,
                                 row_count=5,
                             )
             with gr.Row():
@@ -149,7 +273,7 @@ with demo:
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[t.to_str(" : ") for t in utils.ModelType if t != utils.ModelType.Unknown],
                         label="Model type",
                         multiselect=False,
                         value=None,
@@ -158,14 +282,14 @@ with demo:
 
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        choices=[i.value.name for i in utils.Precision if i != utils.Precision.Unknown],
                         label="Precision",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
+                        choices=[i.value.name for i in utils.WeightType],
                         label="Weights type",
                         multiselect=False,
                         value="Original",
@@ -176,7 +300,7 @@ with demo:
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
             submit_button.click(
-                add_new_eval,
+                submit.add_new_eval,
                 [
                     model_name_textbox,
                     base_model_name_textbox,
@@ -191,8 +315,8 @@ with demo:
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
+                value=about.CITATION_BUTTON_TEXT,
+                label=about.CITATION_BUTTON_LABEL,
                 lines=20,
                 elem_id="citation-button",
                 show_copy_button=True,
@@ -201,4 +325,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

requirements.txt

@@ -1,16 +1,17 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio==4.4.0
+gradio_client==0.7.0
 huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
+litellm==1.15.1
+matplotlib==3.7.1
+numpy==1.24.2
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.28.2
+tqdm==4.65.0
+transformers==4.35.2
 tokenizers>=0.15.0
-sentencepiece
+sentence-transformers==2.2.2

src/display/css_html_js.py

@@ -33,11 +33,17 @@ custom_css = """
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
 
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
 /* Limit the width of the first AutoEvalColumn so that names don't expand too much */
 table td:first-child,
 table th:first-child {

src/display/formatting.py

@@ -1,3 +1,12 @@
+import os
+from datetime import datetime, timezone
+
+from huggingface_hub import HfApi
+from huggingface_hub.hf_api import ModelInfo
+
+
+API = HfApi()
+
 def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.display.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -19,16 +19,18 @@ class ColumnContent:
     displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
+    dummy: bool = False
 
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent,
+                            ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent,
+                            ColumnContent("Model", "markdown", True, never_hidden=True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -39,6 +41,8 @@ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -91,6 +95,9 @@ class WeightType(Enum):
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
+    qt_8bit = ModelDetails("8bit")
+    qt_4bit = ModelDetails("4bit")
+    qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -98,13 +105,32 @@ class Precision(Enum):
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
+        if precision in ["8bit"]:
+            return Precision.qt_8bit
+        if precision in ["4bit"]:
+            return Precision.qt_4bit
+        if precision in ["GPTQ", "None"]:
+            return Precision.qt_GPTQ
         return Precision.Unknown
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -1,19 +1,25 @@
 import os
-
+import torch
 from huggingface_hub import HfApi
 
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# replace this with our token
+TOKEN = os.environ.get("HF_TOKEN", None)
+# print(TOKEN)
+# OWNER = "vectara"
+# REPO_ID = f"{OWNER}/Humanlike"
+# QUEUE_REPO = f"{OWNER}/requests"
+# RESULTS_REPO = f"{OWNER}/results"
+
+
+OWNER = "Simondon" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
+REPO_ID = f"{OWNER}/Humanlike"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 
-# If you setup a cache later, just change HF_HOME
+# print(RESULTS_REPO)
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
@@ -21,5 +27,20 @@ EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
+# print(EVAL_RESULTS_PATH)
+# exit()
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #"cpu"
 API = HfApi(token=TOKEN)
+
+DATASET_PATH = "./src/datasets/Material_Llama2_0603.xlsx" #experiment data
+PROMPT_PATH = "./src/datasets/prompt.xlsx" #prompt for each experiment
+HEM_PATH = 'vectara/hallucination_evaluation_model'
+HUMAN_DATA = "./src/datasets/human_data.csv" #experiment data
+ITEM_4_DATA = "./src/datasets/associataion_dataset.csv" #database
+ITEM_5_DATA = "./src/datasets/Items_5.csv" #experiment 5 need verb words
+
+# SYSTEM_PROMPT = "You are a chat bot answering questions using data. You must stick to the answers provided solely by the text in the passage provided."
+SYSTEM_PROMPT = "You are a participant of a psycholinguistic experiment. You will do a task on English language use."
+'''prompt'''
+# USER_PROMPT = "You are asked the question 'Provide a concise summary of the following passage, covering the core pieces of information described': "
+USER_PROMPT = ""

src/leaderboard/read_evals.py

@@ -1,35 +1,32 @@
 import glob
 import json
-import math
 import os
 from dataclasses import dataclass
 
-import dateutil
 import numpy as np
+import dateutil
 
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+import src.display.formatting as formatting
+import src.display.utils as utils
+import src.submission.check_validity as check_validity
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    precision: utils.Precision = utils.Precision.Unknown
+    model_type: utils.ModelType = utils.ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: utils.WeightType = utils.WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
     @classmethod
@@ -41,43 +38,35 @@ class EvalResult:
         config = data.get("config")
 
         # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
+        precision = utils.Precision.from_str(config.get("model_dtype"))
 
         # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
+        full_model = config.get("model_name", config.get("model_args", None))
+        org, model = full_model.split("/", 1) if "/" in full_model else (None, full_model)
 
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
+        if org:
             result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
+        else:
+            result_key = f"{model}_{precision.value.name}"
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        still_on_hub, _, model_config = check_validity.is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True,
+            test_tokenizer=False)
+
+        if model_config:
+            architecture = ";".join(getattr(model_config, "architectures", ["?"]))
+        else:
+            architecture = "?"
 
         # Extract results available in this file (some results are split in several files)
         results = {}
-        for task in Tasks:
+        for task in utils.Tasks:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
 
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            results[task.benchmark] = accs 
 
         return self(
             eval_name=result_key,
@@ -85,7 +74,7 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
+            precision=precision,
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
@@ -93,40 +82,43 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        request_file = get_request_file_for_model(requests_path, self.full_model,
+                                                self.precision.value.name)
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.model_type = utils.ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = utils.WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+        except FileNotFoundError:
+            print(f"Could not find request file for {self.org}/{self.model}")
+        except json.JSONDecodeError:
+            print(f"Error decoding JSON in request file for {self.org}/{self.model}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            utils.AutoEvalColumn.precision.name: self.precision.value.name,
+            utils.AutoEvalColumn.model_type.name: self.model_type.value.name,
+            utils.AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            utils.AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            utils.AutoEvalColumn.architecture.name: self.architecture,
+            utils.AutoEvalColumn.model.name: formatting.make_clickable_model(self.full_model),
+            utils.AutoEvalColumn.dummy.name: self.full_model,
+            utils.AutoEvalColumn.revision.name: self.revision,
+            utils.AutoEvalColumn.license.name: self.license,
+            utils.AutoEvalColumn.likes.name: self.likes,
+            utils.AutoEvalColumn.params.name: self.num_params,
+            utils.AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
+        for task in utils.Tasks:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
@@ -157,21 +149,26 @@ def get_request_file_for_model(requests_path, model_name, precision):
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
-
+    print("results_path", results_path)
     for root, _, files in os.walk(results_path):
         # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
+        print("file",files)
 
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
+        # if not files or any([not f.endswith(".json") for f in files]):
+            
+        #     continue
+        for f in files:
+            if f.endswith(".json"):
 
+        # Sort the files by date
+                # try:
+                #     files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+                # except dateutil.parser._parser.ParserError:
+                #     files = [files[-1]]
+
+                model_result_filepaths.extend([os.path.join(root, f)])
+    print("model_result_filepaths", model_result_filepaths)
+    # exit()
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
@@ -181,7 +178,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+            eval_results[eval_name].results.update({k: v for k, v in
+                                                    eval_result.results.items() if v is not None})
         else:
             eval_results[eval_name] = eval_result
 

src/populate.py

@@ -3,27 +3,28 @@ import os
 
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+import src.display.formatting as formatting
+import src.display.utils as utils
+import src.leaderboard.read_evals as read_evals
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = read_evals.get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
-
+    print(results_path, requests_path)
+    print(all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    print(df)
+    # exit()
+    df = df.sort_values(by=[utils.AutoEvalColumn.hallucination_rate.name], ascending=True)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+    df = df[formatting.has_no_nan_values(df, benchmark_cols)]
+    return raw_data, df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
@@ -33,8 +34,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[utils.EvalQueueColumn.model.name] = formatting.make_clickable_model(data["model"])
+            data[utils.EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -45,8 +46,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 with open(file_path) as fp:
                     data = json.load(fp)
 
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[utils.EvalQueueColumn.model.name] = formatting.make_clickable_model(data["model"])
+                data[utils.EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]

src/submission/check_validity.py

@@ -1,14 +1,12 @@
 import json
 import os
-import re
 from collections import defaultdict
-from datetime import datetime, timedelta, timezone
 
 import huggingface_hub
 from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
+from transformers import AutoConfig, AutoTokenizer
+from transformers.models.auto.tokenization_auto import tokenizer_class_from_name, get_tokenizer_config
 
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
@@ -31,8 +29,8 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
+
 def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
         if test_tokenizer:
@@ -56,7 +54,8 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         )
 
     except Exception as e:
-        return False, "was not found on hub!", None
+        return False, f"was not found on hub!: {e}", None
+
 
 
 def get_model_size(model_info: ModelInfo, precision: str):
@@ -75,7 +74,6 @@ def get_model_arch(model_info: ModelInfo):
     return model_info.config.get("architectures", "Unknown")
 
 def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
     depth = 1
     file_names = []
     users_to_submission_dates = defaultdict(list)

src/submission/submit.py

@@ -2,14 +2,10 @@ import json
 import os
 from datetime import datetime, timezone
 
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
+import src.display.formatting as formatting
+import src.envs as envs
+import src.submission.check_validity as check_validity
+
 
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
@@ -25,7 +21,7 @@ def add_new_eval(
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = check_validity.already_submitted_models(envs.EVAL_REQUESTS_PATH)
 
     user_name = ""
     model_path = model
@@ -37,7 +33,7 @@ def add_new_eval(
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
     if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
+        return formatting.styled_error("Please select a model type.")
 
     # Does the model actually exist?
     if revision == "":
@@ -45,32 +41,32 @@ def add_new_eval(
 
     # Is the model on the hub?
     if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        base_model_on_hub, error, _ = check_validity.is_model_on_hub(model_name=base_model, revision=revision, token=envs.TOKEN, test_tokenizer=True)
         if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+            return formatting.styled_error(f'Base model "{base_model}" {error}')
 
     if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        model_on_hub, error, _ = check_validity.is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
         if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+            return formatting.styled_error(f'Model "{model}" {error}')
 
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model, revision=revision)
+        model_info = envs.API.model_info(repo_id=model, revision=revision)
     except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
+        return formatting.styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = check_validity.get_model_size(model_info=model_info, precision=precision)
 
     # Were the model card and license filled?
     try:
         license = model_info.cardData["license"]
     except Exception:
-        return styled_error("Please select a license for your model")
+        return formatting.styled_error("Please select a license for your model")
 
-    modelcard_OK, error_msg = check_model_card(model)
+    modelcard_OK, error_msg = check_validity.check_model_card(model)
     if not modelcard_OK:
-        return styled_error(error_msg)
+        return formatting.styled_error(error_msg)
 
     # Seems good, creating the eval
     print("Adding new eval")
@@ -87,15 +83,15 @@ def add_new_eval(
         "likes": model_info.likes,
         "params": model_size,
         "license": license,
-        "private": False,
     }
 
     # Check for duplicate submission
     if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
+        return formatting.styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+
+    OUT_DIR = f"{envs.EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
 
@@ -103,10 +99,10 @@ def add_new_eval(
         f.write(json.dumps(eval_entry))
 
     print("Uploading eval file")
-    API.upload_file(
+    envs.API.upload_file(
         path_or_fileobj=out_path,
         path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
+        repo_id=envs.QUEUE_REPO,
         repo_type="dataset",
         commit_message=f"Add {model} to eval queue",
     )
@@ -114,6 +110,6 @@ def add_new_eval(
     # Remove the local file
     os.remove(out_path)
 
-    return styled_message(
+    return formatting.styled_message(
         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
     )