refactoring...

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.DS_Store

app.py

@@ -1,148 +1,75 @@
 import gradio as gr
 import pandas as pd
-import json
-from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
-from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
-from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
-from datetime import datetime, timezone
 
-LAST_UPDATED = "Feb 10th 2024"
+import constants
+from utils import display, eval_request
 
-column_names = {
-    "MODEL": "Model",
-    "Avg. WER": "Average WER ⬇️",    
-    "RTF": "RTF (1e-3) ⬇️",
-    "AMI WER": "AMI",
-    "Earnings22 WER": "Earnings22",
-    "Gigaspeech WER": "Gigaspeech",
-    "LS Clean WER": "LS Clean",
-    "LS Other WER": "LS Other",
-    "SPGISpeech WER": "SPGISpeech",
-    "Tedlium WER": "Tedlium",
-    "Voxpopuli WER": "Voxpopuli",
-    "Common Voice WER": "Common Voice 9"}
+#
+##
+###
+##
+#
 
-eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+leaderboard_df = pd.read_csv("leaderboard.csv")
 
-if not csv_results.exists():
-    raise Exception(f"CSV file {csv_results} does not exist locally")
-    
-# Get csv with data and parse columns
-original_df = pd.read_csv(csv_results)
-
-# Formats the columns
-def formatter(x):
-    if type(x) is str:
-        x = x
-    else: 
-        x = round(x, 2)
-    return x
-
-for col in original_df.columns:
+# Format the dataframe
+for col in leaderboard_df.columns:
     if col == "model":
-        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+        leaderboard_df[col] = leaderboard_df[col].apply(lambda x: x.replace(x, display.make_clickable_model(x)))
     else:
-        original_df[col] = original_df[col].apply(formatter) # For numerical values
- 
-original_df.rename(columns=column_names, inplace=True)
-original_df.sort_values(by='Average WER ⬇️', inplace=True)
-
-COLS = [c.name for c in fields(AutoEvalColumn)]
-TYPES = [c.type for c in fields(AutoEvalColumn)]
-
-
-def request_model(model_text, chbcoco2017):
-    
-    # Determine the selected checkboxes
-    dataset_selection = []
-    if chbcoco2017:
-        dataset_selection.append("ESB Datasets tests only")
-
-    if len(dataset_selection) == 0:
-        return styled_error("You need to select at least one dataset")
-        
-    base_model_on_hub, error_msg = is_model_on_hub(model_text)
+        leaderboard_df[col] = leaderboard_df[col].apply(display.round_numbers)
+leaderboard_df.rename(
+                columns={"Average WER": "Average WER ⬇️", "RTF (1e-3)": "RTF (1e-3) ⬇️"},
+                inplace=True,
+            )
+leaderboard_df.sort_values(by='Average WER ⬇️', inplace=True)
 
-    if not base_model_on_hub:
-        return styled_error(f"Base model '{model_text}' {error_msg}")
-    
-    # Construct the output dictionary
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    required_datasets = ', '.join(dataset_selection)
-    eval_entry = {
-        "date": current_time,
-        "model": model_text,
-        "datasets_selected": required_datasets
-    }
-    
-    # Prepare file path 
-    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
-    
-    fn_datasets = '@ '.join(dataset_selection)
-    filename = model_text.replace("/","@") + "@@" + fn_datasets 
-    if filename in requested_models:
-        return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.")
-    try:
-        filename_ext = filename + ".txt"
-        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
-
-        # Write the results to a text file
-        with open(out_filepath, "w") as f:
-            f.write(json.dumps(eval_entry))
-            
-        upload_file(filename, out_filepath)
-        
-        # Include file in the list of uploaded files
-        requested_models.append(filename)
-        
-        # Remove the local file
-        out_filepath.unlink()
+with gr.Blocks() as leaderboard_app:
 
-        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
-    except Exception as e:
-        return styled_error(f"Error submitting request!")
-
-with gr.Blocks() as demo:
-    gr.HTML(BANNER, elem_id="banner")
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.HTML(constants.BANNER, elem_id="constants.banner")
+    gr.Markdown(constants.INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
+
         with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
             leaderboard_table = gr.components.Dataframe(
-                value=original_df,
-                datatype=TYPES,
-                max_rows=None,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-                )
+                                    value=leaderboard_df,
+                                    datatype=constants.COLUMN_DTYPES_LIST,
+                                    elem_id="leaderboard-table",
+                                    interactive=False,
+                                    visible=True,
+                                )
 
         with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
-            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+            gr.Markdown(constants.METRICS_TAB_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
+        with gr.TabItem("✉️ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
             with gr.Column():
-                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️ Request results for a new model here!", elem_classes="markdown-text")
             with gr.Column():
-                gr.Markdown("Select a dataset:", elem_classes="markdown-text")
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
-                    chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False)
+                    model_id = gr.Textbox(label="Model ID (user_name/model_name)")
                 with gr.Column():
-                    mdw_submission_result = gr.Markdown()
+                    md_submission_result = gr.Markdown()
                     btn_submitt = gr.Button(value="🚀 Request")
-                    btn_submitt.click(request_model, 
-                                      [model_name_textbox, chb_coco2017], 
-                                      mdw_submission_result)
+                    btn_submitt.click(eval_request.request_model, [model_id], md_submission_result)
 
-    gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
+        with gr.TabItem("☢️ Evaluate", elem_id="od-benchmark-tab-table", id=3):
+            with gr.Column():
+                gr.Markdown("For admins only.", elem_classes="markdown-text")
+            with gr.Column():
+                md_submission_result = gr.Markdown()
+                btn_submitt = gr.Button(value="RUN EVALUATION")
+                # btn_submitt.click(eval_request.request_model, [model_id], md_submission_result)
     
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             gr.Textbox(
-                value=CITATION_TEXT, lines=7,
+                value=constants.CITATION_TEXT, lines=7,
                 label="Copy the BibTeX snippet to cite this source",
                 elem_id="citation-button",
-            ).style(show_copy_button=True)
+                show_label=True,
+                show_copy_button=True,
+            )
 
-demo.launch()
+leaderboard_app.launch(allowed_paths=["banner.png"])

constants.py

@@ -1,15 +1,19 @@
-from pathlib import Path
-
-# Directory where request by models are stored
-DIR_OUTPUT_REQUESTS = Path("requested_models")
-EVAL_REQUESTS_PATH = Path("eval_requests")
-
-##########################
-# Text definitions       #
-##########################
-
-banner_url = "https://huggingface.co/datasets/reach-vb/random-images/resolve/main/asr_leaderboard.png"
-BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+#
+##
+### CONSTANTS
+##
+#
+
+COLUMN_DTYPES_LIST = ["markdown", "number", "number", "number"]		# in accordance with ./leaderboard.csv
+
+#
+##
+### TEXT WALLS
+##
+#
+
+banner_url = "/file=banner.png"		# gradio file path
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 70%;"> </div>'
 
 TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Automatic Speech Recognition Leaderboard </b> </body> </html>"
 

init.py

@@ -1,93 +0,0 @@
-import os 
-from constants import EVAL_REQUESTS_PATH
-from pathlib import Path
-from huggingface_hub import HfApi, Repository
-
-TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
-QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
-QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
-
-hf_api = HfApi(
-    endpoint="https://huggingface.co", 
-    token=TOKEN_HUB, 
-)
-
-def load_all_info_from_dataset_hub():
-    eval_queue_repo = None
-    results_csv_path = None
-    requested_models = None
-
-    passed = True
-    if TOKEN_HUB is None:
-        passed = False
-    else:
-        print("Pulling evaluation requests and results.")
-
-        eval_queue_repo = Repository(
-            local_dir=QUEUE_PATH,
-            clone_from=QUEUE_REPO,
-            use_auth_token=TOKEN_HUB,
-            repo_type="dataset",
-        )
-        eval_queue_repo.git_pull()
-        
-        # Local directory where dataset repo is cloned + folder with eval requests
-        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
-        requested_models = get_all_requested_models(directory)
-        requested_models = [p.stem for p in requested_models]
-        # Local directory where dataset repo is cloned
-        csv_results = get_csv_with_results(QUEUE_PATH)
-        if csv_results is None:
-            passed = False
-    if not passed:
-        print("No HuggingFace token provided. Skipping evaluation requests and results.")
-
-    return eval_queue_repo, requested_models, csv_results
-
-
-def upload_file(requested_model_name, path_or_fileobj):
-    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
-    dest_repo_file = str(dest_repo_file)
-    hf_api.upload_file(
-            path_or_fileobj=path_or_fileobj,
-            path_in_repo=str(dest_repo_file),
-            repo_id=QUEUE_REPO,
-            token=TOKEN_HUB,
-            repo_type="dataset",
-            commit_message=f"Add {requested_model_name} to eval queue")
-
-def get_all_requested_models(directory):
-    directory = Path(directory)
-    all_requested_models = list(directory.glob("*.txt"))
-    return all_requested_models
-
-def get_csv_with_results(directory):
-    directory = Path(directory)
-    all_csv_files = list(directory.glob("*.csv"))
-    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
-    if len(latest) != 1:
-        return None
-    return latest[0]
-
-
-
-def is_model_on_hub(model_name, revision="main") -> bool:
-    try:
-        model_name = model_name.replace(" ","")
-        author = model_name.split("/")[0]
-        model_id = model_name.split("/")[1]
-        if len(author) == 0 or len(model_id) == 0:
-            return False, "is not a valid model name. Please use the format `author/model_name`."
-    except Exception as e:
-        return False, "is not a valid model name. Please use the format `author/model_name`."
-
-    try:
-        models = list(hf_api.list_models(author=author, search=model_id))
-        matched = [model_name for m in models if m.modelId == model_name]
-        if len(matched) != 1:
-            return False, "was not found on the hub!"
-        else:
-            return True, None
-    except Exception as e:
-        print(f"Could not get the model from the hub.: {e}")
-        return False, "was not found on hub!"

leaderboard.csv

@@ -0,0 +1,3 @@
+model,Average WER,RTF (1e-3),AMI WER,Earnings22 WER,Gigaspeech WER,LS Clean WER,LS Other WER,SPGISpeech WER,Tedlium WER,Voxpopuli WER,Common Voice WER
+openai/whisper-large-v3,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.1,1.2
+nvidia/canary-1b,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.1

requested_models.txt

@@ -0,0 +1,3 @@
+nvidia/parakeet-ctc-0.6b
+openai/whisper-large
+distil-whisper/distil-medium.en

utils/display.py

@@ -0,0 +1,10 @@
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def round_numbers(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x

utils/eval_request.py

@@ -0,0 +1,62 @@
+import os
+import huggingface_hub as hf_hub
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+hf_api = hf_hub.HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+
+#
+##
+###
+##
+#
+
+
+def is_model_on_hub(model_id):
+    """Check if a model is on the hub and return a failure message if not."""
+
+    try:
+        author = model_id.split("/")[0]
+        model_name = model_id.split("/")[1]
+        if len(author) == 0 or len(model_name) == 0:
+            return "is not a valid model name. Please use the format `author/model_name`."
+    except:
+        return "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_name))
+        matched = [model_id for m in models if m.modelId == model_id]
+        if len(matched) != 1:
+            return "was not found on the hub!"
+        else:
+            return None
+    except:
+        return "was not found on hub!"
+
+def request_model(model_id):
+
+    def styled_error(error):
+        return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+    def styled_message(message):
+        return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    
+    # Check if the model is on the hub
+    error_msg = is_model_on_hub(model_id)
+    if error_msg != None:
+        return styled_error(f"{model_id} {error_msg}")
+    
+    # Check if the model was already requested
+    with open("requested_models.txt", "r") as f:
+        requested_models_list = f.read().splitlines()
+    if model_id in requested_models_list:
+        return styled_error(f"A request for {model_id} was already made.")
+    
+    # Add the model to the evaluation queue
+    with open("requested_models.txt", "a") as f:
+        f.write(model_id + "\n")
+
+    return styled_message("🤗 Your request has been submitted and will be evaluated as soon as possible!")

utils_display.py

@@ -1,40 +0,0 @@
-from dataclasses import dataclass
-
-# These classes are for user facing column names, to avoid having to change them 
-# all around the code when a modif is needed 
-@dataclass
-class ColumnContent:
-    name: str
-    type: str 
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-@dataclass(frozen=True)
-class AutoEvalColumn: # Auto evals column
-    model = ColumnContent("Model", "markdown")
-    avg_wer = ColumnContent("Average WER ⬇️", "number")    
-    rtf = ColumnContent("RTF (1e-3) ⬇️", "number")
-    ami_wer = ColumnContent("AMI", "number")
-    e22_wer = ColumnContent("Earnings22", "number")
-    gs_wer = ColumnContent("Gigaspeech", "number")
-    lsc_wer = ColumnContent("LS Clean", "number")
-    lso_wer = ColumnContent("LS Other", "number")
-    ss_wer = ColumnContent("SPGISpeech", "number")
-    tl_wer = ColumnContent("Tedlium", "number")
-    vp_wer = ColumnContent("Voxpopuli", "number")
-    cv_wer = ColumnContent("Common Voice", "number")                       
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"