app.py

import os
os.system("pip install gradio==4.31.5 pydantic==2.7.1 gradio_modal==0.0.3")

import gradio as gr
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import snapshot_download
from gradio_space_ci import enable_space_ci

from src.display.about import (
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    EVALUATION_QUEUE_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    FAQ_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css
from src.display.utils import (
    BENCHMARK_COLS,
    COLS,
    EVAL_COLS,
    EVAL_TYPES,
    NUMERIC_INTERVALS,
    NUMERIC_MODELSIZE,
    TYPES,
    AutoEvalColumn,
    GroupDtype,
    ModelType,
    fields,
    WeightType,
    Precision,
    ComputeDtype,
    WeightDtype,
    QuantType
)
from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO, REPO, GIT_REQUESTS_PATH, GIT_STATUS_PATH, GIT_RESULTS_PATH
from src.populate import get_evaluation_queue_df, get_leaderboard_df
from src.submission.submit import add_new_eval
from src.scripts.update_all_request_files import update_dynamic_files
from src.tools.collections import update_collections
from src.tools.plots import (
    create_metric_plot_obj,
    create_plot_df,
    create_scores_df,
)
from gradio_modal import Modal
import plotly.graph_objects as go

selected_indices = []

# Start ephemeral Spaces on PRs (see config in README.md)
#enable_space_ci()

precision_to_dtype = {
    "2bit": ["int2"],
    "3bit": ["int3"],
    "4bit": ["int4", "nf4", "fp4"],
    "?": ["?"]
}

current_weightDtype = ["All", "int2", "int3", "int4", "nf4", "fp4", "?"]

# Global variable to store the selected dtypes
selected_dtypes = ["All"]
init_select = False

def quant_update_Weight_Dtype(selected_precisions):
    global current_weightDtype
    if '✖ None' in selected_precisions:
        if not any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype):
            current_weightDtype += ['float16', 'bfloat16', 'float32']
    else:
        if any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype):
            current_weightDtype = [dtype for dtype in current_weightDtype if dtype not in ['float16', 'bfloat16', 'float32']]
    return gr.Dropdown.update(choices=current_weightDtype, value="All")


def update_Weight_Dtype(selected_precisions):
    global selected_dtypes
    global current_weightDtype
    global init_select
    init_select = True

    if not selected_precisions:  # If no precision is selected, return "All"
        selected_dtypes = ["All"]
        return gr.Dropdown.update(choices=["All"], value="All")

    selected_dtypes_set = set()
    for precision in selected_precisions:
        if precision in precision_to_dtype:
            selected_dtypes_set.update(precision_to_dtype[precision])

    
    # Convert set to sorted list to maintain order
    selected_dtypes = sorted(selected_dtypes_set)
    if any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype) and not any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in selected_dtypes):
        selected_dtypes += ['float16', 'bfloat16', 'float32']
    # Add "All" to the beginning of the list for display purposes
    display_choices = ["All"] + selected_dtypes


    current_weightDtype = display_choices
    return gr.Dropdown.update(choices=display_choices, value="All")



def restart_space():
    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)


def init_space(full_init: bool = True):
    if full_init:
        try:
            branch = REPO.active_branch.name
            REPO.remotes.origin.pull(branch)
        except Exception as e:
            print(str(e))
            restart_space()

        try:
            print(DYNAMIC_INFO_PATH)
            snapshot_download(
                repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
            )
        except Exception:
            restart_space()

    raw_data, original_df = get_leaderboard_df(
        results_path=GIT_RESULTS_PATH, 
        requests_path=GIT_STATUS_PATH, 
        dynamic_path=DYNAMIC_INFO_FILE_PATH, 
        cols=COLS, 
        benchmark_cols=BENCHMARK_COLS
    )
    # update_collections(original_df.copy())
    leaderboard_df = original_df.copy()

    plot_df = create_plot_df(create_scores_df(raw_data))

    (
        finished_eval_queue_df,
        running_eval_queue_df,
        pending_eval_queue_df,
    ) = get_evaluation_queue_df(GIT_STATUS_PATH, EVAL_COLS)

    return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df

leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()

def str_to_bool(value):
    if str(value).lower() == "true":
        return True
    elif str(value).lower() == "false":
        return False
    else:
        return False

# Searching and filtering
def update_table(
    hidden_df: pd.DataFrame,
    columns: list,
    type_query: list,
    precision_query: str,
    size_query: list,
    params_query: list,
    hide_models: list,
    query: str,
    compute_dtype: str,
    weight_dtype: str,
    double_quant: str,
    group_dtype: str
):
    global init_select
    global current_weightDtype


    if selected_dtypes == ['All']:
        weight_dtype = current_weightDtype
    elif weight_dtype == ['All'] or weight_dtype == 'All' or init_select:
        weight_dtype = selected_dtypes
        init_select = False
    else:
        weight_dtype = [weight_dtype]

    if compute_dtype == 'All':
        compute_dtype = ['bfloat16', 'float16', 'int8', 'float32']
    else:
        compute_dtype = [compute_dtype]   
        
    if group_dtype == 'All':
        group_dtype = [-1, 1024, 256, 128, 64, 32]
    else:
        try:
            group_dtype = [int(group_dtype)]
        except ValueError:
            group_dtype = [-1]

    double_quant = [str_to_bool(double_quant)]
    filtered_df = filter_models(df=hidden_df, type_query=type_query, size_query=size_query, precision_query=precision_query, hide_models=hide_models, compute_dtype=compute_dtype, weight_dtype=weight_dtype, double_quant=double_quant, group_dtype=group_dtype, params_query=params_query)
    filtered_df = filter_queries(query, filtered_df)
    df = select_columns(filtered_df, columns)
    return df


def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
    query = request.query_params.get("query") or ""
    return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed


def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]


def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
    dummy_col = [AutoEvalColumn.dummy.name]
    # We use COLS to maintain sorting
    filtered_df = df[
        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col
    ]
    return filtered_df


def filter_queries(query: str, filtered_df: pd.DataFrame):
    """Added by Abishek"""
    final_df = []
    if query != "":
        queries = [q.strip() for q in query.split(";")]
        for _q in queries:
            _q = _q.strip()
            if _q != "":
                temp_filtered_df = search_table(filtered_df, _q)
                if len(temp_filtered_df) > 0:
                    final_df.append(temp_filtered_df)
        if len(final_df) > 0:
            filtered_df = pd.concat(final_df)
            filtered_df = filtered_df.drop_duplicates(
                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
            )

    return filtered_df


def filter_models(
    df: pd.DataFrame, type_query: list, size_query: list, params_query:list, precision_query: list, hide_models: list, compute_dtype: list, weight_dtype: list, double_quant: list, group_dtype: list,
 ) -> pd.DataFrame:
    # Show all models
    if "Private or deleted" in hide_models:
        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
    else:
        filtered_df = df

    if "Contains a merge/moerge" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]

    if "MoE" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.moe.name] == False]

    if "Flagged" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]

    type_emoji = [t[0] for t in type_query]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
    
    filtered_df = filtered_df.loc[df[AutoEvalColumn.weight_dtype.name].isin(weight_dtype)]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.compute_dtype.name].isin(compute_dtype)]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.double_quant.name].isin(double_quant)]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.group_size.name].isin(group_dtype)]

    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
    filtered_df = filtered_df.loc[mask]

    numeric_interval_params = pd.IntervalIndex(sorted([NUMERIC_MODELSIZE[s] for s in params_query]))
    params_column_params = pd.to_numeric(df[AutoEvalColumn.model_size.name], errors="coerce")
    mask_params = params_column_params.apply(lambda x: any(numeric_interval_params.contains(x)))
    filtered_df = filtered_df.loc[mask_params]

    return filtered_df

def select(df, data: gr.SelectData):
    global selected_indices
    selected_index = data.index[0]  
    
    if selected_index in selected_indices:
        selected_indices.remove(selected_index) 
    else:
        selected_indices.append(selected_index)  

    fig = go.Figure()
    for i in selected_indices:
        row = df.iloc[i, :]
        fig.add_trace(go.Scatterpolar(
            r=[row['Average ⬆️'], row['ARC-c'], row['ARC-e'], row['Boolq'], row['HellaSwag'], row['Lambada'], row['MMLU'], row['Openbookqa'], row['Piqa'], row['Truthfulqa'], row['Winogrande']],
            theta=['Average ⬆️', 'ARC-c', 'ARC-e', 'Boolq', 'HellaSwag', 'Lambada', 'MMLU', 'Openbookqa', 'Piqa', 'Truthfulqa', 'Winogrande',],
            fill='toself',
            name=str(row['Model'])  
        ))
    fig.update_layout(
        polar=dict(
            radialaxis=dict(
                visible=True,
            )),
        showlegend=True
    )
    

    return fig    

leaderboard_df = filter_models(
    df=leaderboard_df, 
    type_query=[t.to_str(" : ") for t in QuantType], 
    size_query=list(NUMERIC_INTERVALS.keys()), 
    params_query=list(NUMERIC_MODELSIZE.keys()),
    precision_query=[i.value.name for i in Precision],
    hide_models=["Private or deleted", "Contains a merge/moerge", "Flagged"], # Deleted, merges, flagged, MoEs,
    compute_dtype=[i.value.name for i in ComputeDtype],
    weight_dtype=[i.value.name for i in WeightDtype],
    double_quant=[True, False],
    group_dtype=[-1, 1024, 256, 128, 64, 32]
)

demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
            with gr.Row():
                with gr.Column():
                    with gr.Row():
                        search_bar = gr.Textbox(
                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                            show_label=False,
                            elem_id="search-bar",
                        )
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[
                                c.name
                                for c in fields(AutoEvalColumn)
                                if not c.hidden and not c.never_hidden and not c.dummy
                            ],
                            value=[
                                c.name
                                for c in fields(AutoEvalColumn)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )

                    with gr.Row():
                        filter_columns_parameters = gr.CheckboxGroup(
                        label="Model parameters (in billions of parameters)",
                        choices=list(NUMERIC_INTERVALS.keys()),
                        value=list(NUMERIC_INTERVALS.keys()),
                        interactive=True,
                        elem_id="filter-columns-size",
                    )
                    with gr.Row():
                        filter_columns_size = gr.CheckboxGroup(
                        label="Model sizes (GB, int4)",
                        choices=list(NUMERIC_MODELSIZE.keys()),
                        value=list(NUMERIC_MODELSIZE.keys()),
                        interactive=True,
                        elem_id="filter-columns-size",
                    )
                with gr.Column(min_width=320):
                    #with gr.Box(elem_id="box-filter"):
                    filter_columns_type = gr.CheckboxGroup(
                        label="Quantization types",
                        choices=[t.to_str() for t in QuantType],
                        value=[t.to_str() for t in QuantType if t != QuantType.QuantType_None],
                        interactive=True,
                        elem_id="filter-columns-type",  
                    )
                    filter_columns_precision = gr.CheckboxGroup(
                        label="Weight precision",
                        choices=[i.value.name for i in Precision],
                        value=[i.value.name for i in Precision],
                        interactive=True,
                        elem_id="filter-columns-precision",
                    )
                    with gr.Group() as config:
                        # gr.HTML("""<p style='padding-bottom: 0.5rem; color: #6b7280; '>Quantization config</p>""")
                        gr.HTML("""<p style='padding: 0.7rem; background: #fff; margin: 0; color: #6b7280;'>Quantization config</p>""")
                        with gr.Row():
                            filter_columns_computeDtype = gr.Dropdown(choices=[i.value.name for i in ComputeDtype], label="Compute Dtype", multiselect=False, value="All", interactive=True,)
                            filter_columns_weightDtype = gr.Dropdown(choices=[i.value.name for i in WeightDtype], label="Weight Dtype", multiselect=False, value="All", interactive=True,)
                            filter_columns_doubleQuant = gr.Dropdown(choices=["True", "False"], label="Double Quant", multiselect=False, value=False, interactive=True)
                            filter_columns_groupDtype = gr.Dropdown(choices=[i.value.name for i in GroupDtype], label="Group Size", multiselect=False, value="All", interactive=True,)

            leaderboard_table = gr.components.Dataframe(
                value=leaderboard_df[
                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                    + shown_columns.value
                    + [AutoEvalColumn.dummy.name]
                ],
                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                datatype=TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
                #column_widths=["2%", "33%"] 
            )

            with Modal(visible=False) as modal:
                map = gr.Plot()
            
            leaderboard_table.select(select, leaderboard_table, map)
            leaderboard_table.select(lambda: Modal(visible=True), None, modal)
            
            # Dummy leaderboard for handling the case when the user uses backspace key
            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=original_df[COLS],
                headers=COLS,
                datatype=TYPES,
                visible=False,
            )

            hide_models = gr.Textbox(
                            placeholder="",
                            show_label=False,
                            elem_id="search-bar",
                            value="",
                            visible=False,

                        )
            
            search_bar.submit(
                update_table,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    filter_columns_type,
                    filter_columns_precision,
                    filter_columns_parameters,
                    filter_columns_size,
                    hide_models,
                    search_bar,
                    filter_columns_computeDtype,
                    filter_columns_weightDtype,
                    filter_columns_doubleQuant,
                    filter_columns_groupDtype
                ],
                leaderboard_table,
            )

            """

           

            # Define a hidden component that will trigger a reload only if a query parameter has been set
            hidden_search_bar = gr.Textbox(value="", visible=False)
            hidden_search_bar.change(
                update_table,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    filter_columns_type,
                    filter_columns_precision,
                    filter_columns_size,
                    hide_models,
                    search_bar,
                ],
                leaderboard_table,
            )

            # Check query parameter once at startup and update search bar + hidden component
            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
            
            """
            filter_columns_precision.change(
                update_Weight_Dtype,
                [filter_columns_precision],
                [filter_columns_weightDtype]
            )

            filter_columns_type.change(
                quant_update_Weight_Dtype,
                [filter_columns_type],
                [filter_columns_weightDtype]
            )

            


    
            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, filter_columns_parameters, hide_models, filter_columns_computeDtype, filter_columns_weightDtype, filter_columns_doubleQuant, filter_columns_groupDtype]:
                selector.change(
                    update_table,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        filter_columns_type,
                        filter_columns_precision,
                        filter_columns_parameters,
                        filter_columns_size,
                        hide_models,
                        search_bar,
                        filter_columns_computeDtype,
                        filter_columns_weightDtype,
                        filter_columns_doubleQuant,
                        filter_columns_groupDtype
                    ],
                    leaderboard_table,
                    queue=True,
                )


        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):
            with gr.Row():
                with gr.Column():
                    chart = create_metric_plot_obj(
                        plot_df,
                        [AutoEvalColumn.average.name],
                        title="Average of Top Scores and Human Baseline Over Time (from last update)",
                    )
                    gr.Plot(value=chart, min_width=500) 
                with gr.Column():
                    chart = create_metric_plot_obj(
                        plot_df,
                        BENCHMARK_COLS,
                        title="Top Scores and Human Baseline Over Time (from last update)",
                    )
                    gr.Plot(value=chart, min_width=500) 
        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

        with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=4):
            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")

        with gr.TabItem("🚀 Submit ", elem_id="llm-benchmark-tab-table", id=5):
            with gr.Column():
                with gr.Row():
                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")

            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(label="Model name")
                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)

                with gr.Column():
                    """
                    precision = gr.Dropdown(
                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
                        label="Precision",
                        multiselect=False,
                        value="4bit",
                        interactive=True,
                    )
                    weight_type = gr.Dropdown(
                        choices=[i.value.name for i in WeightDtype],
                        label="Weights dtype",
                        multiselect=False,
                        value="int4",
                        interactive=True,
                    )
                    """
                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)",
                            visible=not IS_PUBLIC)
                    compute_type = gr.Dropdown(
                        choices=[i.value.name for i in ComputeDtype],
                        label="Compute dtype",
                        multiselect=False,
                        value="float16",
                        interactive=True,
                    )

            submit_button = gr.Button("Submit Eval")
            submission_result = gr.Markdown()
            submit_button.click(
                add_new_eval,
                [
                    model_name_textbox,
                    revision_name_textbox,
                    private,
                    compute_type,
                ],
                submission_result,
            )

            with gr.Column():
                with gr.Accordion(
                    f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
                    open=False,
                ):
                    with gr.Row():
                        finished_eval_table = gr.components.Dataframe(
                            value=finished_eval_queue_df,
                            headers=EVAL_COLS,
                            datatype=EVAL_TYPES,
                            row_count=5,
                        )
                with gr.Accordion(
                    f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
                    open=False,
                ):
                    with gr.Row():
                        running_eval_table = gr.components.Dataframe(
                            value=running_eval_queue_df,
                            headers=EVAL_COLS,
                            datatype=EVAL_TYPES,
                            row_count=5,
                        )

                with gr.Accordion(
                    f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
                    open=False,
                ):
                    with gr.Row():
                        pending_eval_table = gr.components.Dataframe(
                            value=pending_eval_queue_df,
                            headers=EVAL_COLS,
                            datatype=EVAL_TYPES,
                            row_count=5,
                        )

    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                lines=20,
                elem_id="citation-button",
                show_copy_button=True,
            )

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h
scheduler.add_job(update_dynamic_files, "interval", hours=12) # launched every 2 hour
scheduler.start()

demo.queue(default_concurrency_limit=40).launch()
# demo.queue(concurrency_count=40).launch()