app.py

import gradio as gr
import pandas as pd

from src.about import (  # CITATION_BUTTON_LABEL,; CITATION_BUTTON_TEXT,; EVALUATION_QUEUE_TEXT,
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css
from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EVAL_COLS,; NUMERIC_INTERVALS,; ModelType,; Precision,
    COLS,
    COST_COLS,
    COST_TYPES,
    TS_COLS,
    TS_TYPES,
    TYPES,
    AutoEvalColumn,
    CostEvalColumn,
    TSEvalColumn,
    fields,
)

# from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.envs import CRM_RESULTS_PATH
from src.populate import get_leaderboard_df_crm

original_df, cost_df, ts_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS, COST_COLS)

# raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
leaderboard_df = original_df.copy()
leaderboard_cost_df = cost_df.copy()
leaderboard_ts_df = ts_df.copy()
# leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"})


# Searching and filtering
def update_table(
    hidden_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
    accuracy_method_query: str,
    accuracy_threshold_query: str,
    use_case_area_query: list,
    use_case_query: list,
    use_case_type_query: list,
):
    filtered_df = filter_llm_func(hidden_df, llm_query)
    filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
    filtered_df = filter_accuracy_method_func(filtered_df, accuracy_method_query)
    filtered_df["Accuracy Threshold"] = filter_accuracy_threshold_func(filtered_df, accuracy_threshold_query)
    filtered_df = filtered_df[filtered_df["Accuracy Threshold"]]
    filtered_df["Use Case Area"] = filtered_df["Use Case Name"].apply(lambda x: x.split(": ")[0])
    filtered_df = filter_use_case_area_func(filtered_df, use_case_area_query)
    filtered_df = filter_use_case_func(filtered_df, use_case_query)
    filtered_df = filter_use_case_type_func(filtered_df, use_case_type_query)
    df = select_columns(filtered_df, columns)
    return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4")


def update_cost_table(
    hidden_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
    use_case_flavor_query: list,
):
    filtered_df = filter_llm_func(hidden_df, llm_query)
    filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
    filtered_df = filter_use_case_flavor_func(filtered_df, use_case_flavor_query)
    df = select_columns_cost_table(filtered_df, columns)
    return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4")


def update_ts_table(
    hidden_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
):
    filtered_df = filter_llm_func(hidden_df, llm_query)
    filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
    df = select_columns_ts_table(filtered_df, columns)
    return df


# def highlight_cols(x):
#     df = x.copy()
#     df.loc[:, :] = "color: black"
#     df.loc[, ["Accuracy"]] = "background-color: #b3d5a4"
#     return df


def highlight_cost_band_low(s, props=""):

    return props if s == "Low" else None


def init_leaderboard_df(
    leaderboard_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
    accuracy_method_query: str,
    accuracy_threshold_query: str,
    use_case_area_query: list,
    use_case_query: list,
    use_case_type_query: list,
):

    # Applying the style function
    # return df.style.apply(highlight_cols, axis=None)
    return update_table(
        leaderboard_df,
        columns,
        llm_query,
        llm_provider_query,
        accuracy_method_query,
        accuracy_threshold_query,
        use_case_area_query,
        use_case_query,
        use_case_type_query,
    )


def init_leaderboard_cost_df(
    leaderboard_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
    use_case_type_query: list,
):

    return update_cost_table(
        leaderboard_df,
        columns,
        llm_query,
        llm_provider_query,
        use_case_type_query,
    )


def init_leaderboard_ts_df(
    leaderboard_df: pd.DataFrame,
    columns: list,
    llm_query: list,
    llm_provider_query: list,
):

    return update_ts_table(
        leaderboard_df,
        columns,
        llm_query,
        llm_provider_query,
    )


def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame:
    return df[df["Accuracy Method"] == accuracy_method_query]


def filter_accuracy_threshold_func(df: pd.DataFrame, accuracy_threshold_query: str) -> pd.DataFrame:
    accuracy_cols = ["Instruction Following", "Conciseness", "Completeness", "Accuracy"]
    return (df.loc[:, accuracy_cols] >= float(accuracy_threshold_query)).all(axis=1)


def filter_use_case_area_func(df: pd.DataFrame, use_case_area_query: list) -> pd.DataFrame:
    return df[
        df["Use Case Area"].apply(
            lambda x: len(set([_.strip() for _ in x.split("&")]).intersection(use_case_area_query))
        )
        > 0
    ]


def filter_use_case_func(df: pd.DataFrame, use_case_query: list) -> pd.DataFrame:
    return df[df["Use Case Name"].isin(use_case_query)]


def filter_use_case_type_func(df: pd.DataFrame, use_case_type_query: list) -> pd.DataFrame:
    return df[df["Use Case Type"].isin(use_case_type_query)]


def filter_use_case_flavor_func(df: pd.DataFrame, use_case_flavor_query: list) -> pd.DataFrame:
    return df[df["Cost and Speed: Flavor"].isin(use_case_flavor_query)]


def filter_llm_func(df: pd.DataFrame, llm_query: list) -> pd.DataFrame:
    return df[df["Model Name"].isin(llm_query)]


def filter_llm_provider_func(df: pd.DataFrame, llm_provider_query: list) -> pd.DataFrame:
    return df[df["LLM Provider"].isin(llm_provider_query)]


def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [
        AutoEvalColumn.model.name,
    ]
    # We use COLS to maintain sorting
    filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns]]
    return filtered_df


def select_columns_cost_table(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [
        CostEvalColumn.model.name,
    ]
    filtered_df = df[always_here_cols + [c for c in COST_COLS if c in df.columns and c in columns]]
    return filtered_df


def select_columns_ts_table(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [
        TSEvalColumn.model.name,
    ]
    filtered_df = df[always_here_cols + [c for c in TS_COLS if c in df.columns and c in columns]]
    return filtered_df


demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
            with gr.Row():
                with gr.Column():
                    # with gr.Row():
                    #     search_bar = gr.Textbox(
                    #         placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                    #         show_label=False,
                    #         elem_id="search-bar",
                    #     )
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden],
                            value=[
                                c.name
                                for c in fields(AutoEvalColumn)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )
                # with gr.Column(min_width=320):
                #     # with gr.Box(elem_id="box-filter"):
                #     filter_columns_type = gr.CheckboxGroup(
                #         label="Model types",
                #         choices=[t.to_str() for t in ModelType],
                #         value=[t.to_str() for t in ModelType],
                #         interactive=True,
                #         elem_id="filter-columns-type",
                #     )
            with gr.Row():
                with gr.Column():
                    filter_llm = gr.CheckboxGroup(
                        choices=list(original_df["Model Name"].unique()),
                        value=list(original_df["Model Name"].unique()),
                        label="Model Name",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_llm_provider = gr.CheckboxGroup(
                        choices=list(original_df["LLM Provider"].unique()),
                        value=list(original_df["LLM Provider"].unique()),
                        label="LLM Provider",
                        info="",
                        interactive=True,
                    )
            with gr.Row():
                filter_use_case = gr.CheckboxGroup(
                    choices=list(original_df["Use Case Name"].unique()),
                    value=list(original_df["Use Case Name"].unique()),
                    label="Use Case",
                    info="",
                    # multiselect=True,
                    interactive=True,
                )
            with gr.Row():
                with gr.Column():
                    filter_use_case_area = gr.CheckboxGroup(
                        choices=["Service", "Sales"],
                        value=["Service", "Sales"],
                        label="Use Case Area",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_use_case_type = gr.CheckboxGroup(
                        choices=["Summary", "Generation"],
                        value=["Summary", "Generation"],
                        label="Use Case Type",
                        info="",
                        interactive=True,
                    )
                # with gr.Column():
                #     filter_use_case = gr.Dropdown(
                #         choices=list(original_df["Use Case Name"].unique()),
                #         value=list(original_df["Use Case Name"].unique()),
                #         label="Use Case",
                #         info="",
                #         multiselect=True,
                #         interactive=True,
                #     )
                # with gr.Column():
                #     filter_metric_area = gr.CheckboxGroup(
                #         choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
                #         value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
                #         label="Metric Area",
                #         info="",
                #         interactive=True,
                #     )
                with gr.Column():
                    filter_accuracy_method = gr.Radio(
                        choices=["Manual", "Auto"],
                        value="Manual",
                        label="Accuracy Method",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_accuracy_threshold = gr.Number(
                        value="3",
                        label="Accuracy Threshold",
                        info="Range: 0.0 to 4.0",
                        interactive=True,
                    )
                # with gr.Column():
                #     filter_llm = gr.CheckboxGroup(
                #         choices=list(original_df["Model Name"].unique()),
                #         value=list(leaderboard_df["Model Name"].unique()),
                #         label="Model Name",
                #         info="",
                #         interactive=True,
                #     )
                # with gr.Column():
                #     filter_llm_provider = gr.CheckboxGroup(
                #         choices=list(original_df["LLM Provider"].unique()),
                #         value=list(leaderboard_df["LLM Provider"].unique()),
                #         label="LLM Provider",
                #         info="",
                #         interactive=True,
                #     )

            leaderboard_table = gr.components.Dataframe(
                # value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                value=init_leaderboard_df(
                    leaderboard_df,
                    shown_columns.value,
                    filter_llm.value,
                    filter_llm_provider.value,
                    filter_accuracy_method.value,
                    filter_accuracy_threshold.value,
                    filter_use_case_area.value,
                    filter_use_case.value,
                    filter_use_case_type.value,
                ),
                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                datatype=TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )

            # Dummy leaderboard for handling the case when the user uses backspace key
            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=original_df[COLS],
                headers=COLS,
                datatype=TYPES,
                visible=False,
            )
            # search_bar.submit(
            #     update_table,
            #     [
            #         hidden_leaderboard_table_for_search,
            #         shown_columns,
            #         filter_columns_type,
            #         filter_columns_precision,
            #         filter_columns_size,
            #         deleted_models_visibility,
            #         search_bar,
            #     ],
            #     leaderboard_table,
            # )
            for selector in [
                shown_columns,
                filter_llm,
                filter_llm_provider,
                filter_accuracy_method,
                filter_accuracy_threshold,
                filter_use_case_area,
                filter_use_case,
                filter_use_case_type,
                # filter_columns_type,
                # filter_columns_precision,
                # filter_columns_size,
                # deleted_models_visibility,
            ]:
                selector.change(
                    update_table,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        filter_llm,
                        filter_llm_provider,
                        filter_accuracy_method,
                        filter_accuracy_threshold,
                        filter_use_case_area,
                        filter_use_case,
                        filter_use_case_type,
                        # filter_columns_type,
                        # filter_columns_precision,
                        # filter_columns_size,
                        # deleted_models_visibility,
                        # search_bar,
                    ],
                    leaderboard_table,
                    queue=True,
                )
        with gr.TabItem("🏅 Latency & Cost", elem_id="llm-benchmark-tab-table", id=1):
            with gr.Row():
                with gr.Column():
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[c.name for c in fields(CostEvalColumn) if not c.hidden and not c.never_hidden],
                            value=[
                                c.name
                                for c in fields(CostEvalColumn)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )
            with gr.Row():
                with gr.Column():
                    filter_llm = gr.CheckboxGroup(
                        choices=list(cost_df["Model Name"].unique()),
                        value=list(cost_df["Model Name"].unique()),
                        label="Model Name",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_llm_provider = gr.CheckboxGroup(
                        choices=list(cost_df["LLM Provider"].unique()),
                        value=list(cost_df["LLM Provider"].unique()),
                        label="LLM Provider",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_use_case_type = gr.CheckboxGroup(
                        choices=["Long", "Short"],
                        value=["Long", "Short"],
                        label="Use Case Flavor",
                        info="Output: 250 tokens, Long input: 3k tokens, Short input: 500 tokens",
                        interactive=True,
                    )

            leaderboard_table = gr.components.Dataframe(
                value=init_leaderboard_cost_df(
                    leaderboard_cost_df,
                    shown_columns.value,
                    filter_llm.value,
                    filter_llm_provider.value,
                    filter_use_case_type.value,
                ),
                headers=[c.name for c in fields(CostEvalColumn) if c.never_hidden] + shown_columns.value,
                datatype=COST_TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )

            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=cost_df[COST_COLS],
                headers=COST_COLS,
                datatype=COST_TYPES,
                visible=False,
            )

            for selector in [
                shown_columns,
                filter_llm,
                filter_llm_provider,
                filter_use_case_type,
            ]:
                selector.change(
                    update_cost_table,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        filter_llm,
                        filter_llm_provider,
                        filter_use_case_type,
                    ],
                    leaderboard_table,
                    queue=True,
                )
        with gr.TabItem("🏅 Trust & Safety", elem_id="llm-benchmark-tab-table", id=2):
            with gr.Row():
                with gr.Column():
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[c.name for c in fields(TSEvalColumn) if not c.hidden and not c.never_hidden],
                            value=[
                                c.name
                                for c in fields(TSEvalColumn)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )
            with gr.Row():
                with gr.Column():
                    filter_llm = gr.CheckboxGroup(
                        choices=list(ts_df["Model Name"].unique()),
                        value=list(ts_df["Model Name"].unique()),
                        label="Model Name",
                        info="",
                        interactive=True,
                    )
                with gr.Column():
                    filter_llm_provider = gr.CheckboxGroup(
                        choices=list(ts_df["LLM Provider"].unique()),
                        value=list(ts_df["LLM Provider"].unique()),
                        label="LLM Provider",
                        info="",
                        interactive=True,
                    )

            leaderboard_table = gr.components.Dataframe(
                value=init_leaderboard_ts_df(
                    leaderboard_ts_df,
                    shown_columns.value,
                    filter_llm.value,
                    filter_llm_provider.value,
                ),
                headers=[c.name for c in fields(TSEvalColumn) if c.never_hidden] + shown_columns.value,
                datatype=TS_TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )

            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=ts_df[TS_COLS],
                headers=TS_COLS,
                datatype=TS_TYPES,
                visible=False,
            )

            for selector in [
                shown_columns,
                filter_llm,
                filter_llm_provider,
            ]:
                selector.change(
                    update_ts_table,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        filter_llm,
                        filter_llm_provider,
                    ],
                    leaderboard_table,
                    queue=True,
                )
        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                lines=20,
                elem_id="citation-button",
                show_copy_button=True,
            )

# scheduler = BackgroundScheduler()
# scheduler.add_job(restart_space, "interval", seconds=1800)
# scheduler.start()
demo.queue(default_concurrency_limit=40).launch()