src/custom_kernels.py

import gradio as gr
import pandas as pd
import plotly.express as px


CUSTOM_KERNELS_DATA = [
    # open llm
    "Model 🤗",
    "Arch 🏛️",
    "DType 📥",
    "Backend 🏭",
    "Params (B)",
    "Open LLM Score (%)",
    # deployment settings
    "DType 📥",
    "Backend 🏭",
    "Optimization 🛠️",
    "Quantization 🗜️",
    "Optimization 🛠️ Custom Kernel",
    "Quantization 🗜️ Custom Kernel",
    # primary measurements
    "Prefill Latency (s)",
    "Prefill Latency (s) Custom Kernel",
    "Decode Throughput (tokens/s)",
    "Decode Throughput (tokens/s) Custom Kernel",
    # speedups
    "Prefill Latency Speedup (%)",
    "Decode Throughput Speedup (%)",
]


def get_custom_kernels_df(llm_perf_df):
    copy_df = llm_perf_df.copy()
    # seperate vanilla GPTQ experiments from Custom Kernel experiments
    vanilla_df = copy_df[
        (copy_df["Backend 🏭"] == "pytorch") & 
        (copy_df["Quantization 🗜️"] == "None") &
        (copy_df["Optimization 🛠️"] == "None") & 
        (copy_df["DType 📥"] == "float16")
    ]
    exllamav1_df = copy_df[(copy_df["Quantization 🗜️"] == "GPTQ.4bit+ExllamaV1")]
    exllamav2_df = copy_df[(copy_df["Quantization 🗜️"] == "GPTQ.4bit+ExllamaV2")]
    gemm_df = copy_df[(copy_df["Quantization 🗜️"] == "AWQ.4bit+GEMM")]
    gemv_df = copy_df[(copy_df["Quantization 🗜️"] == "AWQ.4bit+GEMV")]
    # merge the three dataframes
    exllamav1_df = pd.merge(
        vanilla_df,
        exllamav1_df,
        on=["Model 🤗"],
        suffixes=["", " Custom Kernel"],
    )
    exllamav2_df = pd.merge(
        vanilla_df,
        exllamav2_df,
        on=["Model 🤗"],
        suffixes=["", " Custom Kernel"],
    )
    gemm_df = pd.merge(
        vanilla_df,
        gemm_df,
        on=["Model 🤗"],
        suffixes=["", " Custom Kernel"],
    )
    gemv_df = pd.merge(
        vanilla_df,
        gemv_df,
        on=["Model 🤗"],
        suffixes=["", " Custom Kernel"],
    )
    # concat the two dataframes row-wise
    custom_kernels_df = pd.concat([exllamav1_df, exllamav2_df, gemm_df, gemv_df])
    # compute speedups
    custom_kernels_df["Prefill Latency Speedup (%)"] = (
        (custom_kernels_df["Prefill Latency (s)"] / custom_kernels_df["Prefill Latency (s) Custom Kernel"]) * 100
    ).round(2) - 100
    custom_kernels_df["Decode Throughput Speedup (%)"] = (
        (
            custom_kernels_df["Decode Throughput (tokens/s) Custom Kernel"]
            / custom_kernels_df["Decode Throughput (tokens/s)"]
        )
        * 100
    ).round(2) - 100
    # filter speedups > 1000%
    custom_kernels_df = custom_kernels_df[custom_kernels_df["Prefill Latency Speedup (%)"] < 1000]
    custom_kernels_df = custom_kernels_df[custom_kernels_df["Decode Throughput Speedup (%)"] < 1000]

    return custom_kernels_df


def get_custom_kernels_decode_fig(llm_perf_df):
    custom_kernels_df = get_custom_kernels_df(llm_perf_df)
    # plot
    decode_fig = px.box(
        custom_kernels_df,
        x="Arch 🏛️",
        y="Decode Throughput Speedup (%)",
        color_discrete_sequence=px.colors.qualitative.Light24,
        custom_data=CUSTOM_KERNELS_DATA,
        color="Quantization 🗜️ Custom Kernel",
        points="all",
    )
    # add hover data
    decode_fig.update_traces(
        hovertemplate="<br>".join(
            [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(CUSTOM_KERNELS_DATA)]
        )
    )
    # add layout
    decode_fig.update_layout(
        title={
            "text": "Decode Throughput Speedup per Architecture",
            "y": 0.95,
            "x": 0.5,
            "xanchor": "center",
            "yanchor": "top",
        },
        xaxis_title="LLM Architecture",
        yaxis_title="Decode Speedup (%)",
        legend_title="Quantization Scheme",
        width=1200,
        height=600,
    )

    return decode_fig


def get_custom_kernels_prefill_fig(llm_perf_df):
    custom_kernels_df = get_custom_kernels_df(llm_perf_df)
    # plot
    prefill_fig = px.box(
        custom_kernels_df,
        x="Arch 🏛️",
        y="Prefill Latency Speedup (%)",
        color_discrete_sequence=px.colors.qualitative.Light24,
        custom_data=CUSTOM_KERNELS_DATA,
        color="Quantization 🗜️ Custom Kernel",
        points="all",
    )
    # add hover data
    prefill_fig.update_traces(
        hovertemplate="<br>".join(
            [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(CUSTOM_KERNELS_DATA)]
        )
    )
    # add layout
    prefill_fig.update_layout(
        title={
            "text": "Prefill Latency Speedup per Architecture",
            "y": 0.95,
            "x": 0.5,
            "xanchor": "center",
            "yanchor": "top",
        },
        xaxis_title="LLM Architecture",
        yaxis_title="Prefill Speedup (%)",
        legend_title="Quantization Scheme",
        width=1200,
        height=600,
    )

    return prefill_fig


def create_custom_kernels_plots(llm_perf_df):
    # descriptive text
    gr.HTML("👆 Hover over the points 👆 for additional information.", elem_id="text")
    # get figures
    prefill_fig = get_custom_kernels_prefill_fig(llm_perf_df)
    decode_fig = get_custom_kernels_decode_fig(llm_perf_df)

    # create plots
    prefill_plot = gr.components.Plot(value=prefill_fig, elem_id="plot", show_label=False)
    decode_plot = gr.components.Plot(value=decode_fig, elem_id="plot", show_label=False)

    return prefill_plot, decode_plot