import os import gradio as gr import pandas as pd import plotly.express as px from apscheduler.schedulers.background import BackgroundScheduler from src.assets.text_content import ( TITLE, INTRODUCTION_TEXT, SINGLE_A100_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, ) from src.utils import ( restart_space, load_dataset_repo, make_clickable_model, make_clickable_score, num_to_str, ) from src.assets.css_html_js import custom_css LLM_PERF_LEADERBOARD_REPO = "optimum/llm-perf-leaderboard" LLM_PERF_DATASET_REPO = "optimum/llm-perf-dataset" OPTIMUM_TOKEN = os.environ.get("OPTIMUM_TOKEN", None) COLUMNS_MAPPING = { "model": "Model 🤗", "backend.name": "Backend 🏭", "backend.torch_dtype": "Load Dtype 📥", "optimizations": "Optimizations 🛠️", # "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️", "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️", "average": "Average Open LLM Score ⬆️", # "num_parameters": "#️⃣ Parameters 📏", } COLUMNS_DATATYPES = [ "markdown", "str", "str", "str", # "number", "number", "markdown", # "str", ] SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"] llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN) def get_benchmark_df(benchmark="1xA100-80GB"): if llm_perf_dataset_repo: llm_perf_dataset_repo.git_pull() # load bench_df = pd.read_csv(f"./llm-perf-dataset/reports/{benchmark}.csv") scores_df = pd.read_csv(f"./llm-perf-dataset/reports/additional_data.csv") bench_df = bench_df.merge(scores_df, on="model", how="left") bench_df["optimizations"] = bench_df[ ["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"] ].apply( lambda x: ", ".join([opt for opt in x.index if x[opt] == True]), axis=1, ) return bench_df def get_benchmark_table(bench_df): # filter bench_df = bench_df[list(COLUMNS_MAPPING.keys())] # rename bench_df.rename(columns=COLUMNS_MAPPING, inplace=True) # sort bench_df.sort_values(by=SORTING_COLUMN, ascending=False, inplace=True) # transform bench_df["Model 🤗"] = bench_df["Model 🤗"].apply(make_clickable_model) bench_df["#️⃣ Parameters 📏"] = bench_df["#️⃣ Parameters 📏"].apply(num_to_str) bench_df["Average Open LLM Score ⬆️"] = bench_df["Average Open LLM Score ⬆️"].apply( make_clickable_score ) return bench_df def get_benchmark_plot(bench_df): # untill falcon gets fixed / natively supported bench_df = bench_df[bench_df["generate.latency(s)"] < 100] fig = px.scatter( bench_df, x="generate.latency(s)", y="average", color="model_type", symbol="backend.name", size="forward.peak_memory(MB)", custom_data=[ "model", "backend.name", "backend.torch_dtype", "optimizations", "forward.peak_memory(MB)", "generate.throughput(tokens/s)", ], symbol_sequence=["triangle-up", "circle"], # as many distinct colors as there are model_type,backend.name couples color_discrete_sequence=px.colors.qualitative.Light24, ) fig.update_layout( title={ "text": "Model Score vs. Latency vs. Memory", "y": 0.95, "x": 0.5, "xanchor": "center", "yanchor": "top", }, xaxis_title="Per 1000 Tokens Latency (s)", yaxis_title="Average Open LLM Score", legend_title="Model Type and Backend", width=1200, height=600, ) fig.update_traces( hovertemplate="
".join( [ "Model: %{customdata[0]}", "Backend: %{customdata[1]}", "Datatype: %{customdata[2]}", "Optimizations: %{customdata[3]}", "Peak Memory (MB): %{customdata[4]}", "Throughput (tokens/s): %{customdata[5]}", "Average Open LLM Score: %{y}", "Per 1000 Tokens Latency (s): %{x}", ] ) ) return fig def filter_query( text, backends, datatypes, optimizations, threshold, benchmark="1xA100-80GB" ): raw_df = get_benchmark_df(benchmark=benchmark) filtered_df = raw_df[ raw_df["model"].str.lower().str.contains(text.lower()) & raw_df["backend.name"].isin(backends) & raw_df["backend.torch_dtype"].isin(datatypes) & pd.concat( [ raw_df["optimizations"].str.contains(optimization) for optimization in optimizations ], axis=1, ).any(axis=1) & (raw_df["average"] >= threshold) ] filtered_table = get_benchmark_table(filtered_df) filtered_plot = get_benchmark_plot(filtered_df) return filtered_table, filtered_plot # Dataframes single_A100_df = get_benchmark_df(benchmark="1xA100-80GB") single_A100_table = get_benchmark_table(single_A100_df) single_A100_plot = get_benchmark_plot(single_A100_df) # Demo interface demo = gr.Blocks(css=custom_css) with demo: # leaderboard title gr.HTML(TITLE) # introduction text gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") # control panel title gr.HTML("

Control Panel 🎛️

") # control panel interface with gr.Row(): search_bar = gr.Textbox( label="Model 🤗", info="🔍 Search for a model name", elem_id="search-bar", ) with gr.Row(): backend_checkboxes = gr.CheckboxGroup( label="Backends 🏭", choices=["pytorch", "onnxruntime"], value=["pytorch", "onnxruntime"], info="☑️ Select the backends", elem_id="backend-checkboxes", ) datatype_checkboxes = gr.CheckboxGroup( label="Datatypes 📥", choices=["float32", "float16"], value=["float32", "float16"], info="☑️ Select the load datatypes", elem_id="datatype-checkboxes", ) optimizations_checkboxes = gr.CheckboxGroup( label="Optimizations 🛠️", choices=["BetterTransformer", "LLM.int8", "NF4"], value=[], info="☑️ Select the optimizations", elem_id="optimizations-checkboxes", ) with gr.Row(): score_slider = gr.Slider( label="Average Open LLM Score 📈", info="🎚️ Slide to minimum Average Open LLM score", value=0.0, elem_id="threshold-slider", ) with gr.Row(): filter_button = gr.Button( value="Filter 🚀", elem_id="filter-button", ) # leaderboard tabs with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🖥️ A100-80GB Leaderboard 🏆", id=0): gr.HTML(SINGLE_A100_TEXT) # Original leaderboard table single_A100_leaderboard = gr.components.Dataframe( value=single_A100_table, datatype=COLUMNS_DATATYPES, headers=list(COLUMNS_MAPPING.values()), elem_id="1xA100-table", ) with gr.TabItem("🖥️ A100-80GB Plot 📊", id=1): # Original leaderboard plot gr.HTML(SINGLE_A100_TEXT) # Original leaderboard plot single_A100_plotly = gr.components.Plot( value=single_A100_plot, elem_id="1xA100-plot", show_label=False, ) filter_button.click( filter_query, [ search_bar, backend_checkboxes, datatype_checkboxes, optimizations_checkboxes, score_slider, ], [single_A100_leaderboard, single_A100_plotly], ) with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", ).style(show_copy_button=True) # Restart space every hour scheduler = BackgroundScheduler() scheduler.add_job( restart_space, "interval", seconds=3600, args=[LLM_PERF_LEADERBOARD_REPO, OPTIMUM_TOKEN], ) scheduler.start() # Launch demo demo.queue(concurrency_count=40).launch()