Spaces:
Running
Running
BenchmarkBot
commited on
Commit
Β·
5490c7c
1
Parent(s):
a894537
sort by tradeoff but don't show it
Browse files
app.py
CHANGED
@@ -59,7 +59,7 @@ ALL_COLUMNS_DATATYPES = [
|
|
59 |
"number",
|
60 |
"number",
|
61 |
]
|
62 |
-
SORTING_COLUMN = ["
|
63 |
|
64 |
llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
|
65 |
|
@@ -73,10 +73,10 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
|
|
73 |
scores_df = pd.read_csv(
|
74 |
f"./llm-perf-dataset/reports/Grouped-Open-LLM-Leaderboard.csv"
|
75 |
)
|
76 |
-
|
77 |
|
78 |
# add optimizations
|
79 |
-
|
80 |
["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
|
81 |
].apply(
|
82 |
lambda x: ", ".join(
|
@@ -94,16 +94,23 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
|
|
94 |
axis=1,
|
95 |
)
|
96 |
|
97 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
|
99 |
|
100 |
def get_benchmark_table(bench_df):
|
|
|
|
|
101 |
# filter
|
102 |
bench_df = bench_df[list(ALL_COLUMNS_MAPPING.keys())]
|
103 |
# rename
|
104 |
bench_df.rename(columns=ALL_COLUMNS_MAPPING, inplace=True)
|
105 |
-
# sort
|
106 |
-
bench_df.sort_values(by=SORTING_COLUMN, ascending=True, inplace=True)
|
107 |
# transform
|
108 |
bench_df["Model Type π€"] = bench_df["Model Type π€"].apply(process_model_type)
|
109 |
bench_df["Weight Class ποΈ"] = bench_df["Weight Class ποΈ"].apply(
|
@@ -223,7 +230,9 @@ with demo:
|
|
223 |
|
224 |
# leaderboard tabs
|
225 |
with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
|
226 |
-
with gr.TabItem(
|
|
|
|
|
227 |
gr.HTML(A100_TEXT)
|
228 |
|
229 |
# Original leaderboard table
|
|
|
59 |
"number",
|
60 |
"number",
|
61 |
]
|
62 |
+
SORTING_COLUMN = ["tradeoff"]
|
63 |
|
64 |
llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
|
65 |
|
|
|
73 |
scores_df = pd.read_csv(
|
74 |
f"./llm-perf-dataset/reports/Grouped-Open-LLM-Leaderboard.csv"
|
75 |
)
|
76 |
+
merged_df = bench_df.merge(scores_df, left_on="model", right_on="best_scored_model")
|
77 |
|
78 |
# add optimizations
|
79 |
+
merged_df["optimizations"] = merged_df[
|
80 |
["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
|
81 |
].apply(
|
82 |
lambda x: ", ".join(
|
|
|
94 |
axis=1,
|
95 |
)
|
96 |
|
97 |
+
# create composite score
|
98 |
+
score_distance = 100 - merged_df["best_score"]
|
99 |
+
# normalize latency between 0 and 100
|
100 |
+
latency_distance = merged_df["generate.latency(s)"]
|
101 |
+
merged_df["tradeoff"] = (score_distance**2 + latency_distance**2) ** 0.5
|
102 |
+
merged_df["tradeoff"] = merged_df["tradeoff"].round(2)
|
103 |
+
|
104 |
+
return merged_df
|
105 |
|
106 |
|
107 |
def get_benchmark_table(bench_df):
|
108 |
+
# sort
|
109 |
+
bench_df.sort_values(by=SORTING_COLUMN, ascending=True, inplace=True)
|
110 |
# filter
|
111 |
bench_df = bench_df[list(ALL_COLUMNS_MAPPING.keys())]
|
112 |
# rename
|
113 |
bench_df.rename(columns=ALL_COLUMNS_MAPPING, inplace=True)
|
|
|
|
|
114 |
# transform
|
115 |
bench_df["Model Type π€"] = bench_df["Model Type π€"].apply(process_model_type)
|
116 |
bench_df["Weight Class ποΈ"] = bench_df["Weight Class ποΈ"].apply(
|
|
|
230 |
|
231 |
# leaderboard tabs
|
232 |
with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
|
233 |
+
with gr.TabItem(
|
234 |
+
"π₯οΈ A100-80GB Leaderboar Table οΏ½π
π π
οΏ½eπ
eπ
ππ
οΏ½π
π π
οΏ½eπ
eπ
π", id=0
|
235 |
+
):
|
236 |
gr.HTML(A100_TEXT)
|
237 |
|
238 |
# Original leaderboard table
|