Spaces:
Running
Running
BenchmarkBot
commited on
Commit
·
cdf41e7
1
Parent(s):
a189046
transpose energy
Browse files
app.py
CHANGED
@@ -37,7 +37,7 @@ ALL_COLUMNS_MAPPING = {
|
|
37 |
#
|
38 |
"generate.peak_memory(MB)": "Memory (MB) ⬇️",
|
39 |
"generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
|
40 |
-
"generate.energy_consumption(kWh
|
41 |
"best_score": "Best Score (%) ⬆️",
|
42 |
#
|
43 |
"best_scored_model": "Best Scored LLM 🏆",
|
@@ -84,8 +84,12 @@ def get_benchmark_df(benchmark="Succeeded-1xA100-80GB"):
|
|
84 |
merged_df = benchmark_df.merge(
|
85 |
clusters_df, left_on="model", right_on="best_scored_model"
|
86 |
)
|
87 |
-
#
|
88 |
-
merged_df["generate.energy_consumption(kWh
|
|
|
|
|
|
|
|
|
89 |
|
90 |
# add optimizations
|
91 |
merged_df["optimizations"] = merged_df["backend.bettertransformer"].apply(
|
@@ -95,6 +99,7 @@ def get_benchmark_df(benchmark="Succeeded-1xA100-80GB"):
|
|
95 |
merged_df["quantization"] = merged_df["backend.quantization_strategy"].apply(
|
96 |
lambda x: "BnB.4bit" if x == "bnb" else ("GPTQ.4bit" if x == "gptq" else "None")
|
97 |
)
|
|
|
98 |
# # distance to 100% score
|
99 |
# score_distance = 100 - merged_df["best_score"]
|
100 |
# # distance to 0s latency
|
|
|
37 |
#
|
38 |
"generate.peak_memory(MB)": "Memory (MB) ⬇️",
|
39 |
"generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
|
40 |
+
"generate.energy_consumption(tokens/kWh)": "Energy (tokens/kWh) ⬇️",
|
41 |
"best_score": "Best Score (%) ⬆️",
|
42 |
#
|
43 |
"best_scored_model": "Best Scored LLM 🏆",
|
|
|
84 |
merged_df = benchmark_df.merge(
|
85 |
clusters_df, left_on="model", right_on="best_scored_model"
|
86 |
)
|
87 |
+
# transpose energy consumption
|
88 |
+
merged_df["generate.energy_consumption(tokens/kWh)"] = 1 / merged_df[
|
89 |
+
"generate.energy_consumption(kWh/token)"
|
90 |
+
].fillna(1)
|
91 |
+
# fix nan values
|
92 |
+
merged_df[merged_df["generate.energy_consumption(tokens/kWh)"] == 1] = "N/A"
|
93 |
|
94 |
# add optimizations
|
95 |
merged_df["optimizations"] = merged_df["backend.bettertransformer"].apply(
|
|
|
99 |
merged_df["quantization"] = merged_df["backend.quantization_strategy"].apply(
|
100 |
lambda x: "BnB.4bit" if x == "bnb" else ("GPTQ.4bit" if x == "gptq" else "None")
|
101 |
)
|
102 |
+
|
103 |
# # distance to 100% score
|
104 |
# score_distance = 100 - merged_df["best_score"]
|
105 |
# # distance to 0s latency
|