BenchmarkBot commited on
Commit
804d27e
1 Parent(s): 9904a48

switch to tradeoff distance

Browse files
Files changed (1) hide show
  1. app.py +6 -7
app.py CHANGED
@@ -1,5 +1,4 @@
1
  import os
2
- import math
3
  import gradio as gr
4
  import pandas as pd
5
  import plotly.express as px
@@ -33,10 +32,10 @@ COLUMNS_MAPPING = {
33
  "backend.torch_dtype": "Load Dtype 📥",
34
  "optimizations": "Optimizations 🛠️",
35
  #
36
- "perf": "Open LLM-Perf Score ⬆️",
37
  #
38
- "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
39
  "score": "Open LLM Score ⬆️",
 
40
  "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
41
  "num_params": "#️⃣ Parameters (M) 📏",
42
  }
@@ -47,13 +46,13 @@ COLUMNS_DATATYPES = [
47
  "str",
48
  #
49
  "number",
50
- "number",
51
  #
52
  "number",
53
  "number",
54
  "number",
 
55
  ]
56
- SORTING_COLUMN = ["Open LLM-Perf Score ⬆️"]
57
 
58
 
59
  llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
@@ -74,8 +73,8 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
74
  # create composite score
75
  score_distance = 100 - bench_df["score"]
76
  latency_distance = bench_df["generate.latency(s)"]
77
- bench_df["perf"] = 100 / (1 + (score_distance**2 + latency_distance**2) ** 0.5)
78
- bench_df["perf"] = bench_df["perf"].round(2)
79
 
80
  # add optimizations
81
  bench_df["optimizations"] = bench_df[
 
1
  import os
 
2
  import gradio as gr
3
  import pandas as pd
4
  import plotly.express as px
 
32
  "backend.torch_dtype": "Load Dtype 📥",
33
  "optimizations": "Optimizations 🛠️",
34
  #
35
+ "tradeoff": "Open LLM Tradeoff ⬇️",
36
  #
 
37
  "score": "Open LLM Score ⬆️",
38
+ "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
39
  "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
40
  "num_params": "#️⃣ Parameters (M) 📏",
41
  }
 
46
  "str",
47
  #
48
  "number",
 
49
  #
50
  "number",
51
  "number",
52
  "number",
53
+ "number",
54
  ]
55
+ SORTING_COLUMN = ["Open LLM Tradeoff ⬇️"]
56
 
57
 
58
  llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
 
73
  # create composite score
74
  score_distance = 100 - bench_df["score"]
75
  latency_distance = bench_df["generate.latency(s)"]
76
+ bench_df["tradeoff"] = (score_distance**2 + latency_distance**2) ** 0.5
77
+ bench_df["tradeoff"] = bench_df["tradeoff"].round(2)
78
 
79
  # add optimizations
80
  bench_df["optimizations"] = bench_df[