Spaces:
Running
Running
import gradio as gr | |
import pandas as pd | |
import os | |
import zipfile | |
import base64 | |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, | |
author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, | |
title = {AI Energy Score Leaderboard - February 2025}, | |
year = {2025}, | |
publisher = {Hugging Face}, | |
howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", | |
}""" | |
# List of tasks (CSV filenames) | |
tasks = [ | |
'asr.csv', | |
'object_detection.csv', | |
'text_classification.csv', | |
'image_captioning.csv', | |
'question_answering.csv', | |
'text_generation.csv', | |
'image_classification.csv', | |
'sentence_similarity.csv', | |
'image_generation.csv', | |
'summarization.csv' | |
] | |
### HELPER FUNCTIONS ### | |
def format_stars(score): | |
try: | |
score_int = int(score) | |
except Exception: | |
score_int = 0 | |
return f'<span style="color: #3fa45bff; font-size:1.5em;">{"β " * score_int}</span>' | |
def make_link(mname): | |
parts = str(mname).split('/') | |
display_name = parts[1] if len(parts) > 1 else mname | |
return f'<a href="https://huggingface.co/{mname}" target="_blank">{display_name}</a>' | |
def extract_link_text(html_link): | |
start = html_link.find('>') + 1 | |
end = html_link.rfind('</a>') | |
if start > 0 and end > start: | |
return html_link[start:end] | |
else: | |
return html_link | |
def generate_html_table_from_df(df): | |
# Compute a static width for the Model column based on the longest model name. | |
if not df.empty: | |
max_length = max(len(extract_link_text(link)) for link in df['Model']) | |
else: | |
max_length = 10 | |
static_width = max_length * 10 + 16 | |
max_energy = df['gpu_energy_numeric'].max() if not df.empty else 1 | |
color_map = {"1": "black", "2": "black", "3": "black", "4": "black", "5": "black"} | |
html = '<table class="data-table" style="width:100%; border-collapse: collapse; font-family: Inter, sans-serif;">' | |
html += '<thead><tr style="background-color: #f2f2f2;">' | |
html += '<th style="text-align: left; padding: 8px;" title="Model name with link to Hugging Face">Model</th>' | |
html += '<th style="text-align: left; padding: 8px;" title="AI Provider extracted from the model name">Provider</th>' | |
html += '<th style="text-align: left; padding: 8px;" title="GPU energy consumed in Watt-hours for 1,000 queries">GPU Energy (Wh)</th>' | |
html += '<th style="text-align: left; padding: 8px;" title="Energy efficiency score (stars)">Score</th>' | |
html += '</tr></thead>' | |
html += '<tbody>' | |
for _, row in df.iterrows(): | |
energy_numeric = row['gpu_energy_numeric'] | |
energy_str = f"{energy_numeric:,.2f}" | |
bar_width = (energy_numeric / max_energy) * 100 | |
score_val = row['energy_score'] | |
bar_color = color_map.get(str(score_val), "gray") | |
html += '<tr>' | |
html += f'<td style="padding: 8px; width: {static_width}px;">{row["Model"]}</td>' | |
html += f'<td style="padding: 8px;">{row["Provider"]}</td>' | |
html += (f'<td style="padding: 8px;">{energy_str}<br>' | |
f'<div style="background-color: {bar_color}; width: {bar_width:.1f}%; height: 10px;"></div></td>') | |
html += f'<td style="padding: 8px;">{row["Score"]}</td>' | |
html += '</tr>' | |
html += '</tbody></table>' | |
return f'<div class="table-container">{html}</div>' | |
def process_df(task, sort_order="Low to High", filter_fn=None): | |
df = pd.read_csv(os.path.join("data", "energy", task)) | |
if df.columns[0].startswith("Unnamed:"): | |
df = df.iloc[:, 1:] | |
df['energy_score'] = df['energy_score'].astype(int) | |
df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
if filter_fn is not None: | |
df = filter_fn(df) | |
df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
df['Model'] = df['model'].apply(make_link) | |
df['Score'] = df['energy_score'].apply(format_stars) | |
ascending = True if sort_order == "Low to High" else False | |
df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
return df | |
def compute_efficiency_ratio(df): | |
if df.empty: | |
return 1 | |
min_val = df['gpu_energy_numeric'].min() | |
max_val = df['gpu_energy_numeric'].max() | |
ratio = max_val / min_val if min_val > 0 else 1 | |
return ratio | |
def generate_info_callout(ratio, scope_text): | |
""" | |
Returns a "did you know" callout with a lightbulb emoji. | |
The callout uses a light green background, a small font, and is limited to a max-width of 250px. | |
It is wrapped in a container that aligns it to the right. | |
""" | |
return ( | |
f'<div style="text-align: right;">' | |
f'<div class="info-callout" style="display:inline-block; max-width:250px; font-size:0.8em; background-color:#e6ffe6; padding:8px; border-radius:5px;">' | |
f'π‘ There\'s a <strong>{ratio:,.1f}x</strong> difference between the highest and lowest energy use in {scope_text}.' | |
f'</div></div>' | |
) | |
def get_global_callout(): | |
all_df = pd.DataFrame() | |
for task in tasks: | |
df = pd.read_csv(os.path.join("data", "energy", task)) | |
if df.columns[0].startswith("Unnamed:"): | |
df = df.iloc[:, 1:] | |
df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
all_df = pd.concat([all_df, df], ignore_index=True) | |
ratio = compute_efficiency_ratio(all_df) | |
return generate_info_callout(ratio, "this leaderboard") | |
### ZIP DOWNLOAD FUNCTIONS ### | |
def zip_csv_files(): | |
data_dir = os.path.join("data", "energy") | |
zip_filename = "data.zip" | |
with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: | |
for filename in os.listdir(data_dir): | |
if filename.endswith(".csv"): | |
filepath = os.path.join(data_dir, filename) | |
zipf.write(filepath, arcname=filename) | |
return zip_filename | |
def get_zip_data_link(): | |
zip_filename = zip_csv_files() | |
with open(zip_filename, "rb") as f: | |
data = f.read() | |
b64 = base64.b64encode(data).decode() | |
href = ( | |
f'<a href="data:application/zip;base64,{b64}" ' | |
'download="data.zip" ' | |
'style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: inherit; font-family: \'Inter\', sans-serif;">Download Data</a>' | |
) | |
return href | |
### UPDATE FUNCTIONS (RETURNING CALLOUT AND TABLE HTML) ### | |
def update_text_generation(selected_display, sort_order): | |
mapping = { | |
"A (Single Consumer GPU) <20B parameters": "A", | |
"B (Single Cloud GPU) 20-66B parameters": "B", | |
"C (Multiple Cloud GPUs) >66B parameters": "C" | |
} | |
model_class = mapping.get(selected_display, "A") | |
def filter_fn(df): | |
if 'class' in df.columns: | |
return df[df['class'] == model_class] | |
return df | |
df = process_df('text_generation.csv', sort_order, filter_fn) | |
ratio = compute_efficiency_ratio(df) | |
# For Text Generation, use "this class" as the scope. | |
callout = generate_info_callout(ratio, "this class") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_image_generation(sort_order): | |
df = process_df('image_generation.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_text_classification(sort_order): | |
df = process_df('text_classification.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_image_classification(sort_order): | |
df = process_df('image_classification.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_image_captioning(sort_order): | |
df = process_df('image_captioning.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_summarization(sort_order): | |
df = process_df('summarization.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_asr(sort_order): | |
df = process_df('asr.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_object_detection(sort_order): | |
df = process_df('object_detection.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_sentence_similarity(sort_order): | |
df = process_df('sentence_similarity.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_extractive_qa(sort_order): | |
df = process_df('question_answering.csv', sort_order) | |
ratio = compute_efficiency_ratio(df) | |
callout = generate_info_callout(ratio, "this task") | |
table_html = generate_html_table_from_df(df) | |
return callout, table_html | |
def update_all_tasks(sort_order): | |
all_df = pd.DataFrame() | |
for task in tasks: | |
df = pd.read_csv(os.path.join("data", "energy", task)) | |
if df.columns[0].startswith("Unnamed:"): | |
df = df.iloc[:, 1:] | |
df['energy_score'] = df['energy_score'].astype(int) | |
df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
df['Model'] = df['model'].apply(make_link) | |
df['Score'] = df['energy_score'].apply(format_stars) | |
all_df = pd.concat([all_df, df], ignore_index=True) | |
all_df = all_df.drop_duplicates(subset=['model']) | |
ascending = True if sort_order == "Low to High" else False | |
all_df = all_df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
ratio = compute_efficiency_ratio(all_df) | |
callout = generate_info_callout(ratio, "this leaderboard") | |
table_html = generate_html_table_from_df(all_df) | |
return callout, table_html | |
### GLOBAL HEADER (Logo & Global Callout) ### | |
# Use a <picture> element so that dark mode uses logodark.png. | |
global_header_html = f""" | |
<div style="position: relative; width: 100%; text-align: center; margin-bottom: 20px;"> | |
<picture style="display:inline-block;"> | |
<source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logodark.png"> | |
<img src="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logo.png" | |
alt="Logo" | |
style="width:300px; max-width:300px; height:auto; display:inline-block;"> | |
</picture> | |
<div style="position: absolute; top: 50%; right: 20px; transform: translateY(-50%);"> | |
{get_global_callout()} | |
</div> | |
</div> | |
""" | |
### CUSTOM CSS for Dark Mode and Mobile Responsiveness ### | |
custom_css = """ | |
/* Table and layout */ | |
.data-table { | |
table-layout: fixed; | |
width: 100%; | |
} | |
.data-table th, .data-table td { | |
max-width: 150px; | |
white-space: nowrap; | |
overflow: hidden; | |
text-overflow: ellipsis; | |
} | |
.table-container { | |
width: 100%; | |
margin-left: auto; | |
margin-right: auto; | |
} | |
/* Force header links to be black in light mode */ | |
.header-link { | |
color: black !important; | |
} | |
/* Dark mode styles */ | |
@media (prefers-color-scheme: dark) { | |
body { | |
background-color: #121212; | |
color: #e0e0e0; | |
} | |
.data-table thead { | |
background-color: #333; | |
} | |
.data-table th { | |
color: #e0e0e0; | |
} | |
.data-table td { | |
color: #e0e0e0; | |
} | |
/* Non-header links in dark mode */ | |
a:not(.header-link) { | |
color: #3fa45bff !important; | |
} | |
} | |
/* Mobile styles: hide callout boxes on small screens */ | |
@media (max-width: 600px) { | |
.info-callout { | |
display: none !important; | |
} | |
} | |
""" | |
### GRADIO INTERFACE ### | |
demo = gr.Blocks(css=custom_css) | |
with demo: | |
# --- Header Links --- | |
gr.HTML(f""" | |
<div style="display: flex; justify-content: space-evenly; align-items: center; margin-bottom: 20px;"> | |
<a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/submission_portal" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Submission Portal</a> | |
<a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/Label" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Label Generator</a> | |
<a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#faq" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">FAQ</a> | |
<a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#documentation" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Documentation</a> | |
{get_zip_data_link()} | |
<a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/README/discussions" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Community</a> | |
</div> | |
""") | |
# --- Global Header: Centered Logo with Global Callout at Right Edge --- | |
gr.HTML(global_header_html) | |
# --- Tabs for the different tasks --- | |
with gr.Tabs(): | |
# --- Text Generation Tab --- | |
with gr.TabItem("Text Generation π¬"): | |
with gr.Row(): | |
with gr.Column(scale=4): | |
model_class_options = [ | |
"A (Single Consumer GPU) <20B parameters", | |
"B (Single Cloud GPU) 20-66B parameters", | |
"C (Multiple Cloud GPUs) >66B parameters" | |
] | |
model_class_dropdown = gr.Dropdown(choices=model_class_options, label="Select Model Class", value=model_class_options[0]) | |
with gr.Column(scale=4): | |
sort_dropdown_tg = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
tg_callout = gr.HTML() | |
tg_table = gr.HTML() | |
init_callout, init_table = update_text_generation(model_class_options[0], "Low to High") | |
tg_callout.value = init_callout | |
tg_table.value = init_table | |
model_class_dropdown.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) | |
sort_dropdown_tg.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) | |
# --- Image Generation Tab --- | |
with gr.TabItem("Image Generation π·"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_img = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
img_callout = gr.HTML() | |
img_table = gr.HTML() | |
init_callout, init_table = update_image_generation("Low to High") | |
img_callout.value = init_callout | |
img_table.value = init_table | |
sort_dropdown_img.change(fn=update_image_generation, inputs=sort_dropdown_img, outputs=[img_callout, img_table]) | |
# --- Text Classification Tab --- | |
with gr.TabItem("Text Classification π"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_tc = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
tc_callout = gr.HTML() | |
tc_table = gr.HTML() | |
init_callout, init_table = update_text_classification("Low to High") | |
tc_callout.value = init_callout | |
tc_table.value = init_table | |
sort_dropdown_tc.change(fn=update_text_classification, inputs=sort_dropdown_tc, outputs=[tc_callout, tc_table]) | |
# --- Image Classification Tab --- | |
with gr.TabItem("Image Classification πΌοΈ"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_ic = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
ic_callout = gr.HTML() | |
ic_table = gr.HTML() | |
init_callout, init_table = update_image_classification("Low to High") | |
ic_callout.value = init_callout | |
ic_table.value = init_table | |
sort_dropdown_ic.change(fn=update_image_classification, inputs=sort_dropdown_ic, outputs=[ic_callout, ic_table]) | |
# --- Image Captioning Tab --- | |
with gr.TabItem("Image Captioning π"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_icap = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
icap_callout = gr.HTML() | |
icap_table = gr.HTML() | |
init_callout, init_table = update_image_captioning("Low to High") | |
icap_callout.value = init_callout | |
icap_table.value = init_table | |
sort_dropdown_icap.change(fn=update_image_captioning, inputs=sort_dropdown_icap, outputs=[icap_callout, icap_table]) | |
# --- Summarization Tab --- | |
with gr.TabItem("Summarization π"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_sum = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
sum_callout = gr.HTML() | |
sum_table = gr.HTML() | |
init_callout, init_table = update_summarization("Low to High") | |
sum_callout.value = init_callout | |
sum_table.value = init_table | |
sort_dropdown_sum.change(fn=update_summarization, inputs=sort_dropdown_sum, outputs=[sum_callout, sum_table]) | |
# --- Automatic Speech Recognition Tab --- | |
with gr.TabItem("Automatic Speech Recognition π¬"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_asr = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
asr_callout = gr.HTML() | |
asr_table = gr.HTML() | |
init_callout, init_table = update_asr("Low to High") | |
asr_callout.value = init_callout | |
asr_table.value = init_table | |
sort_dropdown_asr.change(fn=update_asr, inputs=sort_dropdown_asr, outputs=[asr_callout, asr_table]) | |
# --- Object Detection Tab --- | |
with gr.TabItem("Object Detection π"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_od = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
od_callout = gr.HTML() | |
od_table = gr.HTML() | |
init_callout, init_table = update_object_detection("Low to High") | |
od_callout.value = init_callout | |
od_table.value = init_table | |
sort_dropdown_od.change(fn=update_object_detection, inputs=sort_dropdown_od, outputs=[od_callout, od_table]) | |
# --- Sentence Similarity Tab --- | |
with gr.TabItem("Sentence Similarity π"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_ss = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
ss_callout = gr.HTML() | |
ss_table = gr.HTML() | |
init_callout, init_table = update_sentence_similarity("Low to High") | |
ss_callout.value = init_callout | |
ss_table.value = init_table | |
sort_dropdown_ss.change(fn=update_sentence_similarity, inputs=sort_dropdown_ss, outputs=[ss_callout, ss_table]) | |
# --- Extractive QA Tab --- | |
with gr.TabItem("Extractive QA β"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_qa = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
qa_callout = gr.HTML() | |
qa_table = gr.HTML() | |
init_callout, init_table = update_extractive_qa("Low to High") | |
qa_callout.value = init_callout | |
qa_table.value = init_table | |
sort_dropdown_qa.change(fn=update_extractive_qa, inputs=sort_dropdown_qa, outputs=[qa_callout, qa_table]) | |
# --- All Tasks Tab --- | |
with gr.TabItem("All Tasks π‘"): | |
with gr.Row(): | |
with gr.Column(scale=8): | |
sort_dropdown_all = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
with gr.Column(scale=4): | |
all_callout = gr.HTML() | |
all_table = gr.HTML() | |
init_callout, init_table = update_all_tasks("Low to High") | |
all_callout.value = init_callout | |
all_table.value = init_table | |
sort_dropdown_all.change(fn=update_all_tasks, inputs=sort_dropdown_all, outputs=[all_callout, all_table]) | |
with gr.Accordion("π Citation", open=False): | |
citation_button = gr.Textbox( | |
value=CITATION_BUTTON_TEXT, | |
label=CITATION_BUTTON_LABEL, | |
elem_id="citation-button", | |
lines=10, | |
show_copy_button=True, | |
) | |
gr.Markdown("Last updated: February 2025") | |
demo.launch() |