Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import os | |
| import zipfile | |
| import base64 | |
| CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
| CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, | |
| author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, | |
| title = {AI Energy Score Leaderboard - February 2025}, | |
| year = {2025}, | |
| publisher = {Hugging Face}, | |
| howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", | |
| }""" | |
| # List of tasks (CSV filenames) | |
| tasks = [ | |
| 'asr.csv', | |
| 'object_detection.csv', | |
| 'text_classification.csv', | |
| 'image_captioning.csv', | |
| 'question_answering.csv', | |
| 'text_generation.csv', | |
| 'image_classification.csv', | |
| 'sentence_similarity.csv', | |
| 'image_generation.csv', | |
| 'summarization.csv' | |
| ] | |
| ### HELPER FUNCTIONS ### | |
| def format_stars(score): | |
| try: | |
| score_int = int(score) | |
| except Exception: | |
| score_int = 0 | |
| return f'<span style="color: #3fa45bff; font-size:1.5em;">{"★" * score_int}</span>' | |
| def make_link(mname): | |
| parts = str(mname).split('/') | |
| display_name = parts[1] if len(parts) > 1 else mname | |
| return f'<a href="https://huggingface.co/{mname}" target="_blank">{display_name}</a>' | |
| def extract_link_text(html_link): | |
| start = html_link.find('>') + 1 | |
| end = html_link.rfind('</a>') | |
| if start > 0 and end > start: | |
| return html_link[start:end] | |
| else: | |
| return html_link | |
| def generate_html_table_from_df(df): | |
| # Compute a static width for the Model column based on the longest model name. | |
| if not df.empty: | |
| max_length = max(len(extract_link_text(link)) for link in df['Model']) | |
| else: | |
| max_length = 10 | |
| static_width = max_length * 10 + 16 | |
| max_energy = df['gpu_energy_numeric'].max() if not df.empty else 1 | |
| color_map = {"1": "black", "2": "black", "3": "black", "4": "black", "5": "black"} | |
| html = '<table class="data-table" style="width:100%; border-collapse: collapse; font-family: Inter, sans-serif;">' | |
| html += '<thead><tr style="background-color: #f2f2f2;">' | |
| html += '<th style="text-align: left; padding: 8px;" title="Model name with link to Hugging Face">Model</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="AI Provider extracted from the model name">Provider</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="GPU energy consumed in Watt-hours for 1,000 queries">GPU Energy (Wh)</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="Energy efficiency score (stars)">Score</th>' | |
| html += '</tr></thead>' | |
| html += '<tbody>' | |
| for _, row in df.iterrows(): | |
| energy_numeric = row['gpu_energy_numeric'] | |
| energy_str = f"{energy_numeric:,.2f}" | |
| bar_width = (energy_numeric / max_energy) * 100 | |
| score_val = row['energy_score'] | |
| bar_color = color_map.get(str(score_val), "gray") | |
| html += '<tr>' | |
| html += f'<td style="padding: 8px; width: {static_width}px;">{row["Model"]}</td>' | |
| html += f'<td style="padding: 8px;">{row["Provider"]}</td>' | |
| html += (f'<td style="padding: 8px;">{energy_str}<br>' | |
| f'<div style="background-color: {bar_color}; width: {bar_width:.1f}%; height: 10px;"></div></td>') | |
| html += f'<td style="padding: 8px;">{row["Score"]}</td>' | |
| html += '</tr>' | |
| html += '</tbody></table>' | |
| return f'<div class="table-container">{html}</div>' | |
| def process_df(task, sort_order="Low to High", filter_fn=None): | |
| df = pd.read_csv(os.path.join("data", "energy", task)) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['energy_score'] = df['energy_score'].astype(int) | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| if filter_fn is not None: | |
| df = filter_fn(df) | |
| df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
| df['Model'] = df['model'].apply(make_link) | |
| df['Score'] = df['energy_score'].apply(format_stars) | |
| ascending = True if sort_order == "Low to High" else False | |
| df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
| return df | |
| def compute_efficiency_ratio(df): | |
| if df.empty: | |
| return 1 | |
| min_val = df['gpu_energy_numeric'].min() | |
| max_val = df['gpu_energy_numeric'].max() | |
| ratio = max_val / min_val if min_val > 0 else 1 | |
| return ratio | |
| def generate_info_callout(ratio, scope_text): | |
| """ | |
| Returns a "did you know" callout with a lightbulb emoji. | |
| The callout uses a light green background, a small font, and is limited to a max-width of 250px. | |
| It is wrapped in a container that aligns it to the right. | |
| """ | |
| return ( | |
| f'<div style="text-align: right;">' | |
| f'<div class="info-callout" style="display:inline-block; max-width:250px; font-size:0.8em; background-color:#e6ffe6; padding:8px; border-radius:5px;">' | |
| f'💡 There\'s a <strong>{ratio:,.1f}x</strong> difference between the highest and lowest energy use in {scope_text}.' | |
| f'</div></div>' | |
| ) | |
| def get_global_callout(): | |
| all_df = pd.DataFrame() | |
| for task in tasks: | |
| df = pd.read_csv(os.path.join("data", "energy", task)) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| all_df = pd.concat([all_df, df], ignore_index=True) | |
| ratio = compute_efficiency_ratio(all_df) | |
| return generate_info_callout(ratio, "this leaderboard") | |
| ### ZIP DOWNLOAD FUNCTIONS ### | |
| def zip_csv_files(): | |
| data_dir = os.path.join("data", "energy") | |
| zip_filename = "data.zip" | |
| with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: | |
| for filename in os.listdir(data_dir): | |
| if filename.endswith(".csv"): | |
| filepath = os.path.join(data_dir, filename) | |
| zipf.write(filepath, arcname=filename) | |
| return zip_filename | |
| def get_zip_data_link(): | |
| zip_filename = zip_csv_files() | |
| with open(zip_filename, "rb") as f: | |
| data = f.read() | |
| b64 = base64.b64encode(data).decode() | |
| href = ( | |
| f'<a href="data:application/zip;base64,{b64}" ' | |
| 'download="data.zip" ' | |
| 'style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: inherit; font-family: \'Inter\', sans-serif;">Download Data</a>' | |
| ) | |
| return href | |
| ### UPDATE FUNCTIONS (RETURNING CALLOUT AND TABLE HTML) ### | |
| def update_text_generation(selected_display, sort_order): | |
| mapping = { | |
| "A (Single Consumer GPU) <20B parameters": "A", | |
| "B (Single Cloud GPU) 20-66B parameters": "B", | |
| "C (Multiple Cloud GPUs) >66B parameters": "C" | |
| } | |
| model_class = mapping.get(selected_display, "A") | |
| def filter_fn(df): | |
| if 'class' in df.columns: | |
| return df[df['class'] == model_class] | |
| return df | |
| df = process_df('text_generation.csv', sort_order, filter_fn) | |
| ratio = compute_efficiency_ratio(df) | |
| # For Text Generation, use "this class" as the scope. | |
| callout = generate_info_callout(ratio, "this class") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_image_generation(sort_order): | |
| df = process_df('image_generation.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_text_classification(sort_order): | |
| df = process_df('text_classification.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_image_classification(sort_order): | |
| df = process_df('image_classification.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_image_captioning(sort_order): | |
| df = process_df('image_captioning.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_summarization(sort_order): | |
| df = process_df('summarization.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_asr(sort_order): | |
| df = process_df('asr.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_object_detection(sort_order): | |
| df = process_df('object_detection.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_sentence_similarity(sort_order): | |
| df = process_df('sentence_similarity.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_extractive_qa(sort_order): | |
| df = process_df('question_answering.csv', sort_order) | |
| ratio = compute_efficiency_ratio(df) | |
| callout = generate_info_callout(ratio, "this task") | |
| table_html = generate_html_table_from_df(df) | |
| return callout, table_html | |
| def update_all_tasks(sort_order): | |
| all_df = pd.DataFrame() | |
| for task in tasks: | |
| df = pd.read_csv(os.path.join("data", "energy", task)) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['energy_score'] = df['energy_score'].astype(int) | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
| df['Model'] = df['model'].apply(make_link) | |
| df['Score'] = df['energy_score'].apply(format_stars) | |
| all_df = pd.concat([all_df, df], ignore_index=True) | |
| all_df = all_df.drop_duplicates(subset=['model']) | |
| ascending = True if sort_order == "Low to High" else False | |
| all_df = all_df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
| ratio = compute_efficiency_ratio(all_df) | |
| callout = generate_info_callout(ratio, "this leaderboard") | |
| table_html = generate_html_table_from_df(all_df) | |
| return callout, table_html | |
| ### GLOBAL HEADER (Logo & Global Callout) ### | |
| # Use a <picture> element so that dark mode uses logodark.png. | |
| global_header_html = f""" | |
| <div style="position: relative; width: 100%; text-align: center; margin-bottom: 20px;"> | |
| <picture style="display:inline-block;"> | |
| <source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logodark.png"> | |
| <img src="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logo.png" | |
| alt="Logo" | |
| style="width:300px; max-width:300px; height:auto; display:inline-block;"> | |
| </picture> | |
| <div style="position: absolute; top: 50%; right: 20px; transform: translateY(-50%);"> | |
| {get_global_callout()} | |
| </div> | |
| </div> | |
| """ | |
| ### CUSTOM CSS for Dark Mode and Mobile Responsiveness ### | |
| custom_css = """ | |
| /* Table and layout */ | |
| .data-table { | |
| table-layout: fixed; | |
| width: 100%; | |
| } | |
| .data-table th, .data-table td { | |
| max-width: 150px; | |
| white-space: nowrap; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| } | |
| .table-container { | |
| width: 100%; | |
| margin-left: auto; | |
| margin-right: auto; | |
| } | |
| /* Force header links to be black in light mode */ | |
| .header-link { | |
| color: black !important; | |
| } | |
| /* Dark mode styles */ | |
| @media (prefers-color-scheme: dark) { | |
| body { | |
| background-color: #121212; | |
| color: #e0e0e0; | |
| } | |
| .data-table thead { | |
| background-color: #333; | |
| } | |
| .data-table th { | |
| color: #e0e0e0; | |
| } | |
| .data-table td { | |
| color: #e0e0e0; | |
| } | |
| /* Non-header links in dark mode */ | |
| a:not(.header-link) { | |
| color: #3fa45bff !important; | |
| } | |
| } | |
| /* Mobile styles: hide callout boxes on small screens */ | |
| @media (max-width: 600px) { | |
| .info-callout { | |
| display: none !important; | |
| } | |
| } | |
| """ | |
| ### GRADIO INTERFACE ### | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| # --- Header Links --- | |
| gr.HTML(f""" | |
| <div style="display: flex; justify-content: space-evenly; align-items: center; margin-bottom: 20px;"> | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/submission_portal" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Submission Portal</a> | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/Label" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Label Generator</a> | |
| <a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#faq" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">FAQ</a> | |
| <a class="header-link" href="https://huggingface.github.io/AIEnergyScore/#documentation" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Documentation</a> | |
| {get_zip_data_link()} | |
| <a class="header-link" href="https://huggingface.co/spaces/AIEnergyScore/README/discussions" style="text-decoration: none; font-weight: bold; font-size: 1.1em;">Community</a> | |
| </div> | |
| """) | |
| # --- Global Header: Centered Logo with Global Callout at Right Edge --- | |
| gr.HTML(global_header_html) | |
| # --- Tabs for the different tasks --- | |
| with gr.Tabs(): | |
| # --- Text Generation Tab --- | |
| with gr.TabItem("Text Generation 💬"): | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| model_class_options = [ | |
| "A (Single Consumer GPU) <20B parameters", | |
| "B (Single Cloud GPU) 20-66B parameters", | |
| "C (Multiple Cloud GPUs) >66B parameters" | |
| ] | |
| model_class_dropdown = gr.Dropdown(choices=model_class_options, label="Select Model Class", value=model_class_options[0]) | |
| with gr.Column(scale=4): | |
| sort_dropdown_tg = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| tg_callout = gr.HTML() | |
| tg_table = gr.HTML() | |
| init_callout, init_table = update_text_generation(model_class_options[0], "Low to High") | |
| tg_callout.value = init_callout | |
| tg_table.value = init_table | |
| model_class_dropdown.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) | |
| sort_dropdown_tg.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) | |
| # --- Image Generation Tab --- | |
| with gr.TabItem("Image Generation 📷"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_img = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| img_callout = gr.HTML() | |
| img_table = gr.HTML() | |
| init_callout, init_table = update_image_generation("Low to High") | |
| img_callout.value = init_callout | |
| img_table.value = init_table | |
| sort_dropdown_img.change(fn=update_image_generation, inputs=sort_dropdown_img, outputs=[img_callout, img_table]) | |
| # --- Text Classification Tab --- | |
| with gr.TabItem("Text Classification 🎭"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_tc = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| tc_callout = gr.HTML() | |
| tc_table = gr.HTML() | |
| init_callout, init_table = update_text_classification("Low to High") | |
| tc_callout.value = init_callout | |
| tc_table.value = init_table | |
| sort_dropdown_tc.change(fn=update_text_classification, inputs=sort_dropdown_tc, outputs=[tc_callout, tc_table]) | |
| # --- Image Classification Tab --- | |
| with gr.TabItem("Image Classification 🖼️"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_ic = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| ic_callout = gr.HTML() | |
| ic_table = gr.HTML() | |
| init_callout, init_table = update_image_classification("Low to High") | |
| ic_callout.value = init_callout | |
| ic_table.value = init_table | |
| sort_dropdown_ic.change(fn=update_image_classification, inputs=sort_dropdown_ic, outputs=[ic_callout, ic_table]) | |
| # --- Image Captioning Tab --- | |
| with gr.TabItem("Image Captioning 📝"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_icap = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| icap_callout = gr.HTML() | |
| icap_table = gr.HTML() | |
| init_callout, init_table = update_image_captioning("Low to High") | |
| icap_callout.value = init_callout | |
| icap_table.value = init_table | |
| sort_dropdown_icap.change(fn=update_image_captioning, inputs=sort_dropdown_icap, outputs=[icap_callout, icap_table]) | |
| # --- Summarization Tab --- | |
| with gr.TabItem("Summarization 📃"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_sum = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| sum_callout = gr.HTML() | |
| sum_table = gr.HTML() | |
| init_callout, init_table = update_summarization("Low to High") | |
| sum_callout.value = init_callout | |
| sum_table.value = init_table | |
| sort_dropdown_sum.change(fn=update_summarization, inputs=sort_dropdown_sum, outputs=[sum_callout, sum_table]) | |
| # --- Automatic Speech Recognition Tab --- | |
| with gr.TabItem("Automatic Speech Recognition 💬"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_asr = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| asr_callout = gr.HTML() | |
| asr_table = gr.HTML() | |
| init_callout, init_table = update_asr("Low to High") | |
| asr_callout.value = init_callout | |
| asr_table.value = init_table | |
| sort_dropdown_asr.change(fn=update_asr, inputs=sort_dropdown_asr, outputs=[asr_callout, asr_table]) | |
| # --- Object Detection Tab --- | |
| with gr.TabItem("Object Detection 🚘"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_od = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| od_callout = gr.HTML() | |
| od_table = gr.HTML() | |
| init_callout, init_table = update_object_detection("Low to High") | |
| od_callout.value = init_callout | |
| od_table.value = init_table | |
| sort_dropdown_od.change(fn=update_object_detection, inputs=sort_dropdown_od, outputs=[od_callout, od_table]) | |
| # --- Sentence Similarity Tab --- | |
| with gr.TabItem("Sentence Similarity 📚"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_ss = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| ss_callout = gr.HTML() | |
| ss_table = gr.HTML() | |
| init_callout, init_table = update_sentence_similarity("Low to High") | |
| ss_callout.value = init_callout | |
| ss_table.value = init_table | |
| sort_dropdown_ss.change(fn=update_sentence_similarity, inputs=sort_dropdown_ss, outputs=[ss_callout, ss_table]) | |
| # --- Extractive QA Tab --- | |
| with gr.TabItem("Extractive QA ❔"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_qa = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| qa_callout = gr.HTML() | |
| qa_table = gr.HTML() | |
| init_callout, init_table = update_extractive_qa("Low to High") | |
| qa_callout.value = init_callout | |
| qa_table.value = init_table | |
| sort_dropdown_qa.change(fn=update_extractive_qa, inputs=sort_dropdown_qa, outputs=[qa_callout, qa_table]) | |
| # --- All Tasks Tab --- | |
| with gr.TabItem("All Tasks 💡"): | |
| with gr.Row(): | |
| with gr.Column(scale=8): | |
| sort_dropdown_all = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") | |
| with gr.Column(scale=4): | |
| all_callout = gr.HTML() | |
| all_table = gr.HTML() | |
| init_callout, init_table = update_all_tasks("Low to High") | |
| all_callout.value = init_callout | |
| all_table.value = init_table | |
| sort_dropdown_all.change(fn=update_all_tasks, inputs=sort_dropdown_all, outputs=[all_callout, all_table]) | |
| with gr.Accordion("📙 Citation", open=False): | |
| citation_button = gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| elem_id="citation-button", | |
| lines=10, | |
| show_copy_button=True, | |
| ) | |
| gr.Markdown("Last updated: February 2025") | |
| demo.launch() |