app.py

__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
import gradio as gr
import pandas as pd

# Constants
# =========

# Disciplines
DISCIPLINES = [
    "Art & Sports",
    "Business",
    "Science",
    "Health & Medicine",
    "Embodied Tasks",
    "Tech & Engineering",
    "Game"
]

# Model Information Columns
MODEL_INFO = [
    "Model Name (clickable)"
]

# Column Names for DataFrame
COLUMN_NAMES = MODEL_INFO + DISCIPLINES

# Data Types for DataFrame
DATA_TITILE_TYPE = ['markdown'] + ['number'] * len(DISCIPLINES)

# Leaderboard Introduction
LEADERBOARD_INTRODUCTION = """# MMWorld Leaderboard

*"Towards Multi-discipline Multi-faceted World Model Evaluation in Videos"*  
🏆 Welcome to the leaderboard of the **MMWorld**! 🎦 *A new benchmark for multi-discipline, multi-faceted multimodal video understanding*  

<div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
<a href='https://github.com/eric-ai-lab/MMWorld'>
  <img src='https://img.shields.io/badge/Code-GitHub-black?logo=github'>
</a>
<a href='https://arxiv.org/abs/2406.08407'>
  <img src='https://img.shields.io/badge/cs.CV-Paper-b31b1b?logo=arxiv&logoColor=red'>
</a>
<a href='https://mmworld-bench.github.io/'>
  <img src='https://img.shields.io/badge/MMWorld-Website-green?logo=internet-explorer&logoColor=blue'>
</a>
</div>

"""

SUBMIT_INTRODUCTION = """# Submit on MMWorld Benchmark Introduction

## 🎈 Please obtain the evaluation file `*.json` by running MMWorld in Github and upload the json file below.

⚠️ The contact information you filled in will not be made public. 
"""

TABLE_INTRODUCTION = """
The MMWorld Leaderboard showcases the performance of various models across different disciplines. Select the disciplines you're interested in to see how models perform in those areas.
"""

LEADERBOARD_INFO = """
Multimodal Language Language Models (MLLMs) demonstrate the emerging abilities of "world models"—interpreting and reasoning about complex real-world dynamics. To assess these abilities, we posit videos are the ideal medium, as they
encapsulate rich representations of real-world dynamics and causalities. To this end, we introduce MMWorld, a new benchmark for multi-discipline, multi-faceted multimodal video understanding. MMWorld distinguishes itself from previous
video understanding benchmarks with two unique advantages: (1) multi-discipline, covering various disciplines that often require domain expertise for comprehensive understanding; (2) multi-faceted reasoning, including explanation, counterfactual
thinking, future prediction, etc. MMWorld consists of a human-annotated dataset to evaluate MLLMs with questions about the whole videos and a synthetic dataset to analyze MLLMs within a single modality of perception.
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@misc{he2024mmworld,
      title={MMWorld: Towards Multi-discipline Multi-faceted World Model Evaluation in Videos}, 
      author={Xuehai He and Weixi Feng and Kaizhi Zheng and Yujie Lu and Wanrong Zhu and Jiachen Li and Yue Fan and Jianfeng Wang and Linjie Li and Zhengyuan Yang and Kevin Lin and William Yang Wang and Lijuan Wang and Xin Eric Wang},
      year={2024},
      eprint={2406.08407},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}"""

# Data: Models and their scores
data = {
    "Model Name (clickable)": [
        "Random Choice",
        "GPT-4o",
        "Claude 3.5 Sonnet",
        "GPT-4V",
        "Gemini 1.5 Pro",
        "Video-LLaVA-7B",
        "Video-Chat-7B",
        "ChatUnivi-7B",
        "mPLUG-Owl-7B",
        "VideoChatGPT-7B",
        "PandaGPT-7B",
        "ImageBind-LLM-7B",
        "X-Instruct-BLIP-7B",
        "LWM-1M-JAX",
        "Otter-7B",
        "Video-LLaMA-2-13B"
    ],
    "Art & Sports": [25.03, 47.87, 54.58, 36.17, 37.12, 35.91, 39.53, 24.47, 29.16, 26.84, 25.33, 24.82, 21.08, 12.04, 17.12, 6.15],
    "Business": [25.09, 91.14, 63.87, 81.59, 76.69, 51.28, 51.05, 60.84, 64.10, 39.16, 42.66, 42.66, 15.85, 17.48, 18.65, 21.21],
    "Science": [26.44, 73.78, 59.85, 66.52, 62.81, 56.30, 30.81, 52.00, 47.41, 36.45, 39.41, 32.15, 22.52, 15.41, 9.33, 22.22],
    "Health & Medicine": [25.00, 83.33, 54.51, 73.61, 76.74, 32.64, 46.18, 61.11, 60.07, 53.12, 38.54, 30.21, 28.47, 20.49, 6.94, 31.25],
    "Embodied Tasks": [26.48, 62.94, 30.99, 55.48, 43.59, 63.17, 40.56, 46.15, 23.78, 36.60, 35.43, 46.85, 18.41, 25.87, 13.29, 15.38],
    "Tech & Engineering": [30.92, 75.53, 58.87, 61.35, 69.86, 58.16, 39.36, 56.74, 41.84, 41.49, 41.84, 41.49, 22.34, 21.99, 15.96, 19.15],
    "Game": [25.23, 80.32, 59.44, 73.49, 66.27, 49.00, 44.98, 52.61, 62.25, 36.55, 40.16, 41.37, 26.10, 11.65, 15.26, 24.90]
}

# Create DataFrame
df_full = pd.DataFrame(data)

# Function to get leaderboard DataFrame based on selected disciplines
def get_leaderboard_df(selected_disciplines):
    if not selected_disciplines:
        selected_disciplines = DISCIPLINES  # If none selected, default to all
    # Copy the full DataFrame
    df = df_full.copy()
    # Select columns to display
    columns_to_display = MODEL_INFO + selected_disciplines
    df = df[columns_to_display]
    return df

# Function to convert scores to two decimal places
def convert_scores_to_percentage(df):
    for column in df.columns[1:]:
        df[column] = df[column].round(2)
    return df

# Gradio app
block = gr.Blocks()

with block:
    gr.Markdown(
        LEADERBOARD_INTRODUCTION
    )
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("📊 MMWorld", elem_id="mmworld-tab-table", id=1):
            with gr.Row():
                with gr.Accordion("Citation", open=False):
                    citation_button = gr.Textbox(
                        value=CITATION_BUTTON_TEXT,
                        label=CITATION_BUTTON_LABEL,
                        elem_id="citation-button",
                        lines=14,
                    )
            gr.Markdown(
                TABLE_INTRODUCTION
            )
            with gr.Row():
                with gr.Column(scale=0.2):
                    select_all_button = gr.Button("Select All")
                    deselect_all_button = gr.Button("Deselect All")

                with gr.Column(scale=0.8):
                    # Selection for disciplines
                    checkbox_group = gr.CheckboxGroup(
                        choices=DISCIPLINES,
                        value=DISCIPLINES,  # All disciplines selected by default
                        label="Evaluation discipline",
                        interactive=True,
                    )

            # Initial DataFrame
            initial_df = get_leaderboard_df(DISCIPLINES)
            initial_df = convert_scores_to_percentage(initial_df)

            data_component = gr.Dataframe(
                value=initial_df, 
                headers=COLUMN_NAMES,
                type="pandas", 
                datatype=DATA_TITILE_TYPE,
                interactive=False,
                visible=True,
                height=700,
            )

            # Callbacks for buttons and checkbox changes
            def update_table(selected_disciplines):
                updated_df = get_leaderboard_df(selected_disciplines)
                updated_df = convert_scores_to_percentage(updated_df)
                return updated_df

            select_all_button.click(
                fn=lambda: gr.update(value=DISCIPLINES),
                inputs=None,
                outputs=checkbox_group
            ).then(
                fn=update_table,
                inputs=checkbox_group,
                outputs=data_component
            )

            deselect_all_button.click(
                fn=lambda: gr.update(value=[]),
                inputs=None,
                outputs=checkbox_group
            ).then(
                fn=update_table,
                inputs=checkbox_group,
                outputs=data_component
            )

            checkbox_group.change(
                fn=update_table,
                inputs=checkbox_group,
                outputs=data_component
            )

        # About Tab
        with gr.TabItem("📝 About", elem_id="mmworld-table", id=2):
            gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")

        # Submit Tab
        with gr.TabItem("🚀 Submit here!", elem_id="mmworld-tab-table", id=3):
            gr.Markdown(LEADERBOARD_INTRODUCTION, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown("# ✉️✨ Submit your model evaluation JSON file here!", elem_classes="markdown-text")

            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(
                        label="Model name", placeholder="Required field"
                    )
                    revision_name_textbox = gr.Textbox(
                        label="Revision Model Name (Optional)", placeholder="GPT4V"
                    )

                with gr.Column():
                    model_link = gr.Textbox(
                        label="Project Page/Paper Link", placeholder="Required field"
                    )
                    team_name = gr.Textbox(
                        label="Your Team Name (If left blank, it will be user upload)", placeholder="User Upload"
                    )
                    contact_email = gr.Textbox(
                        label="E-Mail (Will not be displayed)", placeholder="Required field"
                    )

            with gr.Column():
                input_file = gr.File(label="Click to Upload a ZIP File", file_count="single", type='binary')
                submit_button = gr.Button("Submit Eval")
                submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
                fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text", visible=False)

                submission_result = gr.Markdown()

                # Placeholder function for submission
                def add_new_eval(
                    input_file,
                    model_name_textbox: str,
                    revision_name_textbox: str,
                    model_link: str,
                    team_name: str,
                    contact_email: str
                ):
                    if input_file is None:
                        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
                    if model_link == '' or model_name_textbox == '' or contact_email == '':
                        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
                    # Process the uploaded file and submission details here
                    # For now, we just simulate a successful submission
                    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)

                submit_button.click(
                    add_new_eval,
                    inputs=[
                        input_file,
                        model_name_textbox,
                        revision_name_textbox,
                        model_link,
                        team_name,
                        contact_email
                    ],
                    outputs=[submit_button, submit_succ_button, fail_textbox]
                )

    def refresh_data():
        value1 = get_leaderboard_df(DISCIPLINES)
        value1 = convert_scores_to_percentage(value1)
        return value1

    with gr.Row():
        data_run = gr.Button("Refresh")
        data_run.click(fn=refresh_data, inputs=None, outputs=data_component)

block.launch()