add leaderboard

README.md

@@ -1,11 +1,20 @@
 ---
-title: MMWorld
-emoji: 🌖
+title: VBench Leaderboard
+emoji: 📊
 colorFrom: indigo
-colorTo: red
-sdk: static
+colorTo: pink
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
 pinned: false
-license: apache-2.0
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VBench Leaderboard
+
+## Space Description
+
+- **Repository:** [VBench](https://github.com/Vchitect/VBench)
+- **Paper:** [2311.17982]
+(arxiv.org/abs/2311.17982)
+- **Point of Contact:** mailto:[Vchitect](vchitect@pjlab.org.cn)

app.py

@@ -0,0 +1,296 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+import gradio as gr
+import pandas as pd
+
+# Constants
+# =========
+
+# Disciplines
+DISCIPLINES = [
+    "Art & Sports",
+    "Business",
+    "Science",
+    "Health & Medicine",
+    "Embodied Tasks",
+    "Tech & Engineering",
+    "Game"
+]
+
+# Model Information Columns
+MODEL_INFO = [
+    "Model Name (clickable)"
+]
+
+# Column Names for DataFrame
+COLUMN_NAMES = MODEL_INFO + DISCIPLINES + ["Average"]
+
+# Data Types for DataFrame
+DATA_TITILE_TYPE = ['markdown'] + ['number'] * len(DISCIPLINES) + ['number']
+
+# Leaderboard Introduction
+LEADERBOARD_INTRODUCTION = """# MMWorld Leaderboard
+
+*"Towards Multi-discipline Multi-faceted World Model Evaluation in Videos"*  
+🏆 Welcome to the leaderboard of the **MMWorld**! 🎦 *A new benchmark for multi-discipline, multi-faceted multimodal video understanding*  
+[![GitHub](https://img.shields.io/badge/Code-GitHub-black?logo=github)](https://github.com/eric-ai-lab/MMWorld) 
+
+<div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
+<a href='https://arxiv.org/abs/2406.08407'>
+  <img src='https://img.shields.io/badge/cs.CV-Paper-b31b1b?logo=arxiv&logoColor=red'>
+</a>
+<a href='https://mmworld-bench.github.io/'>
+  <img src='https://img.shields.io/badge/MMWorld-Website-green?logo=internet-explorer&logoColor=blue'>
+</a>
+</div>
+
+"""
+
+SUBMIT_INTRODUCTION = """# Submit on MMWorld Benchmark Introduction
+
+## 🎈
+Please obtain the evaluation file `*.json` by running MMWorld in Github and upload the json file below.
+
+⚠️ The contact information you filled in will not be made public. 
+"""
+
+TABLE_INTRODUCTION = """
+The MMWorld Leaderboard showcases the performance of various models across different disciplines. Select the disciplines you're interested in to see how models perform in those areas.
+"""
+
+LEADERBOARD_INFO = """
+Multimodal Language Language Models (MLLMs) demonstrate the emerging abilities of "world models"—interpreting and reasoning about complex real-world dynamics. To assess these abilities, we posit videos are the ideal medium, as they
+encapsulate rich representations of real-world dynamics and causalities. To this end, we introduce MMWorld, a new benchmark for multi-discipline, multi-faceted multimodal video understanding. MMWorld distinguishes itself from previous
+video understanding benchmarks with two unique advantages: (1) multi-discipline, covering various disciplines that often require domain expertise for comprehensive understanding; (2) multi-faceted reasoning, including explanation, counterfactual
+thinking, future prediction, etc. MMWorld consists of a human-annotated dataset to evaluate MLLMs with questions about the whole videos and a synthetic dataset to analyze MLLMs within a single modality of perception.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{he2024mmworld,
+      title={MMWorld: Towards Multi-discipline Multi-faceted World Model Evaluation in Videos}, 
+      author={Xuehai He and Weixi Feng and Kaizhi Zheng and Yujie Lu and Wanrong Zhu and Jiachen Li and Yue Fan and Jianfeng Wang and Linjie Li and Zhengyuan Yang and Kevin Lin and William Yang Wang and Lijuan Wang and Xin Eric Wang},
+      year={2024},
+      eprint={2406.08407},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}"""
+
+# Data: Models and their scores
+data = {
+    "Model Name (clickable)": [
+        "Random Choice",
+        "GPT-4o",
+        "Claude 3.5 Sonnet",
+        "GPT-4V",
+        "Gemini 1.5 Pro",
+        "Video-LLaVA-7B",
+        "Video-Chat-7B",
+        "ChatUnivi-7B",
+        "mPLUG-Owl-7B",
+        "VideoChatGPT-7B",
+        "PandaGPT-7B",
+        "ImageBind-LLM-7B",
+        "X-Instruct-BLIP-7B",
+        "LWM-1M-JAX",
+        "Otter-7B",
+        "Video-LLaMA-2-13B"
+    ],
+    "Art & Sports": [25.03, 47.87, 54.58, 36.17, 37.12, 35.91, 39.53, 24.47, 29.16, 26.84, 25.33, 24.82, 21.08, 12.04, 17.12, 6.15],
+    "Business": [25.09, 91.14, 63.87, 81.59, 76.69, 51.28, 51.05, 60.84, 64.10, 39.16, 42.66, 42.66, 15.85, 17.48, 18.65, 21.21],
+    "Science": [26.44, 73.78, 59.85, 66.52, 62.81, 56.30, 30.81, 52.00, 47.41, 36.45, 39.41, 32.15, 22.52, 15.41, 9.33, 22.22],
+    "Health & Medicine": [25.00, 83.33, 54.51, 73.61, 76.74, 32.64, 46.18, 61.11, 60.07, 53.12, 38.54, 30.21, 28.47, 20.49, 6.94, 31.25],
+    "Embodied Tasks": [26.48, 62.94, 30.99, 55.48, 43.59, 63.17, 40.56, 46.15, 23.78, 36.60, 35.43, 46.85, 18.41, 25.87, 13.29, 15.38],
+    "Tech & Engineering": [30.92, 75.53, 58.87, 61.35, 69.86, 58.16, 39.36, 56.74, 41.84, 41.49, 41.84, 41.49, 22.34, 21.99, 15.96, 19.15],
+    "Game": [25.23, 80.32, 59.44, 73.49, 66.27, 49.00, 44.98, 52.61, 62.25, 36.55, 40.16, 41.37, 26.10, 11.65, 15.26, 24.90]
+}
+
+# Create DataFrame
+df_full = pd.DataFrame(data)
+
+# Function to calculate average score
+def calculate_average(df, disciplines):
+    df['Average'] = df[disciplines].mean(axis=1)
+    return df
+
+# Function to get leaderboard DataFrame based on selected disciplines
+def get_leaderboard_df(selected_disciplines):
+    if not selected_disciplines:
+        selected_disciplines = DISCIPLINES  # If none selected, default to all
+    # Copy the full DataFrame
+    df = df_full.copy()
+    # Calculate the average based on selected disciplines
+    df['Average'] = df[selected_disciplines].mean(axis=1)
+    # Select columns to display
+    columns_to_display = MODEL_INFO + selected_disciplines + ['Average']
+    df = df[columns_to_display]
+    # Sort by Average descending
+    df = df.sort_values(by='Average', ascending=False)
+    return df
+
+# Function to convert scores to two decimal places
+def convert_scores_to_percentage(df):
+    for column in df.columns[1:]:
+        df[column] = df[column].round(2)
+    return df
+
+# Gradio app
+block = gr.Blocks()
+
+with block:
+    gr.Markdown(
+        LEADERBOARD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("📊 MMWorld", elem_id="vbench-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=14,
+                    )
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+            with gr.Row():
+                with gr.Column(scale=0.2):
+                    select_all_button = gr.Button("Select All")
+                    deselect_all_button = gr.Button("Deselect All")
+
+                with gr.Column(scale=0.8):
+                    # Selection for disciplines
+                    checkbox_group = gr.CheckboxGroup(
+                        choices=DISCIPLINES,
+                        value=DISCIPLINES,  # All disciplines selected by default
+                        label="Evaluation discipline",
+                        interactive=True,
+                    )
+
+            # Initial DataFrame
+            initial_df = get_leaderboard_df(DISCIPLINES)
+            initial_df = convert_scores_to_percentage(initial_df)
+
+            data_component = gr.Dataframe(
+                value=initial_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                height=700,
+            )
+
+            # Callbacks for buttons and checkbox changes
+            def update_table(selected_disciplines):
+                updated_df = get_leaderboard_df(selected_disciplines)
+                updated_df = convert_scores_to_percentage(updated_df)
+                return updated_df
+
+            select_all_button.click(
+                fn=lambda: gr.update(value=DISCIPLINES),
+                inputs=None,
+                outputs=checkbox_group
+            ).then(
+                fn=update_table,
+                inputs=checkbox_group,
+                outputs=data_component
+            )
+
+            deselect_all_button.click(
+                fn=lambda: gr.update(value=[]),
+                inputs=None,
+                outputs=checkbox_group
+            ).then(
+                fn=update_table,
+                inputs=checkbox_group,
+                outputs=data_component
+            )
+
+            checkbox_group.change(
+                fn=update_table,
+                inputs=checkbox_group,
+                outputs=data_component
+            )
+
+        # About Tab
+        with gr.TabItem("📝 About", elem_id="mmworld-table", id=2):
+            gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
+
+        # Submit Tab
+        with gr.TabItem("🚀 Submit here!", elem_id="mmworld-tab-table", id=3):
+            gr.Markdown(LEADERBOARD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation JSON file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="**Model name**", placeholder="Required field"
+                    )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name (Optional)", placeholder="GPT4V"
+                    )
+
+                with gr.Column():
+                    model_link = gr.Textbox(
+                        label="**Project Page/Paper Link**", placeholder="Required field"
+                    )
+                    team_name = gr.Textbox(
+                        label="Your Team Name (If left blank, it will be user upload)", placeholder="User Upload"
+                    )
+                    contact_email = gr.Textbox(
+                        label="E-Mail (**Will not be displayed**)", placeholder="Required field"
+                    )
+
+            with gr.Column():
+                input_file = gr.File(label="Click to Upload a ZIP File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+                submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
+                fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text", visible=False)
+
+                submission_result = gr.Markdown()
+
+                # Placeholder function for submission
+                def add_new_eval(
+                    input_file,
+                    model_name_textbox: str,
+                    revision_name_textbox: str,
+                    model_link: str,
+                    team_name: str,
+                    contact_email: str
+                ):
+                    if input_file is None:
+                        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+                    if model_link == '' or model_name_textbox == '' or contact_email == '':
+                        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+                    # Process the uploaded file and submission details here
+                    # For now, we just simulate a successful submission
+                    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+
+                submit_button.click(
+                    add_new_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link,
+                        team_name,
+                        contact_email
+                    ],
+                    outputs=[submit_button, submit_succ_button, fail_textbox]
+                )
+
+    def refresh_data():
+        value1 = get_leaderboard_df(DISCIPLINES)
+        value1 = convert_scores_to_percentage(value1)
+        return value1
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(fn=refresh_data, inputs=None, outputs=data_component)
+
+block.launch()

index.html

@@ -1,19 +0,0 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
-</html>

requirements.txt

@@ -0,0 +1,3 @@
+gradio==3.23.0
+numpy
+pandas

style.css

@@ -1,28 +0,0 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}