[Init]

README.md

@@ -1,13 +1,13 @@
 ---
-title: T2V Leaderboard
-emoji: 🏆
+title: Leaderboard
+emoji: 🔥
 colorFrom: indigo
-colorTo: green
+colorTo: pink
 sdk: streamlit
-sdk_version: 1.28.2
+sdk_version: 1.26.0
 app_file: app.py
 pinned: false
-license: mit
+license: cc-by-sa-4.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,130 @@
+import streamlit as st
+st.set_page_config(layout="wide")
+import pandas  as pd
+import os
+import json
+import shutil
+from huggingface_hub import Repository
+
+REFERENCE_NAME = "references"
+SUBMISSION_NAME = "vbench_leaderboard_submission"
+
+
+SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME)
+
+
+TEST_SETS = [
+    "subject consistency",
+    "background consistency",
+    "temporal flickering",
+    "motion smoothness",
+    "dynamic degree",
+    "aesthetic quality",
+    "imaging quality",
+    "object class",
+    "multiple objects",
+    "human action",
+    "color",
+    "spatial relationship",
+    "scene",
+    "appearance style",
+    "temporal style",
+    "overall consistency"
+    ]
+
+style = """
+<style>
+    th {
+        font-size: 10px;
+    }
+</style>
+"""
+
+CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+try:
+    submission_repo = Repository(
+        local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset"
+    )
+except Exception as e:
+    print(e)
+submission_repo.git_pull()
+
+all_submissions = [
+    file_name
+    for file_name in os.listdir(SUBMISSION_NAME)
+    if file_name.endswith('.json')
+]
+
+all_results = pd.read_csv(CSV_RESULTS_FILE)
+with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f:
+    verified_model = [i.strip() for i in f.readlines()]
+all_results['verified'] = all_results['name'].apply(lambda x: '√' if x in verified_model else ' ')
+
+
+# Write table form CSV
+table = all_results.copy()
+table = table.round(2)
+
+
+
+# Streamlit
+st.markdown("# VBench ")
+
+
+st.markdown(
+    f"""
+    This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench).
+    """
+)
+
+sort_option = st.selectbox(
+    'Choose a column to sort by',
+    table.columns[1:-1], 
+)
+table = table.sort_values(by=sort_option, ascending=False)
+
+st.write(style + table.to_markdown(index=False), unsafe_allow_html=True)
+st.markdown(
+    """
+    For more information, refer to the paper submission on [Arxiv](https://).
+    """
+)
+
+st.markdown(
+    """
+    ## Submitting to VBench
+    \n
+    To submit to VBench, download the prompt suite from [VBench/Prompt](https://huggingface.co/datasets//). Upload your zipped submissions for scoring and placement on the leaderboard. 
+    \n
+    Should you experience any issues, open an issue using the link [new discussion](http:) and tag `@vbench`.
+ """
+)
+
+# Using the "with" syntax
+with st.form(key="my_form"):
+    uploaded_file = st.file_uploader("Choose a json file")
+    submit_button = st.form_submit_button(label="Submit")
+
+if submit_button:
+    if uploaded_file is None:
+        raise ValueError("Please make sure to have uploaded a json file.")
+
+    submission = uploaded_file.name.split(".json")[0]
+    with st.spinner(f"Uploading {submission}..."):
+        with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f:
+            f.write(uploaded_file.getvalue())
+        submission_repo.push_to_hub()
+
+    with st.spinner(f"Update Score for {submission}..."):
+        results = {"name": submission}
+        upload_score = json.loads(uploaded_file.getvalue())
+        for info in upload_score:
+            results[info['dimension']] = info['final_score']
+        all_results.loc[len(all_results)] = results
+        all_results.to_csv(CSV_RESULTS_FILE, index=False)
+        commit_url = submission_repo.push_to_hub()
+
+    st.success('Please refresh this space (CTRL+R) to see your result')

requirements.txt

@@ -0,0 +1,3 @@
+huggingface_hub==0.8.1
+pandas
+streamlit