import streamlit as st
st.set_page_config(layout="wide")
import pandas as pd
import os
import json
import shutil
from huggingface_hub import Repository
REFERENCE_NAME = "references"
SUBMISSION_NAME = "vbench_leaderboard_submission"
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Vchitect/", SUBMISSION_NAME)
TEST_SETS = [
"subject consistency",
"background consistency",
"temporal flickering",
"motion smoothness",
"dynamic degree",
"aesthetic quality",
"imaging quality",
"object class",
"multiple objects",
"human action",
"color",
"spatial relationship",
"scene",
"appearance style",
"temporal style",
"overall consistency"
]
# style = """
#
# """
style = """
"""
CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")
HF_TOKEN = os.environ.get("HF_TOKEN")
try:
submission_repo = Repository(
local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset"
)
submission_repo.git_pull()
except Exception as e:
print(e)
all_submissions = [
file_name
for file_name in os.listdir(SUBMISSION_NAME)
if file_name.endswith('.json')
]
all_results = pd.read_csv(CSV_RESULTS_FILE)
with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f:
verified_model = [i.strip().lower() for i in f.readlines()]
all_results['verified'] = all_results['Model Name (clickable)'].apply(lambda x: '√' if x.lower() in verified_model else ' ')
# Write table form CSV
table = all_results.copy()
table = table.round(4)
# columns_to_convert = table.columns[1:-1]
# table[columns_to_convert] = table[columns_to_convert].applymap(lambda x: f'{x * 100}'[:5]+'%')
# Streamlit
st.markdown("# VBench ")
# st.markdown(
# f"""
# This is the leaderboard of VBench.
# > **VBench: Comprehensive Benchmark Suite for Video Generative Models**
# > [[Paper](https://vchitect.github.io/VBench-project/assets/vbench/VBench_paper.pdf)] | [[Project Page](https://vchitect.github.io/VBench-project/)] | [[GitHub Code](https://github.com/Vchitect/VBench)] | [[Video](https://www.youtube.com/watch?v=7IhCC8Qqn8Y)]
# """
# )
st.markdown("This is the leaderboard of VBench.")
st.markdown("> **VBench: Comprehensive Benchmark Suite for Video Generative Models** ([Paper](https://vchitect.github.io/VBench-project/assets/vbench/VBench_paper.pdf), [Project Page](https://vchitect.github.io/VBench-project/), [GitHub Code](https://github.com/Vchitect/VBench), [Video](https://www.youtube.com/watch?v=7IhCC8Qqn8Y))")
default_index = list(table.columns[1:-1]).index("overall consistency") if "overall consistency" in table.columns else 0
sort_option = st.selectbox(
'Choose a column to sort by',
table.columns[1:-1],
index=default_index,
)
table = table.sort_values(by=sort_option, ascending=False)
st.write(style + table.to_markdown(index=False), unsafe_allow_html=True)
# st.markdown(
# """
# For more information, refer to the paper submission on [Arxiv](https://).
# """
# )
st.markdown(
"""
## Submitting to VBench
\n
To submit to VBench, download the prompt suite from [VBench/Prompt](https://github.com/Vchitect/VBench/tree/master/prompts). Upload your zipped submissions for scoring and placement on the leaderboard.
\n
Should you experience any issues, open an issue using the link [new discussion](https://huggingface.co/spaces/Vchitect/VBench_Leaderboard/discussions) and tag `@Ziqi` or `@ynhe`.
"""
)
# Using the "with" syntax
with st.form(key="my_form"):
uploaded_file = st.file_uploader("Choose a json file")
submit_button = st.form_submit_button(label="Submit")
if submit_button:
if uploaded_file is None:
raise ValueError("Please make sure to have uploaded a json file.")
submission = uploaded_file.name.split(".json")[0]
with st.spinner(f"Uploading {submission}..."):
with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f:
f.write(uploaded_file.getvalue())
submission_repo.push_to_hub()
with st.spinner(f"Update Score for {submission}..."):
results = {"name": submission}
upload_score = json.loads(uploaded_file.getvalue())
for info in upload_score:
results[info['dimension']] = info['final_score']
all_results.loc[len(all_results)] = results
all_results.to_csv(CSV_RESULTS_FILE, index=False)
commit_url = submission_repo.push_to_hub()
st.success('Please refresh this space (CTRL+R) to see your result')