Spaces:
Running
Running
import streamlit as st | |
st.set_page_config(layout="wide") | |
import pandas as pd | |
import os | |
import json | |
import shutil | |
from huggingface_hub import Repository | |
REFERENCE_NAME = "references" | |
SUBMISSION_NAME = "vbench_leaderboard_submission" | |
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME) | |
TEST_SETS = [ | |
"subject consistency", | |
"background consistency", | |
"temporal flickering", | |
"motion smoothness", | |
"dynamic degree", | |
"aesthetic quality", | |
"imaging quality", | |
"object class", | |
"multiple objects", | |
"human action", | |
"color", | |
"spatial relationship", | |
"scene", | |
"appearance style", | |
"temporal style", | |
"overall consistency" | |
] | |
style = """ | |
<style> | |
th { | |
font-size: 10px; | |
} | |
</style> | |
""" | |
CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv") | |
HF_TOKEN = os.environ.get("HF_TOKEN") | |
try: | |
submission_repo = Repository( | |
local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset" | |
) | |
except Exception as e: | |
print(e) | |
submission_repo.git_pull() | |
all_submissions = [ | |
file_name | |
for file_name in os.listdir(SUBMISSION_NAME) | |
if file_name.endswith('.json') | |
] | |
all_results = pd.read_csv(CSV_RESULTS_FILE) | |
with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f: | |
verified_model = [i.strip() for i in f.readlines()] | |
all_results['verified'] = all_results['name'].apply(lambda x: 'β' if x in verified_model else ' ') | |
# Write table form CSV | |
table = all_results.copy() | |
table = table.round(2) | |
# Streamlit | |
st.markdown("# VBench ") | |
st.markdown( | |
f""" | |
This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench). | |
""" | |
) | |
sort_option = st.selectbox( | |
'Choose a column to sort by', | |
table.columns[1:-1], | |
) | |
table = table.sort_values(by=sort_option, ascending=False) | |
st.write(style + table.to_markdown(index=False), unsafe_allow_html=True) | |
st.markdown( | |
""" | |
For more information, refer to the paper submission on [Arxiv](https://). | |
""" | |
) | |
st.markdown( | |
""" | |
## Submitting to VBench | |
\n | |
To submit to VBench, download the prompt suite from [VBench/Prompt](https://huggingface.co/datasets//). Upload your zipped submissions for scoring and placement on the leaderboard. | |
\n | |
Should you experience any issues, open an issue using the link [new discussion](http:) and tag `@vbench`. | |
""" | |
) | |
# Using the "with" syntax | |
with st.form(key="my_form"): | |
uploaded_file = st.file_uploader("Choose a json file") | |
submit_button = st.form_submit_button(label="Submit") | |
if submit_button: | |
if uploaded_file is None: | |
raise ValueError("Please make sure to have uploaded a json file.") | |
submission = uploaded_file.name.split(".json")[0] | |
with st.spinner(f"Uploading {submission}..."): | |
with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f: | |
f.write(uploaded_file.getvalue()) | |
submission_repo.push_to_hub() | |
with st.spinner(f"Update Score for {submission}..."): | |
results = {"name": submission} | |
upload_score = json.loads(uploaded_file.getvalue()) | |
for info in upload_score: | |
results[info['dimension']] = info['final_score'] | |
all_results.loc[len(all_results)] = results | |
all_results.to_csv(CSV_RESULTS_FILE, index=False) | |
commit_url = submission_repo.push_to_hub() | |
st.success('Please refresh this space (CTRL+R) to see your result') |