import streamlit as st st.set_page_config(layout="wide") import pandas as pd import os import json import shutil from huggingface_hub import Repository REFERENCE_NAME = "references" SUBMISSION_NAME = "vbench_leaderboard_submission" SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME) TEST_SETS = [ "subject consistency", "background consistency", "temporal flickering", "motion smoothness", "dynamic degree", "aesthetic quality", "imaging quality", "object class", "multiple objects", "human action", "color", "spatial relationship", "scene", "appearance style", "temporal style", "overall consistency" ] style = """ """ CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv") HF_TOKEN = os.environ.get("HF_TOKEN") try: submission_repo = Repository( local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset" ) except Exception as e: print(e) submission_repo.git_pull() all_submissions = [ file_name for file_name in os.listdir(SUBMISSION_NAME) if file_name.endswith('.json') ] all_results = pd.read_csv(CSV_RESULTS_FILE) with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f: verified_model = [i.strip() for i in f.readlines()] all_results['verified'] = all_results['name'].apply(lambda x: '√' if x in verified_model else ' ') # Write table form CSV table = all_results.copy() table = table.round(2) # Streamlit st.markdown("# VBench ") st.markdown( f""" This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench). """ ) sort_option = st.selectbox( 'Choose a column to sort by', table.columns[1:-1], ) table = table.sort_values(by=sort_option, ascending=False) st.write(style + table.to_markdown(index=False), unsafe_allow_html=True) st.markdown( """ For more information, refer to the paper submission on [Arxiv](https://). """ ) st.markdown( """ ## Submitting to VBench \n To submit to VBench, download the prompt suite from [VBench/Prompt](https://huggingface.co/datasets//). Upload your zipped submissions for scoring and placement on the leaderboard. \n Should you experience any issues, open an issue using the link [new discussion](http:) and tag `@vbench`. """ ) # Using the "with" syntax with st.form(key="my_form"): uploaded_file = st.file_uploader("Choose a json file") submit_button = st.form_submit_button(label="Submit") if submit_button: if uploaded_file is None: raise ValueError("Please make sure to have uploaded a json file.") submission = uploaded_file.name.split(".json")[0] with st.spinner(f"Uploading {submission}..."): with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f: f.write(uploaded_file.getvalue()) submission_repo.push_to_hub() with st.spinner(f"Update Score for {submission}..."): results = {"name": submission} upload_score = json.loads(uploaded_file.getvalue()) for info in upload_score: results[info['dimension']] = info['final_score'] all_results.loc[len(all_results)] = results all_results.to_csv(CSV_RESULTS_FILE, index=False) commit_url = submission_repo.push_to_hub() st.success('Please refresh this space (CTRL+R) to see your result')