Spaces:
Running
Running
File size: 3,556 Bytes
bf9e163 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 |
import streamlit as st
st.set_page_config(layout="wide")
import pandas as pd
import os
import json
import shutil
from huggingface_hub import Repository
REFERENCE_NAME = "references"
SUBMISSION_NAME = "vbench_leaderboard_submission"
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME)
TEST_SETS = [
"subject consistency",
"background consistency",
"temporal flickering",
"motion smoothness",
"dynamic degree",
"aesthetic quality",
"imaging quality",
"object class",
"multiple objects",
"human action",
"color",
"spatial relationship",
"scene",
"appearance style",
"temporal style",
"overall consistency"
]
style = """
<style>
th {
font-size: 10px;
}
</style>
"""
CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")
HF_TOKEN = os.environ.get("HF_TOKEN")
try:
submission_repo = Repository(
local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset"
)
except Exception as e:
print(e)
submission_repo.git_pull()
all_submissions = [
file_name
for file_name in os.listdir(SUBMISSION_NAME)
if file_name.endswith('.json')
]
all_results = pd.read_csv(CSV_RESULTS_FILE)
with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f:
verified_model = [i.strip() for i in f.readlines()]
all_results['verified'] = all_results['name'].apply(lambda x: '√' if x in verified_model else ' ')
# Write table form CSV
table = all_results.copy()
table = table.round(2)
# Streamlit
st.markdown("# VBench ")
st.markdown(
f"""
This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench).
"""
)
sort_option = st.selectbox(
'Choose a column to sort by',
table.columns[1:-1],
)
table = table.sort_values(by=sort_option, ascending=False)
st.write(style + table.to_markdown(index=False), unsafe_allow_html=True)
st.markdown(
"""
For more information, refer to the paper submission on [Arxiv](https://).
"""
)
st.markdown(
"""
## Submitting to VBench
\n
To submit to VBench, download the prompt suite from [VBench/Prompt](https://huggingface.co/datasets//). Upload your zipped submissions for scoring and placement on the leaderboard.
\n
Should you experience any issues, open an issue using the link [new discussion](http:) and tag `@vbench`.
"""
)
# Using the "with" syntax
with st.form(key="my_form"):
uploaded_file = st.file_uploader("Choose a json file")
submit_button = st.form_submit_button(label="Submit")
if submit_button:
if uploaded_file is None:
raise ValueError("Please make sure to have uploaded a json file.")
submission = uploaded_file.name.split(".json")[0]
with st.spinner(f"Uploading {submission}..."):
with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f:
f.write(uploaded_file.getvalue())
submission_repo.push_to_hub()
with st.spinner(f"Update Score for {submission}..."):
results = {"name": submission}
upload_score = json.loads(uploaded_file.getvalue())
for info in upload_score:
results[info['dimension']] = info['final_score']
all_results.loc[len(all_results)] = results
all_results.to_csv(CSV_RESULTS_FILE, index=False)
commit_url = submission_repo.push_to_hub()
st.success('Please refresh this space (CTRL+R) to see your result') |