diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 3cf576d8210e7407165b30f7cd6180e539aee411..0000000000000000000000000000000000000000
--- a/.gitignore
+++ /dev/null
@@ -1,173 +0,0 @@
-checkpoints/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-src/
-# C extensions
-*.so
-temp
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-# For a library or package, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-# This is especially recommended for binary packages to ensure reproducibility, and is more
-# commonly ignored for libraries.
-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-# in version control.
-# https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-# and can be added to the global gitignore or merged into this file. For a more nuclear
-# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-/tmp
-/logs
-/*.json
-/*.jpg
-/*.ipynb
-/GenAI-Arena-hf-logs
-/3DGen-Arena-logs*
-/tmp*
-/arena_elo/results/**/*.jpg
-/arena_elo/results/**/*.png
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..dc9ce287bec66a1bb02b5e7f501d77aa77877734 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "GenAI-Arena-hf-logs"]
+ path = GenAI-Arena-hf-logs
+ url = https://github.com/jdf-prog/GenAI-Arena-hf-logs.git
diff --git a/.idea/.gitignore b/.idea/.gitignore
deleted file mode 100644
index 13566b81b018ad684f3a35fee301741b2734c8f4..0000000000000000000000000000000000000000
--- a/.idea/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml
diff --git a/.idea/GenAI-Arena.iml b/.idea/GenAI-Arena.iml
deleted file mode 100644
index 60c1c77e4977a632de9ba2e9b2463c4465fea56e..0000000000000000000000000000000000000000
--- a/.idea/GenAI-Arena.iml
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
deleted file mode 100644
index 105ce2da2d6447d11dfe32bfb846c3d5b199fc99..0000000000000000000000000000000000000000
--- a/.idea/inspectionProfiles/profiles_settings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
index 2b2a0663cae3524b1775e9eec58231359ad104ba..0000000000000000000000000000000000000000
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 35eb1ddfbbc029bcab630581847471d7f238ec53..0000000000000000000000000000000000000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/README.md b/README.md
index 8ad7eb69f34b4ffef8ce62ab407a16779545daba..7e54b0647406152afddbe69e00ec12f53b3e755f 100644
--- a/README.md
+++ b/README.md
@@ -8,39 +8,6 @@ sdk_version: 4.21.0
app_file: app.py
pinned: false
license: mit
-tags:
-- arena
-- leaderboard
-short_description: Realtime Image/Video Gen AI Arena
---
-## Installation
-
-- for cuda 11.8
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
-pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu118
-pip install -r requirements.txt
-```
-- for cuda 12.1
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
-pip install -r requirements.txt
-```
-
-## Start Hugging Face UI
-```bash
-python app.py
-```
-
-## Start Log server
-```bash
-uvicorn serve.log_server:app --reload --port 22005 --host 0.0.0.0
-```
-
-## Update leaderboard
-```bash
-cd arena_elo && bash update_leaderboard.sh
-```
-
-Paper: arxiv.org/abs/2406.04485
\ No newline at end of file
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
diff --git a/app.py b/app.py
index 3c3dd9a0689d1ce43c57f1216aec32cbe98223bb..d685fb9ebf15ffbb83e693547c61defad38eb294 100644
--- a/app.py
+++ b/app.py
@@ -2,7 +2,6 @@ import gradio as gr
import os
from serve.gradio_web import *
from serve.gradio_web_image_editing import *
-from serve.gradio_web_video_generation import *
from serve.leaderboard import build_leaderboard_tab
from model.model_manager import ModelManager
from pathlib import Path
@@ -24,11 +23,13 @@ def build_combine_demo(models, elo_results_file, leaderboard_table_file):
with gr.Tab("Generation Arena (side-by-side)", id=1):
build_side_by_side_ui_named(models)
- with gr.Tab("Generation Playground", id=2): #Direct Chat
+ with gr.Tab("Generation Direct Chat", id=2):
build_single_model_ui(models, add_promotion_links=True)
if elo_results_file:
with gr.Tab("Generation Leaderboard", id=3):
build_leaderboard_tab(elo_results_file['t2i_generation'], leaderboard_table_file['t2i_generation'])
+ with gr.Tab("About Us", id=4):
+ build_about()
with gr.Tab("Image Edition", id=5):
with gr.Tabs() as tabs_ie:
@@ -38,27 +39,13 @@ def build_combine_demo(models, elo_results_file, leaderboard_table_file):
with gr.Tab("Edition Arena (side-by-side)", id=6):
build_side_by_side_ui_named_ie(models)
- with gr.Tab("Edition Playground", id=7): #Direct Chat
+ with gr.Tab("Edition Direct Chat", id=7):
build_single_model_ui_ie(models, add_promotion_links=True)
if elo_results_file:
with gr.Tab("Edition Leaderboard", id=8):
build_leaderboard_tab(elo_results_file['image_editing'], leaderboard_table_file['image_editing'])
-
- with gr.Tab("Video Generation", id=10):
- with gr.Tabs() as tabs_vg:
- with gr.Tab("Video Generation Arena (battle)", id=10):
- build_side_by_side_ui_anony_vg(models)
-
- with gr.Tab("Video Generation Arena (side-by-side)", id=11):
- build_side_by_side_ui_named_vg(models)
-
- with gr.Tab("Video Generation Playground", id=12): #Direct Chat
- build_single_model_ui_vg(models, add_promotion_links=True)
- if elo_results_file and 'video_generation' in elo_results_file:
- with gr.Tab("Video Generation Leaderboard", id=13):
- build_leaderboard_tab(elo_results_file['video_generation'], leaderboard_table_file['video_generation'])
- with gr.Tab("About Us", id=4):
- build_about()
+ with gr.Tab("About Us", id=9):
+ build_about()
return demo
@@ -76,8 +63,6 @@ def load_elo_results(elo_results_dir):
elo_results_file['t2i_generation'] = file
elif 'image_editing' in file.name:
elo_results_file['image_editing'] = file
- elif 'video_generation' in file.name:
- elo_results_file['video_generation'] = file
else:
raise ValueError(f"Unknown file name: {file.name}")
for file in elo_results_dir.glob('*_leaderboard.csv'):
@@ -85,15 +70,13 @@ def load_elo_results(elo_results_dir):
leaderboard_table_file['t2i_generation'] = file
elif 'image_editing' in file.name:
leaderboard_table_file['image_editing'] = file
- elif 'video_generation' in file.name:
- leaderboard_table_file['video_generation'] = file
else:
raise ValueError(f"Unknown file name: {file.name}")
return elo_results_file, leaderboard_table_file
if __name__ == "__main__":
- server_port = int(SERVER_PORT)
+ server_port = SERVER_PORT
root_path = ROOT_PATH
elo_results_dir = ELO_RESULTS_DIR
models = ModelManager()
diff --git a/arena_elo/edition_model_info.json b/arena_elo/edition_model_info.json
index d3dfbfbefcdd2815d2efa87ec4a999290c9c947c..8d698281eeead890d862beeaa9528951eb01b37f 100644
--- a/arena_elo/edition_model_info.json
+++ b/arena_elo/edition_model_info.json
@@ -33,15 +33,5 @@
"Link": "https://sde-image-editing.github.io",
"License": "MIT License",
"Organization": "Stanford University"
- },
- "CosXLEdit": {
- "Link": "https://huggingface.co/spaces/multimodalart/cosxl",
- "License": "cosxl-nc-community",
- "Organization": "Stability AI"
- },
- "InfEdit": {
- "Link": "https://huggingface.co/spaces/sled-umich/InfEdit",
- "License": "CC BY-NC-ND 4.0",
- "Organization": "University of Michigan, University of California, Berkeley"
}
}
\ No newline at end of file
diff --git a/arena_elo/elo_rating/clean_battle_data.py b/arena_elo/elo_rating/clean_battle_data.py
index fa5719e70aa3c5aca2126a51ca154efd3a80a760..e829f4f37a5a37b065cd105514713f77babc3ae6 100644
--- a/arena_elo/elo_rating/clean_battle_data.py
+++ b/arena_elo/elo_rating/clean_battle_data.py
@@ -82,8 +82,6 @@ def replace_model_name(old_name, tstamp):
"claude-instant-v1": "claude-instant-1",
"oasst-sft-1-pythia-12b": "oasst-pythia-12b",
"claude-2": "claude-2.0",
- "PlayGroundV2": "PlayGround V2",
- "PlayGroundV2.5": "PlayGround V2.5",
}
if old_name in ["gpt-4", "gpt-3.5-turbo"]:
if tstamp > 1687849200:
@@ -107,27 +105,18 @@ def read_file(filename):
break
except FileNotFoundError:
time.sleep(2)
- except json.JSONDecodeError:
- print(f"Error in reading {filename}")
- print(row)
- exit(0)
return data
def read_file_parallel(log_files, num_threads=16):
data_all = []
- if num_threads == 1:
- for log_file in tqdm(log_files, desc="Reading"):
- data_all.extend(read_file(log_file))
- return data_all
- else:
- from multiprocessing import Pool
-
- with Pool(num_threads) as p:
- ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
- for ret in ret_all:
- data_all.extend(ret)
- return data_all
+ from multiprocessing import Pool
+
+ with Pool(num_threads) as p:
+ ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+ for ret in ret_all:
+ data_all.extend(ret)
+ return data_all
def load_image(image_path):
try:
@@ -138,7 +127,7 @@ def load_image(image_path):
def clean_battle_data(
log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple", task_name="image_editing"
):
- data = read_file_parallel(log_files, num_threads=1)
+ data = read_file_parallel(log_files, num_threads=16)
convert_type = {
"leftvote": "model_a",
@@ -156,7 +145,6 @@ def clean_battle_data(
battles = []
for row in tqdm(data, desc="Cleaning"):
if row["models"][0] is None or row["models"][1] is None:
- print(f"Invalid model names: {row['models']}")
continue
# Resolve model names
@@ -175,7 +163,6 @@ def clean_battle_data(
models_public[1] == "" and models_public[0] != ""
):
ct_invalid += 1
- print(f"Invalid model names: {models_public}")
continue
if models_public[0] == "" or models_public[0] == "Model A":
@@ -186,7 +173,6 @@ def clean_battle_data(
anony = False
models = models_public
if not models_public == models_hidden:
- print(f"Model names mismatch: {models_public} vs {models_hidden}")
ct_invalid += 1
continue
@@ -215,88 +201,27 @@ def clean_battle_data(
# if leaked_identity:
# ct_leaked_identity += 1
# continue
-
- def preprocess_model_name(m):
- if m == "Playground v2":
- return 'playground_PlayGroundV2_generation'
- if m == "Playground v2.5":
- return 'playground_PlayGroundV2.5_generation'
- return m
- models = [preprocess_model_name(m) for m in models]
# Replace bard with palm
+ models = [replace_model_name(m, row["tstamp"]) for m in models]
if task_name == "image_editing":
- valid = True
- for _model in models:
- try:
- platform, model_name, task = _model.split("_")
- except ValueError:
- valid = False
- break
- if not (platform in ["playground", "imagenhub"] and task == "edition"):
- valid = False
- break
- if not valid:
+ if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
+ # print(f"Invalid model names: {models}")
ct_invalid += 1
continue
- for i, _model in enumerate(models):
- platform, model_name, task = _model.split("_")
- models[i] = model_name
-
- # if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
- # # print(f"Invalid model names: {models}")
- # ct_invalid += 1
- # continue
-
- # models = [x[len("imagenhub_"):-len("_edition")] for x in models]
+ models = [x[len("imagenhub_"):-len("_edition")] for x in models]
elif task_name == "t2i_generation":
- valid = True
- for _model in models:
- try:
- platform, model_name, task = _model.split("_")
- except ValueError:
- valid = False
- break
- if not (platform.lower() in ["playground", "imagenhub", 'fal'] and (task == "generation" or task == "text2image")):
- valid = False
- break
- if not valid:
+ if not all("playground" in x.lower() or (x.startswith("imagenhub_") and x.endswith("_generation")) for x in models):
+ # print(f"Invalid model names: {models}")
ct_invalid += 1
continue
- for i, _model in enumerate(models):
- platform, model_name, task = _model.split("_")
- models[i] = model_name
- # if not all("playground" in x.lower() or (x.startswith("imagenhub_") and x.endswith("_generation")) for x in models):
- # print(f"Invalid model names: {models}")
- # ct_invalid += 1
- # continue
# models = [x[len("imagenhub_"):-len("_generation")] for x in models]
- # for i, model_name in enumerate(models):
- # mode
- # if model_name.startswith("imagenhub_"):
- # models[i] = model_name[len("imagenhub_"):-len("_generation")]
-
- elif task_name == "video_generation":
- valid = True
- for _model in models:
- try:
- platform, model_name, task = _model.split("_")
- except ValueError:
- valid = False
- break
- if not (platform in ["videogenhub", "fal"] and task == "generation" or task == "text2video"):
- valid = False
- break
- if not valid:
- ct_invalid += 1
- continue
- for i, _model in enumerate(models):
- platform, model_name, task = _model.split("_")
- models[i] = model_name
+ for i, model_name in enumerate(models):
+ if model_name.startswith("imagenhub_"):
+ models[i] = model_name[len("imagenhub_"):-len("_generation")]
else:
raise ValueError(f"Invalid task_name: {task_name}")
- models = [replace_model_name(m, row["tstamp"]) for m in models]
# Exclude certain models
if exclude_model_names and any(x in exclude_model_names for x in models):
@@ -356,7 +281,6 @@ def clean_battle_data(
if ban_ip_list is not None and ip in ban_ip_list:
ct_banned += 1
- print(f"User {user_id} is banned")
continue
# Save the results
@@ -408,7 +332,7 @@ if __name__ == "__main__":
parser.add_argument(
"--mode", type=str, choices=["simple", "conv_release"], default="simple"
)
- parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation", "video_generation"])
+ parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation"])
parser.add_argument("--exclude-model-names", type=str, nargs="+")
parser.add_argument("--ban-ip-file", type=str)
parser.add_argument("--sanitize-ip", action="store_true", default=False)
diff --git a/arena_elo/elo_rating/elo_analysis.py b/arena_elo/elo_rating/elo_analysis.py
index a66a018a8e76f96a2692468848bbda001d3e016b..67a5795c61cf846ac9a1b8419509e02cccd568d1 100644
--- a/arena_elo/elo_rating/elo_analysis.py
+++ b/arena_elo/elo_rating/elo_analysis.py
@@ -171,20 +171,8 @@ def visualize_pairwise_win_fraction(battles, model_order):
xaxis_side="top",
title_y=0.07,
title_x=0.5,
- # xaxis=dict(
- # tickfont=dict(size=16),
- # title=dict(font=dict(size=16)),
- # ),
- # yaxis=dict(
- # tickfont=dict(size=16),
- # title=dict(font=dict(size=16)),
- # ),
)
fig.update_traces(
- # textfont=dict(size=16),
- # colorbar=dict(
- # title=dict(font=dict(size=16))
- # ),
hovertemplate="Model A: %{y}
Model B: %{x}
Fraction of A Wins: %{z}"
)
@@ -208,20 +196,8 @@ def visualize_battle_count(battles, model_order):
xaxis_side="top",
title_y=0.07,
title_x=0.5,
- # xaxis=dict(
- # tickfont=dict(size=16),
- # title=dict(font=dict(size=16)),
- # ),
- # yaxis=dict(
- # tickfont=dict(size=16),
- # title=dict(font=dict(size=16)),
- # ),
)
fig.update_traces(
- # textfont=dict(size=16),
- # colorbar=dict(
- # title=dict(font=dict(size=16))
- # ),
hovertemplate="Model A: %{y}
Model B: %{x}
Count: %{z}