o0o

.gitattributes

@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-examples/duck.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,175 +0,0 @@
-checkpoints/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-src/
-# C extensions
-*.so
-temp
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-/tmp
-/logs
-/*.json
-/*.jpg
-/*.ipynb
-/GenAI-Arena-hf-logs
-/3DGen-Arena-logs*
-/tmp*
-/arena_elo/results/**/*.jpg
-/arena_elo/results/**/*.png
-/arena_elo/6_04_log_results
-/arena_elo/update_elo_rating_6_04.sh

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/GenAI-Arena.iml

@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="GOOGLE" />
-    <option name="myDocStringFormat" value="Google" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/GenAI-Arena.iml" filepath="$PROJECT_DIR$/.idea/GenAI-Arena.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

README.md

@@ -1,47 +1,11 @@
 ---
-title: GenAI Arena
-emoji: 📈
-colorFrom: purple
-colorTo: pink
-sdk: gradio
-sdk_version: 4.41.0
-python_version: 3.12
-app_file: app.py
-pinned: true
-license: mit
-tags:
-- arena
-- leaderboard
-short_description: Realtime Image/Video Gen AI Arena
+title: GenAI-Arena
+emoji: 🚀
+colorFrom: indigo
+colorTo: yellow
+sdk: static
+pinned: false
+header: mini
 ---
 
-## Installation
-
-- for cuda 11.8
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
-pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu118
-pip install -r requirements.txt
-```
-- for cuda 12.1
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
-pip install -r requirements.txt
-```
-
-## Start Hugging Face UI
-```bash
-python app.py
-```
-
-## Start Log server
-```bash
-uvicorn serve.log_server:app --reload --port 22005 --host 0.0.0.0
-```
-
-## Update leaderboard
-```bash
-cd arena_elo && bash update_leaderboard.sh
-```
-
-Paper: arxiv.org/abs/2406.04485
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,104 +0,0 @@
-import gradio as gr
-import os
-from serve.gradio_web import *
-from serve.gradio_web_image_editing import *
-from serve.gradio_web_video_generation import *
-from serve.leaderboard import build_leaderboard_tab
-from model.model_manager import ModelManager
-from pathlib import Path
-from serve.constants import SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR
-
-def build_combine_demo(models, elo_results_file, leaderboard_table_file):
-
-    with gr.Blocks(
-        title="Play with Open Vision Models",
-        theme=gr.themes.Default(),
-        css=block_css,
-    ) as demo:
-        with gr.Tabs() as tabs_combine:
-            with gr.Tab("Image Generation", id=0):
-                with gr.Tabs() as tabs_ig:
-                    with gr.Tab("Generation Arena (battle)", id=0):
-                        build_side_by_side_ui_anony(models)
-
-                    with gr.Tab("Generation Arena (side-by-side)", id=1):
-                        build_side_by_side_ui_named(models)
-
-                    with gr.Tab("Generation Playground", id=2): #Direct Chat
-                        build_single_model_ui(models, add_promotion_links=True)
-                    if elo_results_file:
-                        with gr.Tab("Generation Leaderboard", id=3):
-                            build_leaderboard_tab(elo_results_file['t2i_generation'], leaderboard_table_file['t2i_generation'])
-            
-            with gr.Tab("Image Edition", id=5):
-                with gr.Tabs() as tabs_ie:
-                    with gr.Tab("Edition Arena (battle)", id=5):
-                        build_side_by_side_ui_anony_ie(models)
-
-                    with gr.Tab("Edition Arena (side-by-side)", id=6):
-                        build_side_by_side_ui_named_ie(models)
-
-                    with gr.Tab("Edition Playground", id=7): #Direct Chat
-                        build_single_model_ui_ie(models, add_promotion_links=True)
-                    if elo_results_file:
-                        with gr.Tab("Edition Leaderboard", id=8):
-                            build_leaderboard_tab(elo_results_file['image_editing'], leaderboard_table_file['image_editing'])
-
-            with gr.Tab("Video Generation", id=10):
-                with gr.Tabs() as tabs_vg:
-                    with gr.Tab("Video Generation Arena (battle)", id=10):
-                        build_side_by_side_ui_anony_vg(models)
-
-                    with gr.Tab("Video Generation Arena (side-by-side)", id=11):
-                        build_side_by_side_ui_named_vg(models)
-
-                    with gr.Tab("Video Generation Playground", id=12): #Direct Chat
-                        build_single_model_ui_vg(models, add_promotion_links=True)
-                    if elo_results_file and 'video_generation' in elo_results_file:
-                        with gr.Tab("Video Generation Leaderboard", id=13):
-                            build_leaderboard_tab(elo_results_file['video_generation'], leaderboard_table_file['video_generation'])
-            with gr.Tab("About Us", id=4): 
-                build_about()
-
-    return demo
-
-
-def load_elo_results(elo_results_dir):
-    from collections import defaultdict
-    elo_results_file = defaultdict(lambda: None)
-    leaderboard_table_file = defaultdict(lambda: None)
-    if elo_results_dir is not None:
-        elo_results_dir = Path(elo_results_dir)
-        elo_results_file = {}
-        leaderboard_table_file = {}
-        for file in elo_results_dir.glob('elo_results_*.pkl'):
-            if 't2i_generation' in file.name:
-                elo_results_file['t2i_generation'] = file
-            elif 'image_editing' in file.name:
-                elo_results_file['image_editing'] = file
-            elif 'video_generation' in file.name:
-                elo_results_file['video_generation'] = file
-            else:
-                raise ValueError(f"Unknown file name: {file.name}")
-        for file in elo_results_dir.glob('*_leaderboard.csv'):
-            if 't2i_generation' in file.name:
-                leaderboard_table_file['t2i_generation'] = file
-            elif 'image_editing' in file.name:
-                leaderboard_table_file['image_editing'] = file
-            elif 'video_generation' in file.name:
-                leaderboard_table_file['video_generation'] = file
-            else:
-                raise ValueError(f"Unknown file name: {file.name}")
-            
-    return elo_results_file, leaderboard_table_file
-    
-if __name__ == "__main__":
-    server_port = int(SERVER_PORT)
-    root_path = ROOT_PATH
-    elo_results_dir = ELO_RESULTS_DIR
-    models = ModelManager(enable_nsfw=False, do_pre_download=True, do_debug_packages=True)
-    # models = ModelManager(enable_nsfw=False, do_pre_download=False, do_debug_packages=False)
-    
-    elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
-    demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
-    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)

arena_elo/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 WildVision-Bench
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

arena_elo/README.md

@@ -1,46 +0,0 @@
-## Computing the Elo Ratings
-
-
-```bash
-apt-get -y install pkg-config
-pip install -r requirements.txt
-```
-
-
-### to update the leaderboard
-
-```bash
-export LOGDIR="/path/to/your/logdir"
-bash update_elo_rating.sh
-```
-
-### to inspect the leaderboard status
-```bash
-python -m elo_rating.inspect_elo_rating_pkl
-```
-
-### to inspect the collected data status and cost
-```bash
-export LOGDIR="/path/to/your/logdir"
-python -m elo_rating.inspect_cost
-```
-
-### to upload the battle data to hugging face🤗 
-```bash
-export HUGGINGFACE_TOKEN="your_huggingface_token"
-bash get_latest_data.sh
-python -m elo_rating.upload_battle_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
-```
-
-### to upload the chat data to hugging face🤗 
-```bash
-export HUGGINGFACE_TOKEN="your_huggingface_token"
-bash get_latest_data.sh
-python -m elo_rating.upload_chat_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
-```
-
-
-### to get the collected data
-```bash
-python -m 
-

arena_elo/elo_rating/basic_stats.py

@@ -1,227 +0,0 @@
-import argparse
-import code
-import datetime
-import json
-import os
-from pytz import timezone
-import time
-
-import pandas as pd  # pandas>=2.0.3
-import plotly.express as px
-import plotly.graph_objects as go
-from tqdm import tqdm
-
-NUM_SERVERS = 1
-LOG_ROOT_DIR = os.getenv("LOGDIR", None)
-if LOG_ROOT_DIR is None:
-    raise ValueError("LOGDIR environment variable not set, please set it by `export LOGDIR=...`")
-
-def get_log_files(max_num_files=None):
-    log_root = os.path.expanduser(LOG_ROOT_DIR)
-    filenames = []
-    if NUM_SERVERS == 1:
-        for filename in os.listdir(log_root):
-            if filename.endswith("-conv.json"):
-                filepath = f"{log_root}/{filename}"
-                name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
-                filenames.append(name_tstamp_tuple)
-    else:
-        for i in range(NUM_SERVERS):
-            for filename in os.listdir(f"{log_root}/server{i}"):
-                if filename.endswith("-conv.json"):
-                    filepath = f"{log_root}/server{i}/{filename}"
-                    name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
-                    filenames.append(name_tstamp_tuple)
-    # sort by tstamp
-    filenames = sorted(filenames, key=lambda x: x[1])
-    filenames = [x[0] for x in filenames]
-
-    max_num_files = max_num_files or len(filenames)
-    filenames = filenames[-max_num_files:]
-    return filenames
-
-
-def load_log_files(filename):
-    data = []
-    for retry in range(5):
-        try:
-            lines = open(filename).readlines()
-            break
-        except FileNotFoundError:
-            time.sleep(2)
-
-    for l in lines:
-        row = json.loads(l)
-        data.append(
-            dict(
-                type=row["type"],
-                tstamp=row["tstamp"],
-                model=row.get("model", ""),
-                models=row.get("models", ["", ""]),
-            )
-        )
-    return data
-
-
-def load_log_files_parallel(log_files, num_threads=16):
-    data_all = []
-    from multiprocessing import Pool
-
-    with Pool(num_threads) as p:
-        ret_all = list(tqdm(p.imap(load_log_files, log_files), total=len(log_files)))
-        for ret in ret_all:
-            data_all.extend(ret)
-    return data_all
-
-
-def get_anony_vote_df(df):
-    anony_vote_df = df[
-        df["type"].isin(["leftvote", "rightvote", "tievote", "bothbad_vote"])
-    ]
-    anony_vote_df = anony_vote_df[anony_vote_df["models"].apply(lambda x: x[0] == "")]
-    return anony_vote_df
-
-
-def merge_counts(series, on, names):
-    ret = pd.merge(series[0], series[1], on=on)
-    for i in range(2, len(series)):
-        ret = pd.merge(ret, series[i], on=on)
-    ret = ret.reset_index()
-    old_names = list(ret.columns)[-len(series) :]
-    rename = {old_name: new_name for old_name, new_name in zip(old_names, names)}
-    ret = ret.rename(columns=rename)
-    return ret
-
-
-def report_basic_stats(log_files):
-    df_all = load_log_files_parallel(log_files)
-    df_all = pd.DataFrame(df_all)
-    now_t = df_all["tstamp"].max()
-    df_1_hour = df_all[df_all["tstamp"] > (now_t - 3600)]
-    df_1_day = df_all[df_all["tstamp"] > (now_t - 3600 * 24)]
-    anony_vote_df_all = get_anony_vote_df(df_all)
-
-    # Chat trends
-    chat_dates = [
-        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
-            "%Y-%m-%d"
-        )
-        for x in df_all[df_all["type"] == "chat"]["tstamp"]
-    ]
-    chat_dates_counts = pd.value_counts(chat_dates)
-    vote_dates = [
-        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
-            "%Y-%m-%d"
-        )
-        for x in anony_vote_df_all["tstamp"]
-    ]
-    vote_dates_counts = pd.value_counts(vote_dates)
-    chat_dates_bar = go.Figure(
-        data=[
-            go.Bar(
-                name="Anony. Vote",
-                x=vote_dates_counts.index,
-                y=vote_dates_counts,
-                text=[f"{val:.0f}" for val in vote_dates_counts],
-                textposition="auto",
-            ),
-            go.Bar(
-                name="Chat",
-                x=chat_dates_counts.index,
-                y=chat_dates_counts,
-                text=[f"{val:.0f}" for val in chat_dates_counts],
-                textposition="auto",
-            ),
-        ]
-    )
-    chat_dates_bar.update_layout(
-        barmode="stack",
-        xaxis_title="Dates",
-        yaxis_title="Count",
-        height=300,
-        width=1200,
-    )
-
-    # Model call counts
-    model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
-    model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()
-    model_hist_1_hour = df_1_hour[df_1_hour["type"] == "chat"]["model"].value_counts()
-    model_hist = merge_counts(
-        [model_hist_all, model_hist_1_day, model_hist_1_hour],
-        on="model",
-        names=["All", "Last Day", "Last Hour"],
-    )
-    model_hist_md = model_hist.to_markdown(index=False, tablefmt="github")
-
-    # Action counts
-    action_hist_all = df_all["type"].value_counts()
-    action_hist_1_day = df_1_day["type"].value_counts()
-    action_hist_1_hour = df_1_hour["type"].value_counts()
-    action_hist = merge_counts(
-        [action_hist_all, action_hist_1_day, action_hist_1_hour],
-        on="type",
-        names=["All", "Last Day", "Last Hour"],
-    )
-    action_hist_md = action_hist.to_markdown(index=False, tablefmt="github")
-
-    # Anony vote counts
-    anony_vote_hist_all = anony_vote_df_all["type"].value_counts()
-    anony_vote_df_1_day = get_anony_vote_df(df_1_day)
-    anony_vote_hist_1_day = anony_vote_df_1_day["type"].value_counts()
-    # anony_vote_df_1_hour = get_anony_vote_df(df_1_hour)
-    # anony_vote_hist_1_hour = anony_vote_df_1_hour["type"].value_counts()
-    anony_vote_hist = merge_counts(
-        [anony_vote_hist_all, anony_vote_hist_1_day],
-        on="type",
-        names=["All", "Last Day"],
-    )
-    anony_vote_hist_md = anony_vote_hist.to_markdown(index=False, tablefmt="github")
-
-    # Last 24 hours
-    chat_1_day = df_1_day[df_1_day["type"] == "chat"]
-    num_chats_last_24_hours = []
-    base = df_1_day["tstamp"].min()
-    for i in range(24, 0, -1):
-        left = base + (i - 1) * 3600
-        right = base + i * 3600
-        num = ((chat_1_day["tstamp"] >= left) & (chat_1_day["tstamp"] < right)).sum()
-        num_chats_last_24_hours.append(num)
-    times = [
-        datetime.datetime.fromtimestamp(
-            base + i * 3600, tz=timezone("US/Pacific")
-        ).strftime("%Y-%m-%d %H:%M:%S %Z")
-        for i in range(24, 0, -1)
-    ]
-    last_24_hours_df = pd.DataFrame({"time": times, "value": num_chats_last_24_hours})
-    last_24_hours_md = last_24_hours_df.to_markdown(index=False, tablefmt="github")
-
-    # Last update datetime
-    last_updated_tstamp = now_t
-    last_updated_datetime = datetime.datetime.fromtimestamp(
-        last_updated_tstamp, tz=timezone("US/Pacific")
-    ).strftime("%Y-%m-%d %H:%M:%S %Z")
-
-    # code.interact(local=locals())
-
-    return {
-        "chat_dates_bar": chat_dates_bar,
-        "model_hist_md": model_hist_md,
-        "action_hist_md": action_hist_md,
-        "anony_vote_hist_md": anony_vote_hist_md,
-        "num_chats_last_24_hours": last_24_hours_md,
-        "last_updated_datetime": last_updated_datetime,
-    }
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--max-num-files", type=int)
-    args = parser.parse_args()
-
-    log_files = get_log_files(args.max_num_files)
-    basic_stats = report_basic_stats(log_files)
-
-    print(basic_stats["action_hist_md"] + "\n")
-    print(basic_stats["model_hist_md"] + "\n")
-    print(basic_stats["anony_vote_hist_md"] + "\n")
-    print(basic_stats["num_chats_last_24_hours"] + "\n")

arena_elo/elo_rating/clean_battle_data.py

@@ -1,378 +0,0 @@
-"""
-Clean chatbot arena battle log.
-
-Usage:
-python3 clean_battle_data.py --mode conv_release
-"""
-import argparse
-import datetime
-import json
-import os
-import sys
-from pytz import timezone
-import time
-import PIL
-from PIL import ImageFile
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-from tqdm import tqdm
-
-from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
-from .utils import detect_language, get_time_stamp_from_date, get_model_info
-
-VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
-
-def parse_model_name(model_name):
-    return NotImplementedError()
-    return model_source, model_name, model_type
-
-def remove_html(raw):
-    if raw.startswith("<h3>"):
-        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
-    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
-        return raw[13:]
-    return raw
-
-
-def to_openai_format(messages):
-    roles = ["user", "assistant"]
-    ret = []
-    for i, x in enumerate(messages):
-        ret.append({"role": roles[i % 2], "content": x[1]})
-    return ret
-
-
-def replace_model_name(old_name, tstamp):
-    replace_dict = {
-        "PlayGroundV2": "PlayGround V2",
-        "PlayGroundV2.5": "PlayGround V2.5",
-        "FluxTimestep": "FLUX1schnell",
-        "FluxGuidance": "FLUX1dev",
-        "CogVideoX": "CogVideoX-2B"
-    }
-    if old_name in replace_dict:
-        old_name =  replace_dict[old_name]
-    if "Flux" in old_name:
-        print(f"Invalid model names: {old_name}")
-        exit(1)
-    model_info = get_model_info(old_name)
-    old_name = model_info.simple_name
-    return old_name
-
-
-def read_file(filename):
-    data = []
-    for retry in range(5):
-        try:
-            # lines = open(filename).readlines()
-            for l in open(filename):
-                row = json.loads(l)
-                if row["type"] in VOTES:
-                    data.append(row)
-            break
-        except FileNotFoundError:
-            time.sleep(2)
-        except json.JSONDecodeError:
-            print(f"Error in reading {filename}")
-            print(row)
-            exit(0)
-    return data
-
-
-def read_file_parallel(log_files, num_threads=16):
-    data_all = []
-    if num_threads == 1:
-        for log_file in tqdm(log_files, desc="Reading"):
-            data_all.extend(read_file(log_file))
-        return data_all
-    else:
-        from multiprocessing import Pool
-
-        with Pool(num_threads) as p:
-            ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
-            for ret in ret_all:
-                data_all.extend(ret)
-        return data_all
-
-def load_image(image_path):
-    try:
-        return PIL.Image.open(image_path)
-    except:
-        return None
-
-def clean_battle_data(
-    log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple", task_name="image_editing"
-):
-    data = read_file_parallel(log_files, num_threads=1)
-
-    convert_type = {
-        "leftvote": "model_a",
-        "rightvote": "model_b",
-        "tievote": "tie",
-        "bothbad_vote": "tie (bothbad)",
-    }
-    
-    all_models = set()
-    all_ips = dict()
-    ct_anony = 0
-    ct_invalid = 0
-    ct_leaked_identity = 0
-    ct_banned = 0
-    battles = []
-    for row in tqdm(data, desc="Cleaning"):
-        if row["models"][0] is None or row["models"][1] is None:
-            print(f"Invalid model names: {row['models']}")
-            continue
-
-        # Resolve model names
-        models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
-        if "model_name" in row["states"][0]:
-            models_hidden = [
-                row["states"][0]["model_name"],
-                row["states"][1]["model_name"],
-            ]
-            if models_hidden[0] is None:
-                models_hidden = models_public
-        else:
-            models_hidden = models_public
-
-        if (models_public[0] == "" and models_public[1] != "") or (
-            models_public[1] == "" and models_public[0] != ""
-        ):
-            ct_invalid += 1
-            print(f"Invalid model names: {models_public}")
-            continue
-
-        if models_public[0] == "" or models_public[0] == "Model A":
-            anony = True
-            models = models_hidden
-            ct_anony += 1
-        else:
-            anony = False
-            models = models_public
-            if not models_public == models_hidden:
-                print(f"Model names mismatch: {models_public} vs {models_hidden}")
-                ct_invalid += 1
-                continue
-        
-        def preprocess_model_name(m):
-            if m == "Playground v2":
-                return 'playground_PlayGroundV2_generation'
-            if m == "Playground v2.5":
-                return 'playground_PlayGroundV2.5_generation'
-            return m
-        models = [preprocess_model_name(m) for m in models]
-
-        # Replace bard with palm
-        if task_name == "image_editing":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform in ["playground", "imagenhub"] and task == "edition"):
-                    valid = False
-                    break
-            if not valid:
-                ct_invalid += 1
-                continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
-
-        elif task_name == "t2i_generation":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform.lower() in ["playground", "imagenhub", 'fal'] and (task == "generation" or task == "text2image")):
-                    valid = False
-                    break
-            if not valid:
-                ct_invalid += 1
-                continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
-            
-        elif task_name == "video_generation":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform in ["videogenhub", "fal"] and task == "generation" or task == "text2video"):
-                    valid = False
-                    break
-            if not valid:
-                ct_invalid += 1
-                continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
-
-        else:
-            raise ValueError(f"Invalid task_name: {task_name}")
-        
-        models = [replace_model_name(m, row["tstamp"]) for m in models]
-        
-        # Exclude certain models
-        if exclude_model_names and any(x in exclude_model_names for x in models):
-            ct_invalid += 1
-            continue
-
-        if mode == "conv_release":
-            # assert the two images are the same
-            date = datetime.datetime.fromtimestamp(row["tstamp"], tz=timezone("US/Pacific")).strftime("%Y-%m-%d") # 2024-02-29
-            image_path_format = f"{LOG_ROOT_DIR}/{date}-convinput_images/input_image_"
-            image_path_0 = image_path_format + str(row["states"][0]["conv_id"]) + ".png"
-            image_path_1 = image_path_format + str(row["states"][1]["conv_id"]) + ".png"
-            if not os.path.exists(image_path_0) or not os.path.exists(image_path_1):
-                print(f"Image not found for {image_path_0} or {image_path_1}")
-                ct_invalid += 1
-                continue
-            
-            image_0 = load_image(image_path_0)
-            image_1 = load_image(image_path_1)
-            if image_0 is None or image_1 is None:
-                print(f"Image not found for {image_path_0} or {image_path_1}")
-                ct_invalid += 1
-                continue
-            if image_0.tobytes() != image_1.tobytes():
-                print(f"Image not the same for {image_path_0} and {image_path_1}")
-                ct_invalid += 1
-                continue
-        
-        
-        ip = row["ip"]
-        if ip not in all_ips:
-            all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
-        all_ips[ip]["count"] += 1
-        if sanitize_ip:
-            user_id = f"arena_user_{all_ips[ip]['sanitized_id']}"
-        else:
-            user_id = f"{all_ips[ip]['ip']}"
-
-        if ban_ip_list is not None and ip in ban_ip_list:
-            ct_banned += 1
-            print(f"User {user_id} is banned")
-            continue
-        required_keys_each_task = {
-            "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
-            "t2i_generation": ["prompt"],
-            "video_generation": ["prompt"]
-        }
-        
-        model_a_inputs = row["states"][0].copy()
-        # pop conv_id and model_name
-        model_a_inputs.pop("conv_id")
-        model_a_inputs.pop("model_name")
-        model_b_inputs = row["states"][1].copy()
-        model_b_inputs.pop("conv_id")
-        model_b_inputs.pop("model_name")
-        for key in model_a_inputs:
-            if not (key in model_b_inputs and model_a_inputs[key] == model_b_inputs[key]):
-                print(f"Inconsistent inputs: {model_a_inputs} vs {model_b_inputs}")
-                ct_invalid += 1
-                continue
-        if mode == "conv_release":
-            if any(key not in model_a_inputs for key in required_keys_each_task[task_name]):
-                print(f"Missing required keys: {model_a_inputs}, {required_keys_each_task[task_name]}")
-                ct_invalid += 1
-                continue
-            
-        inputs = model_a_inputs
-        # Save the results
-        battles.append(
-            dict(
-                model_a_conv_id=row["states"][0]["conv_id"],
-                model_b_conv_id=row["states"][1]["conv_id"],
-                inputs=inputs,
-                model_a=models[0],
-                model_b=models[1],
-                vote_type=row["type"],
-                winner=convert_type[row["type"]],
-                judge=f"arena_user_{user_id}",
-                anony=anony,
-                tstamp=row["tstamp"],
-            )
-        )
-
-        all_models.update(models_hidden)
-    battles.sort(key=lambda x: x["tstamp"])
-    last_updated_tstamp = battles[-1]["tstamp"]
-
-    last_updated_datetime = datetime.datetime.fromtimestamp(
-        last_updated_tstamp, tz=timezone("US/Pacific")
-    ).strftime("%Y-%m-%d %H:%M:%S %Z")
-
-    print(
-        f"#votes: {len(data)}, #invalid votes: {ct_invalid}, "
-        f"#leaked_identity: {ct_leaked_identity} "
-        f"#banned: {ct_banned} "
-    )
-    print(f"#battles: {len(battles)}, #anony: {ct_anony}")
-    print(f"#models: {len(all_models)}, {all_models}")
-    print(f"last-updated: {last_updated_datetime}")
-
-    if ban_ip_list is not None:
-        for ban_ip in ban_ip_list:
-            if ban_ip in all_ips:
-                del all_ips[ban_ip]
-    print("Top 30 IPs:")
-    print(sorted(all_ips.values(), key=lambda x: x["count"], reverse=True)[:30])
-    return battles
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--max-num-files", type=int)
-    parser.add_argument(
-        "--mode", type=str, choices=["simple", "conv_release"], default="simple"
-    )
-    parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation", "video_generation"])
-    parser.add_argument("--exclude-model-names", type=str, nargs="+")
-    parser.add_argument("--ban-ip-file", type=str)
-    parser.add_argument("--sanitize-ip", action="store_true", default=False)
-    args = parser.parse_args()
-
-    log_files = get_log_files(args.max_num_files)
-    ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None
-
-    battles = clean_battle_data(
-        log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, args.mode, args.task_name
-    )
-    last_updated_tstamp = battles[-1]["tstamp"]
-    cutoff_date = datetime.datetime.fromtimestamp(
-        last_updated_tstamp, tz=timezone("US/Pacific")
-    ).strftime("%Y%m%d")
-
-    if args.mode == "simple":
-        # for x in battles:
-        #     for key in [
-        #         "conversation_a",
-        #         "conversation_b",
-        #         "question_id",
-        #     ]:
-        #         if key in x:
-        #             del x[key]
-        print("Samples:")
-        for i in range(min(4, len(battles))):
-            print(battles[i])
-        output = f"clean_battle_{args.task_name}_{cutoff_date}.json"
-    elif args.mode == "conv_release":
-        output = f"clean_battle_{args.task_name}_conv_{cutoff_date}.json"
-
-    with open(output, "w") as fout:
-        json.dump(battles, fout, indent=2, ensure_ascii=False)
-    print(f"Write cleaned data to {output}")
-    
-    with open("cut_off_date.txt", "w") as fout:
-        fout.write(cutoff_date)

arena_elo/elo_rating/elo_analysis.py

@@ -1,413 +0,0 @@
-import argparse
-from collections import defaultdict
-import datetime
-import json
-import math
-import pickle
-from pytz import timezone
-
-import numpy as np
-import pandas as pd
-import plotly.express as px
-from tqdm import tqdm
-
-from .basic_stats import get_log_files
-from .clean_battle_data import clean_battle_data
-from .utils import get_model_info
-        
-pd.options.display.float_format = "{:.2f}".format
-
-
-def compute_elo(battles, K=4, SCALE=400, BASE=10, INIT_RATING=1000):
-    rating = defaultdict(lambda: INIT_RATING)
-
-    for rd, model_a, model_b, winner in battles[
-        ["model_a", "model_b", "winner"]
-    ].itertuples():
-        ra = rating[model_a]
-        rb = rating[model_b]
-        ea = 1 / (1 + BASE ** ((rb - ra) / SCALE))
-        eb = 1 / (1 + BASE ** ((ra - rb) / SCALE))
-        if winner == "model_a":
-            sa = 1
-        elif winner == "model_b":
-            sa = 0
-        elif winner == "tie" or winner == "tie (bothbad)":
-            sa = 0.5
-        else:
-            raise Exception(f"unexpected vote {winner}")
-        rating[model_a] += K * (sa - ea)
-        rating[model_b] += K * (1 - sa - eb)
-
-    return dict(rating)
-
-
-def get_bootstrap_result(battles, func_compute_elo, num_round=1000):
-    rows = []
-    for i in tqdm(range(num_round), desc="bootstrap"):
-        tmp_battles = battles.sample(frac=1.0, replace=True)
-        rows.append(func_compute_elo(tmp_battles))
-    df = pd.DataFrame(rows)
-    return df[df.median().sort_values(ascending=False).index]
-
-
-def compute_elo_mle_with_tie(df, SCALE=400, BASE=10, INIT_RATING=1000):
-    from sklearn.linear_model import LogisticRegression
-
-    models = pd.concat([df["model_a"], df["model_b"]]).unique()
-    models = pd.Series(np.arange(len(models)), index=models)
-
-    # duplicate battles
-    df = pd.concat([df, df], ignore_index=True)
-    p = len(models.index)
-    n = df.shape[0]
-
-    X = np.zeros([n, p])
-    X[np.arange(n), models[df["model_a"]]] = +math.log(BASE)
-    X[np.arange(n), models[df["model_b"]]] = -math.log(BASE)
-
-    # one A win => two A win
-    Y = np.zeros(n)
-    Y[df["winner"] == "model_a"] = 1.0
-
-    # one tie => one A win + one B win
-    # find tie + tie (both bad) index
-    tie_idx = (df["winner"] == "tie") | (df["winner"] == "tie (bothbad)")
-    tie_idx[len(tie_idx) // 2 :] = False
-    Y[tie_idx] = 1.0
-
-    lr = LogisticRegression(fit_intercept=False)
-    lr.fit(X, Y)
-
-    elo_scores = SCALE * lr.coef_[0] + INIT_RATING
-    # calibrate llama-13b to 800 if applicable
-    if "llama-13b" in models.index:
-        elo_scores += 800 - elo_scores[models["llama-13b"]]
-    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
-
-
-def get_median_elo_from_bootstrap(bootstrap_df):
-    median = dict(bootstrap_df.quantile(0.5))
-    median = {k: int(v + 0.5) for k, v in median.items()}
-    return median
-
-
-def compute_pairwise_win_fraction(battles, model_order, limit_show_number=None):
-    # Times each model wins as Model A
-    a_win_ptbl = pd.pivot_table(
-        battles[battles["winner"] == "model_a"],
-        index="model_a",
-        columns="model_b",
-        aggfunc="size",
-        fill_value=0,
-    )
-
-    # Table counting times each model wins as Model B
-    b_win_ptbl = pd.pivot_table(
-        battles[battles["winner"] == "model_b"],
-        index="model_a",
-        columns="model_b",
-        aggfunc="size",
-        fill_value=0,
-    )
-
-    # Table counting number of A-B pairs
-    num_battles_ptbl = pd.pivot_table(
-        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
-    )
-
-    # Computing the proportion of wins for each model as A and as B
-    # against all other models
-    row_beats_col_freq = (a_win_ptbl + b_win_ptbl.T) / (
-        num_battles_ptbl + num_battles_ptbl.T
-    )
-
-    if model_order is None:
-        prop_wins = row_beats_col_freq.mean(axis=1).sort_values(ascending=False)
-        model_order = list(prop_wins.keys())
-
-    if limit_show_number is not None:
-        model_order = model_order[:limit_show_number]
-
-    # Arrange ordering according to proprition of wins
-    row_beats_col = row_beats_col_freq.loc[model_order, model_order]
-    return row_beats_col
-
-
-def visualize_leaderboard_table(rating):
-    models = list(rating.keys())
-    models.sort(key=lambda k: -rating[k])
-
-    emoji_dict = {
-        1: "🥇",
-        2: "🥈",
-        3: "🥉",
-    }
-
-    md = ""
-    md += "| Rank | Model | Elo Rating | Description |\n"
-    md += "| --- | --- | --- | --- |\n"
-    for i, model in enumerate(models):
-        rank = i + 1
-        minfo = get_model_info(model)
-        emoji = emoji_dict.get(rank, "")
-        md += f"| {rank} | {emoji} [{model}]({minfo.link}) | {rating[model]:.0f} | {minfo.description} |\n"
-
-    return md
-
-
-def visualize_pairwise_win_fraction(battles, model_order):
-    row_beats_col = compute_pairwise_win_fraction(battles, model_order)
-    fig = px.imshow(
-        row_beats_col,
-        color_continuous_scale="RdBu",
-        text_auto=".2f",
-        height=700,
-        width=700,
-    )
-    fig.update_layout(
-        xaxis_title="Model B",
-        yaxis_title="Model A",
-        xaxis_side="top",
-        title_y=0.07,
-        title_x=0.5,
-    )
-    fig.update_traces(
-        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
-    )
-
-    return fig
-
-
-def visualize_battle_count(battles, model_order):
-    ptbl = pd.pivot_table(
-        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
-    )
-    battle_counts = ptbl + ptbl.T
-    fig = px.imshow(
-        battle_counts.loc[model_order, model_order],
-        text_auto=True,
-        height=700,
-        width=700,
-    )
-    fig.update_layout(
-        xaxis_title="Model B",
-        yaxis_title="Model A",
-        xaxis_side="top",
-        title_y=0.07,
-        title_x=0.5,
-    )
-    fig.update_traces(
-        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
-    )
-    return fig
-
-
-def visualize_average_win_rate(battles, limit_show_number):
-    row_beats_col_freq = compute_pairwise_win_fraction(
-        battles, None, limit_show_number=limit_show_number
-    )
-    fig = px.bar(
-        row_beats_col_freq.mean(axis=1).sort_values(ascending=False),
-        text_auto=".2f",
-        height=500,
-        width=700,
-    )
-    fig.update_layout(
-        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False,
-    )
-    fig.update_traces(textfont_size=16)
-    return fig
-
-
-def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
-    bars = (
-        pd.DataFrame(
-            dict(
-                lower=df.quantile(0.025),
-                rating=df_final,
-                upper=df.quantile(0.975),
-            )
-        )
-        .reset_index(names="model")
-        .sort_values("rating", ascending=False)
-    )
-    bars = bars[:limit_show_number]
-    bars["error_y"] = bars["upper"] - bars["rating"]
-    bars["error_y_minus"] = bars["rating"] - bars["lower"]
-    bars["rating_rounded"] = np.round(bars["rating"], 2)
-    fig = px.scatter(
-        bars,
-        x="model",
-        y="rating",
-        error_y="error_y",
-        error_y_minus="error_y_minus",
-        text="rating_rounded",
-        height=500,
-        width=700,
-    )
-    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
-    fig.update_traces(textfont_size=16)
-    return fig
-
-
-def report_elo_analysis_results(battles_json, rating_system="bt", num_bootstrap=100, anony_only=True):
-    battles = pd.DataFrame(battles_json)
-    battles = battles.sort_values(ascending=True, by=["tstamp"])
-    # Only use anonymous votes
-    if anony_only:
-        battles = battles[battles["anony"]].reset_index(drop=True)
-    battles_no_ties = battles[~battles["winner"].str.contains("tie")]
-
-    # Online update
-    elo_rating_online = compute_elo(battles)
-
-    if rating_system == "bt":
-        bootstrap_df = get_bootstrap_result(
-            battles, compute_elo_mle_with_tie, num_round=num_bootstrap
-        )
-        elo_rating_final = compute_elo_mle_with_tie(battles)
-    elif rating_system == "elo":
-        bootstrap_df = get_bootstrap_result(
-            battles, compute_elo, num_round=num_bootstrap
-        )
-        elo_rating_median = get_median_elo_from_bootstrap(bootstrap_df)
-        elo_rating_final = elo_rating_median
-
-    model_order = list(elo_rating_final.keys())
-    model_order.sort(key=lambda k: -elo_rating_final[k])
-
-    limit_show_number = 25  # limit show number to make plots smaller
-    model_order = model_order[:limit_show_number]
-
-    # leaderboard_table_df: elo rating, variance, 95% interval, number of battles
-    leaderboard_table_df = pd.DataFrame(
-        {
-            "rating": elo_rating_final,
-            "variance": bootstrap_df.var(),
-            "rating_q975": bootstrap_df.quantile(0.975),
-            "rating_q025": bootstrap_df.quantile(0.025),
-            "num_battles": battles["model_a"].value_counts()
-            + battles["model_b"].value_counts(),
-        }
-    )
-
-    # Plots
-    leaderboard_table = visualize_leaderboard_table(elo_rating_final)
-    win_fraction_heatmap = visualize_pairwise_win_fraction(battles_no_ties, model_order)
-    battle_count_heatmap = visualize_battle_count(battles_no_ties, model_order)
-    average_win_rate_bar = visualize_average_win_rate(
-        battles_no_ties, limit_show_number
-    )
-    bootstrap_elo_rating = visualize_bootstrap_elo_rating(
-        bootstrap_df, elo_rating_final, limit_show_number
-    )
-
-    last_updated_tstamp = battles["tstamp"].max()
-    last_updated_datetime = datetime.datetime.fromtimestamp(
-        last_updated_tstamp, tz=timezone("US/Pacific")
-    ).strftime("%Y-%m-%d %H:%M:%S %Z")
-
-    return {
-        "rating_system": rating_system,
-        "elo_rating_online": elo_rating_online,
-        "elo_rating_final": elo_rating_final,
-        "leaderboard_table": leaderboard_table,
-        "win_fraction_heatmap": win_fraction_heatmap,
-        "battle_count_heatmap": battle_count_heatmap,
-        "average_win_rate_bar": average_win_rate_bar,
-        "bootstrap_elo_rating": bootstrap_elo_rating,
-        "last_updated_datetime": last_updated_datetime,
-        "last_updated_tstamp": last_updated_tstamp,
-        "bootstrap_df": bootstrap_df,
-        "leaderboard_table_df": leaderboard_table_df,
-    }
-
-
-def pretty_print_elo_rating(rating):
-    model_order = list(rating.keys())
-    model_order.sort(key=lambda k: -rating[k])
-    for i, model in enumerate(model_order):
-        print(f"{i+1:2d}, {model:25s}, {rating[model]:.0f}")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--clean-battle-file", type=str)
-    parser.add_argument("--max-num-files", type=int)
-    parser.add_argument("--num-bootstrap", type=int, default=100)
-    parser.add_argument(
-        "--rating-system", type=str, choices=["bt", "elo"], default="bt"
-    )
-    parser.add_argument("--exclude-tie", action="store_true", default=False)
-    parser.add_argument("--min_num_battles_per_model", type=int, default=25)
-    args = parser.parse_args()
-
-    np.random.seed(42)
-
-    if args.clean_battle_file:
-        # Read data from a cleaned battle files
-        battles = pd.read_json(args.clean_battle_file)
-    else:
-        # Read data from all log files
-        log_files = get_log_files(args.max_num_files)
-        battles = clean_battle_data(log_files)
-        
-    if args.min_num_battles_per_model:
-        num_battles_per_model = defaultdict(int)
-        # use pd
-        for _, battle in battles.iterrows():
-            num_battles_per_model[battle["model_a"]] += 1
-            num_battles_per_model[battle["model_b"]] += 1
-        to_remove_models = [
-            model for model, num_battles in num_battles_per_model.items() if num_battles < args.min_num_battles_per_model
-        ]
-        battles_with_enough_battles = battles[
-            ~battles["model_a"].isin(to_remove_models) & ~battles["model_b"].isin(to_remove_models)
-        ]
-        print(f"Remove models with less than {args.min_num_battles_per_model} battles: {to_remove_models}")
-        print(f"Number of battles: {len(battles)} -> {len(battles_with_enough_battles)}")
-        battles = battles_with_enough_battles
-        
-    anony_results = report_elo_analysis_results(
-        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=True
-    )
-    full_results = report_elo_analysis_results(
-        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=False
-    )
-    
-
-    print("# Online Elo")
-    pretty_print_elo_rating(anony_results["elo_rating_online"])
-    print("# Median")
-    pretty_print_elo_rating(anony_results["elo_rating_final"])
-    print(f"Annoy last update : {anony_results['last_updated_datetime']}")
-    print(f"Full last update : {full_results['last_updated_datetime']}")
-    
-    
-    # # save heatmap results in the same directory of the cleaned battle file
-    win_fraction_heatmap_file = args.clean_battle_file.replace(".json", "_win_fraction_heatmap.jpg")
-    battle_count_heatmap_file = args.clean_battle_file.replace(".json", "_battle_count_heatmap.jpg")
-    average_win_rate_bar_file = args.clean_battle_file.replace(".json", "_average_win_rate_bar.jpg")
-    bootstrap_elo_rating_file = args.clean_battle_file.replace(".json", "_bootstrap_elo_rating.jpg")
-    anony_results["win_fraction_heatmap"].write_image(win_fraction_heatmap_file)
-    anony_results["battle_count_heatmap"].write_image(battle_count_heatmap_file)
-    anony_results["average_win_rate_bar"].write_image(average_win_rate_bar_file)
-    anony_results["bootstrap_elo_rating"].write_image(bootstrap_elo_rating_file)
-    
-
-    last_updated_tstamp = full_results["last_updated_tstamp"]
-    cutoff_date = datetime.datetime.fromtimestamp(
-        last_updated_tstamp, tz=timezone("US/Pacific")
-    ).strftime("%Y%m%d")
-
-
-    results = {
-        "anony": anony_results,
-        "full": full_results,
-    }
-    with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
-        pickle.dump(results, fout)
-
-    with open("cut_off_date.txt", "w") as fout:
-        fout.write(cutoff_date)

arena_elo/elo_rating/generate_leaderboard.py

@@ -1,68 +0,0 @@
-import fire
-import json
-import pandas as pd
-import pickle
-from .utils import get_model_info
-
-def main(
-    elo_rating_pkl: str,
-    output_csv: str
-):
-    with open(elo_rating_pkl, "rb") as fin:
-        elo_rating_results = pickle.load(fin)
-
-    anony_elo_rating_results = elo_rating_results["anony"]
-    full_elo_rating_results = elo_rating_results["full"]
-    anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
-    full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
-
-    print(anony_leaderboard_data)
-    # Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
-    fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
-    # set Organization and license to empty for now
-    all_models = anony_leaderboard_data.index.tolist()
-
-    model_info = {}
-    for model in all_models:
-    
-        registered_model_info = get_model_info(model)
-        model_info[model] = {
-            "key": model,
-            "Model": model,
-            "License": registered_model_info.license,
-            "Organization": registered_model_info.organization,
-            "Link": registered_model_info.link
-        }
-
-        if model in anony_leaderboard_data.index:
-            model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
-        else:
-            model_info[model]["Arena Elo rating (anony)"] = 0
-
-        if model in full_elo_rating_results["leaderboard_table_df"].index:
-            model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
-        else:
-            model_info[model]["Arena Elo rating (full)"] = 0
-
-    final_model_info = {}
-    for model in model_info:
-        if "Model" in model_info[model]:
-            final_model_info[model] = model_info[model]
-    model_info = final_model_info
-
-    exclude_keys = ['starting_from']
-    for key in exclude_keys:
-        for model in model_info:
-            if key in model_info[model]:
-                del model_info[model][key]
-    df = pd.DataFrame(model_info).T
-    df = df[fields]
-    # sort by anony rating
-    df = df.sort_values(by=["Arena Elo rating (anony)"], ascending=False)
-    df.to_csv(output_csv, index=False)
-    print("Leaderboard data saved to", output_csv)
-    print(df)
-
-
-if __name__ == "__main__":
-    fire.Fire(main)

arena_elo/elo_rating/inspect_conv_rating.py

@@ -1,234 +0,0 @@
-import argparse
-import code
-import datetime
-import json
-import os
-from pytz import timezone
-import time
-
-import pandas as pd
-from tqdm import tqdm
-import csv
-
-import base64
-from icecream import ic
-from openai import OpenAI
-
-# Function to encode the image
-def encode_image(image_path):
-    with open(image_path, "rb") as image_file:
-        return base64.b64encode(image_file.read()).decode('utf-8')
-
-def get_log_files(max_num_files=None):
-    dates = []
-    for month in [2, 3]:
-        for day in range(1, 32):
-            dates.append(f"2024-{month:02d}-{day:02d}")
-
-    num_servers = 1
-    filenames = []
-    for d in dates:
-        for i in range(num_servers):
-            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
-            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
-            if os.path.exists(name):
-                filenames.append(name)
-    max_num_files = max_num_files or len(filenames)
-    filenames = filenames[-max_num_files:]
-    return filenames
-
-
-def pretty_print_conversation(messages):
-    for role, msg in messages:
-        print(f"[[{role}]]: {msg}")
-
-
-def get_gpt4v_response(client, img_bs64=None, text_prompt="", use_vision=False):
-    if use_vision:
-        response = client.chat.completions.create(
-        model="gpt-4-vision-preview",
-        messages=[
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": text_prompt},
-                    {
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{img_bs64}"
-                        }
-                    },
-                ],
-            }
-        ],
-        max_tokens=100,
-        )
-    else:
-        response = client.chat.completions.create(
-        model="gpt-4-vision-preview",
-        messages=[
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": text_prompt},
-                ],
-            }
-        ],
-        max_tokens=100,
-        )
-    return response.choices[0].message.content
-
-task_template_map = {
-    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
-    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
-    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
-    "pair_rate_wexplanation": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
-    "pair_rate": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
-}
-
-def inspect_convs(log_files):
-    ic(log_files)
-    data = []
-    total_vote = 0
-    correct_vote = 0
-        
-    client = OpenAI()
-    with open('all_pairvote_log_wgpt_prtchatbot.csv', 'w', newline='') as csvfile:
-        # fieldnames = ['tstamp', 'type', 'model_1', 'model_2', 'template_name_1', 'template_name_2', 'system_message_1', 'system_message_2', 'role_1', 'role_2', 'instruction_1', 'instruction_2', 'message_1', 'message_2', 'offset_1', 'offset_2', 'conv_id_1', 'conv_id_2', 'model_name_1', 'model_name_2', 'ip']
-        fieldnames = ['tstamp', 'type', 'models', 'states', 'ip', 'gpt_vote']
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-        
-        # Write the header
-        writer.writeheader()
-
-        for filename in tqdm(log_files, desc="read files"):
-            for retry in range(5):
-                try:
-                    lines = open(filename).readlines()
-                    break
-                except FileNotFoundError:
-                    time.sleep(2)
-            
-            for l in lines:
-                row = json.loads(l)
-                
-                if "states" not in row:
-                    continue
-                if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
-                    continue
-                
-                model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
-                
-
-                # Iterate through each state and write the relevant information
-                if not len(row["states"][0]['messages']): continue
-                # ic(row["states"][0]['messages'][1][1])
-
-                if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
-                total_vote += 1
-                # row = {
-                #     'tstamp': row['tstamp'],
-                #     'type': row['type'],
-                #     'model_1': row['models'][0],
-                #     'model_2': row['models'][1],
-                #     'template_name_1': row["states"][0]['template_name'],
-                #     'system_message_1': row["states"][0]['system_message'],
-                #     'template_name_2': row["states"][1]['template_name'],
-                #     'system_message_2': row["states"][1]['system_message'],
-                #     'role_1': row["states"][0]['roles'],
-                #     'role_2': row["states"][1]['roles'],
-                #     'instruction_1': row["states"][0]['messages'][0][1],
-                #     'instruction_2': row["states"][1]['messages'][0][1],
-                #     'message_1': row["states"][0]['messages'][1][1],
-                #     'message_2': row["states"][1]['messages'][1][1],
-                #     'offset_1': row["states"][0]['offset'],
-                #     'offset_2': row["states"][1]['offset'],
-                #     'conv_id_1': row["states"][0]['conv_id'],
-                #     'conv_id_2': row["states"][1]['conv_id'],
-                #     'model_name_1': row["states"][0]['model_name'],
-                #     'model_name_2': row["states"][1]['model_name'],
-                #     'ip': row['ip']
-                # }
-                # writer.writerow(row)
-                # Convert complex objects to JSON strings
-                # TODO: check two image are the same
-                conv_id = row["states"][0]['conv_id']
-                image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
-                if not os.path.exists(image_path):
-                    response = "NA"
-                    ic(image_path)
-                else:
-                    base64_image = encode_image(image_path)
-                    left_response = row["states"][0]['messages'][1][1]
-                    right_response = row["states"][1]['messages'][1][1]
-                    sep = "-" * 20
-                    instruction = row["states"][0]['messages'][0][1]
-                    generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
-                    text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
-                    # ic(text_prompt)
-                    try:
-                        response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
-                    except:
-                        ic(">>> skip")
-                        response = "NA"
-                    
-                    # response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
-                    ic(row['type'], response)
-                    if response.strip() not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
-                        response = "NA"
-                    # ic(generated_sentence)
-                    
-                    # if row['type'] == "leftvote":
-                    #     row['type'] = "A"
-                    # elif row['type'] == "rightvote":
-                    #     row['type'] = "B"
-                    # elif row['type'] in ["bothbad_vote", "tievote"]:
-                    #     row['type'] = "C"
-                    if row['type'] == response.strip():
-                        correct_vote += 1
-                row['models'] = json.dumps(row['models'])
-                row['states'] = json.dumps(row['states'], ensure_ascii=False)
-                row['gpt_vote'] = response
-                
-                # Write the modified row to the CSV file
-                writer.writerow(row)
-                # if row["type"] == "leftvote":
-                #     winner, loser = model_names[0], model_names[1]
-                #     winner_conv, loser_conv = row["states"][0], row["states"][1]
-                # elif row["type"] == "rightvote":
-                #     loser, winner = model_names[0], model_names[1]
-                #     loser_conv, winner_conv = row["states"][0], row["states"][1]
-
-                # if loser == "llava-v1.5-13b" and winner == "llava-v1.5-13b":
-                #     print("=" * 20)
-                #     print(f"Winner: {winner}")
-                #     pretty_print_conversation(winner_conv["messages"])
-                #     print(f"Loser: {loser}")
-                #     pretty_print_conversation(loser_conv["messages"])
-                #     print("=" * 20)
-                #     input()
-                # if row['type'] == 'bothbad_vote':
-                #     from icecream import ic
-                #     ic(model_names)
-                # if row["type"] == "bothbad_vote" and "gpt-4-vision-preview" in model_names:
-                #    print("=" * 20)
-                #    print(f"Model A: {model_names[0]}")
-                #    pretty_print_conversation(row["states"][0]["messages"])
-                #    print(f"Model B: {model_names[1]}")
-                #    pretty_print_conversation(row["states"][1]["messages"])
-                #    print("=" * 20)
-                #    input()
-                # if correct_vote >= 300: break
-    ic(total_vote, correct_vote)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--max-num-files", type=int)
-    args = parser.parse_args()
-
-    log_files = get_log_files(args.max_num_files)
-    
-
-            
-    inspect_convs(log_files)

arena_elo/elo_rating/inspect_cost.py

@@ -1,177 +0,0 @@
-import fire
-import time
-import json
-from collections import defaultdict
-from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
-from .utils import detect_language, get_time_stamp_from_date, get_input_image_path, load_image_from_path
-from tqdm import tqdm
-VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote", "chat"]
-
-
-def remove_html(raw):
-    if raw.startswith("<h3>"):
-        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
-    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
-        return raw[13:]
-    return raw
-
-
-def read_file(filename):
-    data = []
-    for retry in range(5):
-        try:
-            # lines = open(filename).readlines()
-            for l in open(filename):
-                row = json.loads(l)
-                if row["type"] in VOTES:
-                    data.append(row)
-            break
-        except FileNotFoundError:
-            time.sleep(2)
-    return data
-
-
-def read_file_parallel(log_files, num_threads=16):
-    data_all = []
-    from multiprocessing import Pool
-
-    with Pool(num_threads) as p:
-        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
-        for ret in ret_all:
-            data_all.extend(ret)
-    return data_all
-
-def num_tokens(s:str):
-    if s is None:
-        return 0
-    return len(s) / 4
-
-def main(
-):
-    log_files = get_log_files()
-    data = read_file_parallel(log_files)
-    
-    all_model_counts = defaultdict(int)
-    all_model_input_tokens_counts = defaultdict(list)
-    all_model_output_tokens_counts = defaultdict(list)
-    all_model_image_sizes = defaultdict(list)
-    chat_battle_counts = defaultdict(int)
-    for row in tqdm(data, desc="counting"):
-        if row['type'] == "chat":
-            chat_battle_counts["chat"] += 1
-            all_model_counts[row['model']] += 1
-            tstamp = row["tstamp"]
-            conv_id = row["state"]["conv_id"]
-            
-            image = load_image_from_path(get_input_image_path(tstamp, conv_id))
-            if image is None:
-                image_size = None
-            else:
-                image_size = load_image_from_path(get_input_image_path(tstamp, conv_id)).size
-            all_model_image_sizes[row['model']].append(image_size)
-            try:
-                for message in row["state"]["messages"][row["state"]["offset"] :: 2]:
-                    all_model_input_tokens_counts[row['model']].append(num_tokens(message[1]))
-                for message in row["state"]["messages"][row["state"]["offset"] + 1 :: 2]:
-                    all_model_output_tokens_counts[row['model']].append(num_tokens(message[1]))
-            except Exception as e:
-                print(row)
-                raise e
-            
-        else:
-            chat_battle_counts[row['type']] += 1
-            if row["models"][0] is None or row["models"][1] is None:
-                continue
-
-            # Resolve model names
-            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
-            if "model_name" in row["states"][0]:
-                models_hidden = [
-                    row["states"][0]["model_name"],
-                    row["states"][1]["model_name"],
-                ]
-                if models_hidden[0] is None:
-                    models_hidden = models_public
-            else:
-                models_hidden = models_public
-
-            if (models_public[0] == "" and models_public[1] != "") or (
-                models_public[1] == "" and models_public[0] != ""
-            ):
-                continue
-
-            if models_public[0] == "" or models_public[0] == "Model A":
-                anony = True
-                models = models_hidden
-            else:
-                anony = False
-                models = models_public
-                if not models_public == models_hidden:
-                    continue
-            
-            all_model_counts[models[0]] += 1
-            all_model_counts[models[1]] += 1
-            tstamp = row["tstamp"]
-            conv_id1 = row["states"][0]["conv_id"]
-            conv_id2 = row["states"][1]["conv_id"]
-            
-            image1 = load_image_from_path(get_input_image_path(tstamp, conv_id1))
-            image2 = load_image_from_path(get_input_image_path(tstamp, conv_id2))
-            all_model_image_sizes[models[0]].append(None if image1 is None else image1.size)
-            all_model_image_sizes[models[1]].append(None if image2 is None else image2.size)
-            
-            for message in row["states"][0]["messages"][row["states"][0]["offset"] :: 2]:
-                all_model_input_tokens_counts[models[0]].append(num_tokens(message[1]))
-            for message in row["states"][0]["messages"][row["states"][0]["offset"] + 1 :: 2]:
-                all_model_output_tokens_counts[models[0]].append(num_tokens(message[1]))
-            for message in row["states"][1]["messages"][row["states"][1]["offset"] :: 2]:
-                all_model_input_tokens_counts[models[1]].append(num_tokens(message[1]))
-            for message in row["states"][1]["messages"][row["states"][1]["offset"] + 1 :: 2]:
-                all_model_output_tokens_counts[models[1]].append(num_tokens(message[1]))
-
-    print("### Chat battle counts (requests)")
-    print(json.dumps(chat_battle_counts, indent=4))
-    
-    print("### Model counts (requests)")
-    print(json.dumps(all_model_counts, indent=4))
-    
-    print("### Model Avg input tokens counts (tokens)")
-    average_input_tokens_counts = {}
-    for model, counts in all_model_input_tokens_counts.items():
-        average_input_tokens_counts[model] = sum(counts) / len(counts)
-    print(json.dumps(average_input_tokens_counts, indent=4))
-    
-    print("### Model AVg output tokens counts (tokens)")
-    average_output_tokens_counts = {}
-    for model, counts in all_model_output_tokens_counts.items():
-        average_output_tokens_counts[model] = sum(counts) / len(counts)
-    print(json.dumps(average_output_tokens_counts, indent=4))
-    
-    print("### Model Avg image sizes (height, width)")
-    average_image_sizes = {}
-    for model, sizes in all_model_image_sizes.items():
-        avg_height = sum([size[0] for size in sizes if size is not None]) / len(sizes)
-        avg_width = sum([size[1] for size in sizes if size is not None]) / len(sizes)
-        average_image_sizes[model] = (avg_height, avg_width)
-    print(json.dumps(average_image_sizes, indent=4))
-    
-    print("### GPT-4V estimated cost (USD)")
-    gpt_4v_name = "gpt-4-vision-preview"
-    gpt_4v_cost = {}
-    gpt_4v_cost['input'] = sum(all_model_input_tokens_counts[gpt_4v_name]) / 1000 * 0.01
-    gpt_4v_cost['output'] = sum(all_model_output_tokens_counts[gpt_4v_name]) / 1000 * 0.03
-    
-    all_image_cost = 0
-    for size in all_model_image_sizes[gpt_4v_name]:
-        if size is None:
-            continue
-        all_image_tokens = (size[0] // 512 + 1) * (size[1] // 512 + 1) * 170 + 85
-        all_image_cost += all_image_tokens / 1000 * 0.01
-    gpt_4v_cost['image'] = all_image_cost
-    print(json.dumps(gpt_4v_cost, indent=4))
-        
-    
-    
-    
-if __name__ == "__main__":
-    fire.Fire(main)

arena_elo/elo_rating/inspect_elo_rating_pkl.py

@@ -1,33 +0,0 @@
-import pickle
-import plotly.graph_objects as go
-
-def output_figure(data, figure_name="battle_count_heatmap", label="annoy"):
-    fig = data[label][figure_name]
-    fig.update_layout(
-        height=700,
-        width=700,
-        title={'text': f'{figure_name}', 'x': 0.5, 'y': 0.07},
-        xaxis_title="Model B",
-        yaxis_title="Model A",
-        # coloraxis_colorscale=[[0.0, '#0d0887'], [1.0, '#f0f921']],
-        margin={'t': 60}
-    )
-    fig.write_image(f"{figure_name}.png")
-
-with open("./results/latest/elo_results.pkl",'rb') as f:
-    data = pickle.load(f)
-    print()
-    df = data["anony"]["leaderboard_table_df"]
-    # sort by rating 
-    print(data["anony"].keys())
-    
-    for figure_name in [ 'win_fraction_heatmap', 'battle_count_heatmap',]:
-        output_figure(data, figure_name, "anony")
-        
-    df = df.sort_values(by=["rating"], ascending=False)
-    print(df) 
-    df = data["full"]["leaderboard_table_df"]
-    # sort by rating 
-    df = df.sort_values(by=["rating"], ascending=False)
-    print(df)
-    print('done') 

arena_elo/elo_rating/upload_battle_data.py

@@ -1,168 +0,0 @@
-import fire
-import json
-import os
-import datasets
-import random
-import datetime
-from pathlib import Path
-from datetime import datetime
-from PIL import Image
-
-datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
-
-def create_hf_battle_dataset(data_file: str, split="test", task_type="t2i_generation"):
-    if task_type == "t2i_generation":
-        features = datasets.Features(
-            {
-                "index": datasets.Value("int32"),
-                "tstamp": datasets.Value("int32"),
-                "prompt": datasets.Value("string"),
-                "left_model": datasets.Value("string"),
-                "left_image": datasets.Image(),
-                "right_model": datasets.Value("string"),
-                "right_image": datasets.Image(),
-                "vote_type": datasets.Value("string"),
-                "winner": datasets.Value("string"),
-                "anony": datasets.Value("bool"),
-                "judge": datasets.Value("string"),
-            }
-        )
-    else:
-        raise ValueError(f"Task type {task_type} not supported")
-    hf_dataset = datasets.Dataset.from_list(
-        data_file,
-        features=features,
-        split=split,
-    )
-    return hf_dataset
-                           
-                                            
-
-
-def load_image(path:str):
-    try:
-        return Image.open(path)
-    except Exception as e:
-        print(f"Error loading image {path}: {e}")
-        return None
-
-def get_date_from_time_stamp(unix_timestamp: int):
-    # Create a datetime object from the Unix timestamp
-    dt = datetime.fromtimestamp(unix_timestamp)
-
-    # Convert the datetime object to a string with the desired format
-    date_str = dt.strftime("%Y-%m-%d")
-    return date_str
-
-def load_battle_image(battle, log_dir):
-    image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
-    return load_image(image_path)
-
-def find_media_path(conv_id, task_type, log_dir):
-    media_directory_map = {
-        "t2i_generation": "images/generation",
-        "image_edition": "images/edition",
-        "text2video": "videos/generation"
-    }
-    if task_type == "t2i_generation":
-        media_path = Path(log_dir) / media_directory_map[task_type] / f"{conv_id}.jpg"
-    else:
-        raise ValueError(f"Task type {task_type} not supported")
-    return media_path
-    
-
-def main(
-    task_type='t2i_generation',
-    # data_file: str = "./results/latest/clean_battle_conv.json",
-    data_file: str = None,
-    repo_id: str = "TIGER-Lab/GenAI-Arena-human-eval",
-    log_dir: str = os.getenv("LOGDIR", "../GenAI-Arena-hf-logs/vote_log"),
-    config_name='battle',
-    split='test',
-    token = os.environ.get("HUGGINGFACE_TOKEN", None),
-    seed=42,
-):
-    if data_file is None:
-        data_file = f"./results/latest/clean_battle_{task_type}.json"
-    if not os.path.exists(data_file):
-        raise ValueError(f"Data file {data_file} does not exist")
-    with open(data_file, "r") as f:
-        data = json.load(f)
-    
-    # add index according to the tsamp
-    if seed is not None:
-        random.seed(seed)
-    
-    
-    data = sorted(data, key=lambda x: x['tstamp'])
-    required_keys_each_task = {
-        "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
-        "t2i_generation": ["prompt"],
-        "video_generation": ["prompt"]
-    }
-    valid_data = []
-    for i, battle in enumerate(data):
-        if any(key not in battle['inputs'] for key in required_keys_each_task[task_type]):
-            # print(battle['inputs'])
-            # print(f"Skipping battle {i} due to missing keys")
-            continue
-        valid_data.append(battle)
-    print(f"Total battles: {len(data)}, valid battles: {len(valid_data)}, removed battles: {len(data) - len(valid_data)}")
-    data = valid_data
-    
-    # data = random.sample(data, 50 * 7+2)
-    
-    for i, battle in enumerate(data):
-        battle['index'] = i
-    
-
-    new_data = []
-    if task_type == 't2i_generation':
-        for battle in data:
-            prompt = battle['inputs']['prompt']
-            model_a = battle['model_a']
-            model_b = battle['model_b']
-            model_a_conv_id = battle['model_a_conv_id']
-            model_b_conv_id = battle['model_b_conv_id']
-            tstamp = battle['tstamp']
-            vote_type = battle['vote_type']
-            left_image_path = find_media_path(model_a_conv_id, task_type, log_dir)
-            right_image_path = find_media_path(model_b_conv_id, task_type, log_dir)
-            left_image = load_image(left_image_path)
-            right_image = load_image(right_image_path)
-            if left_image is None or right_image is None:
-                print(f"Skipping battle {battle['index']} due to missing images")
-                continue
-            new_data.append({
-                "index": battle['index'],
-                "tstamp": tstamp,
-                "prompt": prompt,
-                "left_model": model_a,
-                "left_image": left_image,
-                "right_model": model_b,
-                "right_image": right_image,
-                "vote_type": vote_type,
-                "winner": battle['winner'],
-                "anony": battle['anony'],
-                "judge": battle['judge'],
-            })
-        split = "test"
-        hf_dataset = create_hf_battle_dataset(new_data, split, task_type)
-    else:
-        raise ValueError(f"Task type {task_type} not supported")
-
-    print(hf_dataset)
-    print(f"Uploading to part {repo_id}:{split}...")
-    hf_dataset.push_to_hub(
-        repo_id=repo_id,
-        config_name=config_name,
-        split=split,
-        token=token,
-        commit_message=f"Add vision-arena {split} dataset",
-    )
-    
-    print("Done!")
-    
-    
-if __name__ == "__main__":
-    fire.Fire(main)

arena_elo/elo_rating/utils.py

@@ -1,91 +0,0 @@
-from datetime import datetime
-import pytz
-import PIL
-import os 
-
-import sys
-sys.path.append('../')
-from model.model_registry import get_model_info
-
-def detect_language(text: str) -> str:
-    """Detect the langauge of a string."""
-    try:
-        import polyglot  # pip3 install polyglot pyicu pycld2
-        from polyglot.detect import Detector
-        from polyglot.detect.base import logger as polyglot_logger
-        import pycld2
-    except ImportError as e:
-        print("Please install the required libraries: polyglot, pycld2: pip3 install polyglot pyicu pycld2")
-        exit(1)
-
-    polyglot_logger.setLevel("ERROR")
-
-    try:
-        lang_code = Detector(text).language.name
-    except (pycld2.error, polyglot.detect.base.UnknownLanguage):
-        lang_code = "unknown"
-    return lang_code
-
-
-def get_time_stamp_from_date(date_str:str):
-    """
-    Convert a date string to a Unix timestamp
-    Args:
-        date_str (str): The input date string in the format 'YYYY-MM-DD-HH:MM-TZ', e.g. '2024-02-10-14:00-PT'
-    """
-    
-    # Convert the date string into a format that Python's datetime can understand
-    # and specify the correct timezone for PT, which is 'US/Pacific'
-    date_format = "%Y-%m-%d-%H:%M-%Z"
-
-    # Parse the date string into a datetime object
-    # Note: PT is not directly recognized by pytz, so we manually map it to 'US/Pacific'
-    timezone_map = {
-        "PT": "US/Pacific",
-    }
-
-    # Extract the timezone abbreviation
-    tz_abbr = date_str.split("-")[-1]
-    # Map the abbreviation to a pytz timezone
-    tz_info = pytz.timezone(timezone_map[tz_abbr])
-
-    # Remove the timezone abbreviation for parsing
-    date_str_parsed = date_str.rsplit("-", 1)[0]
-
-    # Create a datetime object with the corresponding timezone
-    dt = datetime.strptime(date_str_parsed, "%Y-%m-%d-%H:%M").replace(tzinfo=tz_info)
-
-    # Convert the datetime object to a Unix timestamp
-    unix_timestamp = dt.timestamp()
-    return unix_timestamp
-
-def get_date_from_time_stamp(unix_timestamp: int):
-    # Create a datetime object from the Unix timestamp
-    dt = datetime.fromtimestamp(unix_timestamp)
-
-    # Convert the datetime object to a string with the desired format
-    date_str = dt.strftime("%Y-%m-%d %H:%M:%S %Z")
-    return date_str
-
-
-def get_input_image_path(tstamp, conv_id):
-    # from tstamp to date e.g. 2024-02-10
-    date_str = datetime.fromtimestamp(tstamp, tz=pytz.timezone("US/Pacific")).strftime("%Y-%m-%d")
-    LOGDIR = os.getenv("LOGDIR")
-    return f"{LOGDIR}/{date_str}-convinput_images/input_image_{conv_id}.png"
-
-def load_image_from_path(image_path):
-    # Load the image from the specified
-    # path using the Python Imaging Library (PIL)
-    try:
-        image = PIL.Image.open(image_path)
-        return image
-    except FileNotFoundError:
-        print(f"Image not found at path: {image_path}")
-        return None
-    except PIL.UnidentifiedImageError:
-        print(f"Unidentified image format at path: {image_path}")
-        return None
-    
-
-    

arena_elo/evaluator/convert_to_evaluator_data.py

@@ -1,134 +0,0 @@
-import argparse
-import json
-import os
-import time
-from pytz import timezone
-from tqdm import tqdm
-import base64
-from icecream import ic
-from PIL import Image
-
-
-# Function to encode the image
-def encode_image(image_path):
-    with open(image_path, "rb") as image_file:
-        return base64.b64encode(image_file.read()).decode('utf-8')
-
-def get_log_files(max_num_files=None):
-    dates = []
-    for month in [2, 3]:
-        for day in range(1, 32):
-            dates.append(f"2024-{month:02d}-{day:02d}")
-
-    num_servers = 1
-    filenames = []
-    for d in dates:
-        for i in range(num_servers):
-            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
-            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
-            if os.path.exists(name):
-                filenames.append(name)
-    max_num_files = max_num_files or len(filenames)
-    filenames = filenames[-max_num_files:]
-    return filenames
-
-
-def pretty_print_conversation(messages):
-    for role, msg in messages:
-        print(f"[[{role}]]: {msg}")
-
-task_template_map = {
-    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
-    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
-    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
-    "pair_rate_wexplanation": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
-    "pair_rate": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
-}
-
-def inspect_convs(log_files):
-    json_data = []
-
-    ic(log_files)
-    total_vote = 0
-
-    for filename in tqdm(log_files, desc="read files"):
-        for retry in range(5):
-            try:
-                lines = open(filename).readlines()
-                break
-            except FileNotFoundError:
-                time.sleep(2)
-        
-        for l in lines:
-            row = json.loads(l)
-            
-            if "states" not in row:
-                continue
-            if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
-                continue
-            
-            model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
-            
-
-            # Iterate through each state and write the relevant information
-            if not len(row["states"][0]['messages']): continue
-            # ic(row["states"][0]['messages'][1][1])
-
-            if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
-            total_vote += 1
-
-            conv_id = row["states"][0]['conv_id']
-            image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
-            if not os.path.exists(image_path) :
-                continue
-            try:
-                image = Image.open(image_path).convert("RGB")
-            except:
-                continue
-
-            left_response = row["states"][0]['messages'][1][1]
-            right_response = row["states"][1]['messages'][1][1]
-            instruction = row["states"][0]['messages'][0][1]
-            generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
-            text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
-            
-            user_input = text_prompt
-            # Create the conversation structure
-            conversation = [
-                {
-                    "from": "human",
-                    "value": user_input
-                },
-                {
-                    "from": "gpt",
-                    "value": row["type"]
-                }
-            ]
-            
-            # Create the JSON object for each row
-            json_obj = {
-                "id": conv_id,
-                "image": image_path,
-                "conversations": conversation
-            }
-            
-            # Append the JSON object to the list
-            json_data.append(json_obj)
-
-        # Write the JSON data to a file
-        with open('output_evaluator_data.json', 'w') as json_file:
-            json.dump(json_data, json_file, indent=2)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--max-num-files", type=int)
-    args = parser.parse_args()
-
-    log_files = get_log_files(args.max_num_files)
-    
-
-            
-    inspect_convs(log_files)
-
-
-

arena_elo/evaluator/rating_analysis.ipynb

@@ -1,321 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 43,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1338\n",
-      "1044\n"
-     ]
-    }
-   ],
-   "source": [
-    "\n",
-    "import pandas as pd\n",
-    "import json\n",
-    "\n",
-    "# Replace 'your_file_name.csv' with the path to your CSV file\n",
-    "file_name = 'all_pairvote_log_wgpt.csv'\n",
-    "\n",
-    "# Load the CSV file into a DataFrame\n",
-    "df = pd.read_csv(file_name)\n",
-    "\n",
-    "# Define a function to parse JSON data\n",
-    "def parse_json(data):\n",
-    "    try:\n",
-    "        # Parse the JSON data\n",
-    "        return json.loads(data)\n",
-    "    except ValueError as e:\n",
-    "        # Return None or an empty dictionary if the data cannot be parsed\n",
-    "        return None\n",
-    "\n",
-    "# Apply the parse_json function to the 'models' and 'states' columns\n",
-    "df['models'] = df['models'].apply(parse_json)\n",
-    "df['states'] = df['states'].apply(parse_json)\n",
-    "# row[\"states\"][0]['messages'][0][1]\n",
-    "\n",
-    "# Now df contains the parsed JSON data in the 'models' and 'states' columns\n",
-    "# print(df.head())\n",
-    "print(len(df))\n",
-    "# filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"])]#, \"tievote\", \"bothbad_vote\"\n",
-    "# \\#1\n",
-    "filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])]\n",
-    "# \\#2\n",
-    "# filter_vote_df = df\n",
-    "filter_vote_df.loc[~filter_vote_df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]), \"gpt_vote\"] = \"tie\"\n",
-    "filter_vote_df.loc[~filter_vote_df[\"type\"].isin([\"leftvote\", \"rightvote\"]), \"type\"] = \"tie\"\n",
-    "# \\#3\n",
-    "#[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]) & df[\"type\"].isin([\"leftvote\", \"rightvote\"])]\n",
-    "filtered_df = filter_vote_df[filter_vote_df[\"states\"].apply(lambda x: len(x[0]['messages'][0][1]) > 10)]\n",
-    "print(len(filtered_df))\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Confusion Matrix:\n",
-      "[[300  61  34]\n",
-      " [102 269  27]\n",
-      " [ 99 111  41]]\n",
-      "\n",
-      "Accuracy: 0.5842911877394636\n"
-     ]
-    }
-   ],
-   "source": [
-    "import warnings\n",
-    "warnings.filterwarnings('ignore')\n",
-    "\n",
-    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
-    "import pandas as pd\n",
-    "\n",
-    "# Assuming df is your DataFrame\n",
-    "\n",
-    "# True labels\n",
-    "y_true = filtered_df[\"type\"]\n",
-    "\n",
-    "# Predictions\n",
-    "y_pred = filtered_df[\"gpt_vote\"]\n",
-    "\n",
-    "# Compute the confusion matrix\n",
-    "# conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
-    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tie\"])\n",
-    "\n",
-    "# Compute the accuracy\n",
-    "accuracy = accuracy_score(y_true, y_pred)\n",
-    "\n",
-    "print(\"Confusion Matrix:\")\n",
-    "print(conf_matrix)\n",
-    "\n",
-    "print(\"\\nAccuracy:\", accuracy)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 45,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0YAAAJwCAYAAACtcHEcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB0yUlEQVR4nO3deZyN5f/H8feZfcaYMZaxMwyGkTWJURSKKFshS/YtREQoe4oKaVFpsaRBZUlRUrLLUtYsk2VkaYSxjGEYZq7fH37O12ksc8aMM+N+PR+P83jMXPd9X+dzxrnHec913ddtM8YYAQAAAICFubm6AAAAAABwNYIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAWsWTJElWsWFE+Pj6y2Ww6c+ZMuvY/ffp02Ww2HTx4MF37zcpsNptGjhzp6jIApALBCECafPjhh7LZbHrwwQddXcodu/Zh7vfff7/h9kceeUT33XffXa7q3hQdHa3evXurVKlS8vPzk5+fn8LDw9WrVy9t377dYd+RI0fKZrPZH9f2HTp0qOLi4iTJYfutHitWrLhtbZcvX1Z4eLhsNpvGjx9vb+/Tp49sNpv27dt302NfffVV2Wy2FK/hRlasWKFmzZopX7588vLyUnBwsJ566inNnz//tsfeidjYWLVo0UK+vr6aPHmyZs6cqWzZsmXoc95NISEhstlsqlu37g23f/rpp/b3w83O9VtZt26dRo4cme5hEkDm4eHqAgBkTZGRkQoJCdHGjRu1b98+lShRwtUlIZNbtGiRWrZsKQ8PD7Vp00YVKlSQm5ub9uzZo/nz5+ujjz5SdHS0ihYt6nDcRx99JH9/f8XHx2vp0qV6/fXX9euvv2rt2rWaOXOmw75ffPGFfv755xTtZcqUuW1977//vg4dOpSivU2bNnr//fc1a9YsDR8+/IbHzp49W+XKlVP58uVv+RwjRozQ6NGjVbJkSXXv3l1FixZVbGysfvjhBz399NOKjIxU69atb1trWmzatEnnzp3Ta6+9dtPwcKeee+45Pfvss/L29s6Q/m/Hx8dHy5cv17Fjx5QvXz6HbZGRkfLx8dHFixfT1Pe6des0atQodejQQTly5Ej1cQkJCfLw4OMWkCUYAHDSgQMHjCQzf/58kydPHjNy5MhUHXf58mVz6dKlDK7OedOmTTOSzKZNm264vVatWqZs2bJ3uap7y759+0y2bNlMmTJlzD///JNi++XLl827775rDh06ZG8bMWKEkWROnDjhsG+zZs2MJLNu3boU/fTq1cuk5b+2f//91wQGBprRo0cbSebtt9922F6iRAlTunTpGx67bt06I8mMGzfuls/xzTffGEnmmWeeMYmJiSm2L1myxHz//fdO155aM2bMuOX7PKsrWrSoqVOnjgkICDCTJk1y2Hb48GHj5uZmnn766TT/DN5++20jyURHR99236SkJJOQkOD0cwBwLabSAXBaZGSkgoKC1LBhQz3zzDOKjIxMsc/BgwftU5ImTZqk0NBQeXt7a9euXZKkPXv26JlnnlHOnDnl4+OjKlWq6LvvvnPo49oUt7Vr16p///7KkyePsmXLpqZNm+rEiRMO+y5cuFANGzZUgQIF5O3trdDQUL322mtKSkpK99d/7bVNnz49xbb/Xk9wbTrYX3/9pbZt2yowMFB58uTRsGHDZIzR4cOH1bhxYwUEBChfvnyaMGGCQ3+JiYkaPny47r//fgUGBipbtmx6+OGHtXz58hvWNH78eH3yySf2n/cDDzygTZs23fL1/P7777LZbJoxY0aKbT/99JNsNpsWLVokSTp37pxefPFFhYSEyNvbW8HBwXrssce0efPmWz7HW2+9pfPnz2vatGnKnz9/iu0eHh7q06ePChcufMt+JKl27dqSrk7LSy+DBw9WWFiY2rZte8Ptbdq00Z49e274OmfNmiWbzaZWrVrd8jmGDRumnDlzaurUqfL09EyxvV69enryySft3x8/flydO3dW3rx55ePjowoVKqT4N0rtv/sjjzyi9u3bS5IeeOAB2Ww2dejQQdLVKWjXvr7eI488okceecSh7f3331fZsmXl5+enoKAgValSRbNmzbJvv9k1Rh9++KHKli0rb29vFShQQL169UoxJe3alNVdu3bp0UcflZ+fnwoWLKi33nrrZj/SFHx8fNSsWTOHmqSrI3pBQUGqV69eimO2b9+uDh06qHjx4vLx8VG+fPnUqVMnxcbG2vcZOXKkBg4cKEkqVqyYfUretddps9nUu3dvRUZG2l/nkiVL7Nuu/U5ISEhQ6dKlVbp0aSUkJNj7P3XqlPLnz6+IiIgM+Z0FIHUY2wXgtMjISDVr1kxeXl5q1aqVPvroI23atEkPPPBAin2nTZumixcvqlu3bvL29lbOnDm1c+dO1ahRQwULFtTgwYOVLVs2ff3112rSpInmzZunpk2bOvTxwgsvKCgoSCNGjNDBgwc1adIk9e7dW1999ZV9n+nTp8vf31/9+/eXv7+/fv31Vw0fPlxxcXF6++23U/W6zp49q5MnT6Zov3z5spM/oZRatmypMmXKaNy4cVq8eLHGjBmjnDlzasqUKapdu7befPNNRUZGasCAAXrggQdUs2ZNSVJcXJw+++wztWrVSl27dtW5c+f0+eefq169etq4caMqVqzo8DyzZs3SuXPn1L17d9lsNr311ltq1qyZDhw4cMMP45JUpUoVFS9eXF9//bX9w/M1X331lcMHyh49emju3Lnq3bu3wsPDFRsbqzVr1mj37t2qXLnyTV//okWLVKJEiXS5Jm3//v2SpFy5ct1xX5K0ceNGzZgxQ2vWrJHNZrvhPm3atNGoUaM0a9Ysh9eZlJSkr7/+Wg8//LCKFCly0+fYu3ev9uzZo06dOil79uy3rSkhIUGPPPKI9u3bp969e6tYsWL65ptv1KFDB505c0Z9+/Z12P92/+6vvvqqwsLC9Mknn2j06NEqVqyYQkNDU/kTuurTTz9Vnz599Mwzz6hv3766ePGitm/frg0bNtxy+t/IkSM1atQo1a1bV88//7yioqLsvzPWrl3r8L48ffq06tevr2bNmqlFixaaO3euBg0apHLlyumJJ55IVZ2tW7fW448/rv3799tf46xZs/TMM8/c8Bz4+eefdeDAAXXs2FH58uXTzp079cknn2jnzp1av369bDabmjVrpr/++kuzZ8/WO++8o9y5c0uS8uTJY+/n119/1ddff63evXsrd+7cCgkJSfFcvr6+mjFjhmrUqKFXX31VEydOlCT16tVLZ8+e1fTp0+Xu7p6q1wkgA7h6yApA1vL7778bSebnn382xhiTnJxsChUqZPr27euwX3R0tJFkAgICzPHjxx221alTx5QrV85cvHjR3pacnGwiIiJMyZIl7W3XprjVrVvXJCcn29v79etn3N3dzZkzZ+xtFy5cSFFr9+7djZ+fn8Pz3Mi157nV4/qpdNde27Rp01L0JcmMGDHC/v216WDdunWzt125csUUKlTI2Gw2h+lXp0+fNr6+vqZ9+/YO+/53+uHp06dN3rx5TadOnVLUlCtXLnPq1Cl7+8KFC42k207RGjJkiPH09HQ49tKlSyZHjhwOzxMYGGh69ep1y77+6+zZs0aSadKkSYptp0+fNidOnLA/rv93vPazi4qKMidOnDDR0dFmypQpxtvb2+TNm9ecP38+RX/OTqVLTk42VatWNa1atTLG/O/n+N+pdMYY88ADD5hChQqZpKQke9uSJUuMJDNlypRbPs+1f4d33nknVXVNmjTJSDJffvmlvS0xMdFUr17d+Pv7m7i4OId6U/PvfrMpo0WLFnV4z11Tq1YtU6tWLfv3jRs3vu2U0mvPcW262fHjx42Xl5d5/PHHHX5uH3zwgZFkpk6d6vB8kswXX3xhb7t06ZLJly+fefrpp2/5vNdeR8OGDc2VK1dMvnz5zGuvvWaMMWbXrl1Gklm5cuUNfwY3+t0xe/ZsI8msWrXK3narqXSSjJubm9m5c+cNt13/O8GYq+ebm5ubWbVqlX2K5X+n/wG4+5hKB8ApkZGRyps3rx599FFJV6eJtGzZUnPmzLnhFJCnn37a4a+qp06d0q+//qoWLVro3LlzOnnypE6ePKnY2FjVq1dPe/fu1dGjRx366Natm8Nf8h9++GElJSXp77//trf5+vrav77W78MPP6wLFy5oz549qXptkydP1s8//5zicbsL6lOjS5cu9q/d3d1VpUoVGWPUuXNne3uOHDkUFhamAwcOOOzr5eUlSUpOTtapU6d05coVValS5YbTulq2bKmgoCD79w8//LAkOfR5Iy1bttTly5cdVkZbunSpzpw5o5YtWzrUuGHDBv3zzz+pfen2FeT8/f1TbHvkkUeUJ08e+2Py5Mkp9gkLC1OePHlUrFgxde/eXSVKlNDixYvl5+eX6hpuZvr06dqxY4fefPPN2+7btm1bHTlyRKtWrbK3zZo1S15eXmrevPktj732M0jNaJEk/fDDD8qXL5/D9DxPT0/16dNH8fHxWrlypcP+af13d0aOHDl05MiR207NvN4vv/yixMREvfjii3Jz+99Hjq5duyogIECLFy922N/f399hOqOXl5eqVq3q1Otwd3dXixYtNHv2bElXf2cVLlzY/jP5r+t/d1y8eFEnT55UtWrVJOm2U0SvV6tWLYWHh6dq35EjR6ps2bJq3769evbsqVq1aqlPnz6pfi4AGYNgBCDVkpKSNGfOHD366KOKjo7Wvn37tG/fPj344IP6999/tWzZshTHFCtWzOH7ffv2yRijYcOGOXwgzpMnj0aMGCHp6rUV1/vvFKVrHwBPnz5tb9u5c6eaNm2qwMBABQQEKE+ePPYPWGfPnk3V66tatarq1q2b4nH9B860+u9rCAwMlI+Pj31KzvXt178uSZoxY4bKly8vHx8f5cqVS3ny5NHixYtv+LpS87O6kQoVKqh06dIO0xO/+uor5c6d235Nj3T1WqE///xThQsXVtWqVTVy5Mjbfmi9Fgbi4+NTbJsyZYp+/vlnffnllzc9ft68efr555+1YsUK7du3T3/++afuv//+Wz7n9eLj43Xs2DH749r1aXFxcRoyZIgGDhyYqmubnn32Wbm7u9uvX7l48aIWLFigJ5544rbvkYCAAElXQ3tq/P333ypZsqRDmJD+t7re9X8UkNL+7+6MQYMGyd/fX1WrVlXJkiXVq1cvrV279pbHXKszLCzMod3Ly0vFixdP8ToKFSqUYjpjUFCQ06+jdevW2rVrl7Zt26ZZs2bp2Wefvek0yVOnTqlv377KmzevfH197SFcSv3vDinl77pb8fLy0tSpUxUdHa1z585p2rRpN60PwN3DNUYAUu3XX39VTEyM5syZozlz5qTYHhkZqccff9yh7fq/xkpXRz0kacCAATe8EFpSiqW/bzbn3hgjSTpz5oxq1aqlgIAAjR49WqGhofLx8dHmzZs1aNAg+3Oml5t9gLnVRdM3eg23e12S9OWXX6pDhw5q0qSJBg4cqODgYLm7u2vs2LH2a22c7fNmWrZsqddff10nT55U9uzZ9d1336lVq1YOSw23aNFCDz/8sBYsWKClS5fq7bff1ptvvqn58+ff9BqQwMBA5c+fX3/++WeKbdeuObrVDUFr1qyZIkA6Y/z48Ro1apT9+6JFi+rgwYMaP368EhMT1bJlS/vzHzlyRNLVQHHw4EEVKFDAPmJ3baGJefPmafLkyfr+++917tw5tWnT5rY1lC5dWpK0Y8eONL+OW7mTf/dbvZ+v77dMmTKKiorSokWLtGTJEs2bN08ffvihhg8f7vDzvRN38jqu9+CDDyo0NFQvvviioqOjb3kNVIsWLbRu3ToNHDhQFStWlL+/v5KTk1W/fn2nfnf893fd7fz000+SrgbsvXv3OhWsAGQMghGAVIuMjFRwcPANpzvNnz9fCxYs0Mcff3zLDwjFixeXdHVaUHrdS2XFihWKjY3V/Pnz7YsWSOm7atn1rv01/r+rav33r9/pYe7cuSpevLjmz5/v8AH22uhaemrZsqVGjRqlefPmKW/evIqLi9Ozzz6bYr/8+fOrZ8+e6tmzp44fP67KlSvr9ddfv+XF8Q0bNtRnn32mjRs3qmrVqule+620a9dODz30kP37a+/PQ4cO6fTp0ypbtmyKY9544w298cYb2rJli8MCF23atNGSJUv0448/atasWQoICNBTTz112xpKlSqlsLAwLVy4UO++++4NpxVer2jRotq+fbuSk5MdRo2uTQv9772e7kRQUNANb1r6999/28/Xa7Jly6aWLVuqZcuWSkxMVLNmzfT6669ryJAh8vHxueHrkKSoqCiHvhITExUdHZ1h91OSpFatWmnMmDEqU6ZMikVKrjl9+rSWLVumUaNGOdyjau/evSn2Tc8Rne3bt2v06NHq2LGjtm7dqi5dumjHjh0KDAxMt+cA4Dym0gFIlYSEBM2fP19PPvmknnnmmRSP3r1769y5cymW3P6v4OBgPfLII5oyZYpiYmJSbP/vMtypce2vzNf/VTkxMVEffvih032lRkBAgHLnzu1wrYmkDHm+G722DRs26Lfffkv35ypTpozKlSunr776Sl999ZXy58/vEDSTkpJSTC0KDg5WgQIFdOnSpVv2/fLLL8vPz0+dOnXSv//+m2K7syMCzihevLjD1MgaNWpIkvr06aMFCxY4PKZMmSJJ6tChgxYsWJDir/hNmjSRn5+fPvzwQ/34449q1qzZDQPBjYwaNUqxsbHq0qWLrly5kmL70qVL7cuiN2jQQMeOHXOY2njlyhW9//778vf3V61atdL0s7iR0NBQrV+/XomJifa2RYsW6fDhww77Xb98tXR1Olh4eLiMMTddubFu3bry8vLSe++95/Bv/Pnnn+vs2bNq2LBhur2O/+rSpYtGjBiRYgn8693o/JKkSZMmpdg3W7ZsklL+QcRZly9fVocOHVSgQAG9++67mj59uv7991/169fvjvoFcOcYMQKQKt99953OnTunRo0a3XB7tWrVlCdPHkVGRjpcrH8jkydP1kMPPaRy5cqpa9euKl68uP7991/99ttvOnLkiLZt2+ZUbREREQoKClL79u3Vp08f2Ww2zZw5M0M/bHfp0kXjxo1Tly5dVKVKFa1atUp//fVXuj/Pk08+qfnz56tp06Zq2LChoqOj9fHHHys8PPyG1+zcqZYtW2r48OHy8fFR586dHUYrzp07p0KFCumZZ55RhQoV5O/vr19++UWbNm265YdPSSpZsqRmzZqlVq1aKSwsTG3atFGFChVkjFF0dLRmzZolNzc3FSpUKN1f081Urlw5xRLj16bUlS1bVk2aNElxjL+/v5o0aWK/zig10+iuadmypXbs2KHXX39dW7ZsUatWrVS0aFHFxsZqyZIlWrZsmb3fbt26acqUKerQoYP++OMPhYSEaO7cuVq7dq0mTZqU6kUcUqNLly6aO3eu6tevrxYtWmj//v368ssvUyzn/fjjjytfvnyqUaOG8ubNq927d+uDDz5Qw4YNb1pPnjx5NGTIEI0aNUr169dXo0aNFBUVpQ8//FAPPPDATe8blR6KFi3qcE+xGwkICFDNmjX11ltv6fLlyypYsKCWLl16w9Hma9e1vfrqq3r22Wfl6empp556yh6YUmvMmDHaunWrli1bpuzZs6t8+fIaPny4hg4dqmeeeUYNGjRwqj8A6YdgBCBVIiMj5ePjo8cee+yG293c3NSwYUNFRkam+Mvyf4WHh+v333/XqFGjNH36dMXGxio4OFiVKlVymM6SWrly5dKiRYv00ksvaejQoQoKClLbtm1Vp06dm17HdKeGDx+uEydOaO7cufr666/1xBNP6Mcff1RwcHC6Pk+HDh107NgxTZkyRT/99JPCw8P15Zdf6ptvvtGKFSvS9bmkqx/ehw4dqgsXLqQIuH5+furZs6eWLl2q+fPnKzk5WSVKlNCHH36o559//rZ9N27cWDt27NCECRO0dOlSTZ06VTabTUWLFlXDhg3Vo0cPVahQId1fU3pr06aNZs2apfz58zssTJEaY8aMUe3atfXee+/po48+0qlTpxQUFKRq1app4cKF9j88+Pr6asWKFRo8eLBmzJihuLg4hYWFadq0aTe8GeudqFevniZMmKCJEyfqxRdfVJUqVezn0/W6d++uyMhITZw4UfHx8SpUqJD69OmjoUOH3rL/kSNHKk+ePPrggw/Ur18/5cyZU926ddMbb7xx03tr3U2zZs3SCy+8oMmTJ8sYo8cff1w//vijChQo4LDfAw88oNdee00ff/yxlixZouTkZEVHRzsVjDZv3qw33nhDvXv3tq/sKV29wfDChQvVtWtX7dy5Uzly5EivlwfACTaTkX9SBQAAAIAsgGuMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFjePXmDV99KvV1dAuByvy0c6+oSAJcrXSC7q0sAALiYTyoTDyNGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACwv0wSj/fv3a+jQoWrVqpWOHz8uSfrxxx+1c+dOF1cGAAAA4F6XKYLRypUrVa5cOW3YsEHz589XfHy8JGnbtm0aMWKEi6sDAAAAcK/LFMFo8ODBGjNmjH7++Wd5eXnZ22vXrq3169e7sDIAAAAAVpApgtGOHTvUtGnTFO3BwcE6efKkCyoCAAAAYCWZIhjlyJFDMTExKdq3bNmiggULuqAiAAAAAFaSKYLRs88+q0GDBunYsWOy2WxKTk7W2rVrNWDAALVr187V5QEAAAC4x2WKYPTGG2+odOnSKly4sOLj4xUeHq6aNWsqIiJCQ4cOdXV5AAAAAO5xNmOMcXUR1xw+fFg7duxQfHy8KlWqpJIlS6apH99KvdO5MiDr+W3hWFeXALhc6QLZXV0CAMDFfDxSt1+mGDEaPXq0Lly4oMKFC6tBgwZq0aKFSpYsqYSEBI0ePdrV5QEAAAC4x2WKESN3d3fFxMQoODjYoT02NlbBwcFKSkpyqj9GjABGjACJESMAQBYbMTLGyGazpWjftm2bcubM6YKKAAAAAFhJKvNTxggKCpLNZpPNZlOpUqUcwlFSUpLi4+PVo0cPF1YIAAAAwApcGowmTZokY4w6deqkUaNGKTAw0L7Ny8tLISEhql69ugsrBAAAAGAFLg1G7du3lyQVK1ZMERER8vT0dGU5AAAAACzKpcHomlq1aikpKUnz5s3T7t27JUlly5ZVo0aN5O7u7uLqAAAAANzrMkUw2rdvnxo0aKCjR48qLCxMkjR27FgVLlxYixcvVmhoqIsrBAAAAHAvyxSr0vXp00ehoaE6fPiwNm/erM2bN+vQoUMqVqyY+vTp4+ryAAAAANzjMsWI0cqVK7V+/XqHpblz5cqlcePGqUaNGi6sDAAAAIAVZIpg5O3trXPnzqVoj4+Pl5eXlwsqwn91bf6Quj7zsIoWuBpedx84pjc++VFL1+6SJHl7eWhc/2ZqXu9+eXt56JffdqvvG1/p+Kn//bsWzhekd19pqVpVSik+4ZIiv9+gYe9/p6SkZJe8JiA9nDp5XJGfva+tG9fp0qWLylegkJ4fMEKhYeGSpA2rf9Uvi+bpwN49ij93Vm9+FKmQEmEurhrIeHNmRWrGtM918uQJlQorrcGvDFO58uVdXRZwV3EeZC2ZYirdk08+qW7dumnDhg0yxsgYo/Xr16tHjx5q1KiRq8uDpKP/ntGw9xcqos1bqtHmba3Y+Je+eaebyhTPJ0l6a8DTaljzPrV5+XM93mWS8ucJ1JwJXezHu7nZNP+95+Xl6aFHO0xQ1+Ez1bbRgxr+fENXvSTgjsWfi9PwFzvL3d1DQ954VxM/+1rPde+nbNkD7PtcupigsPsqqnWXF1xYKXB3LfnxB41/a6y69+ylOd8sUFhYaT3fvbNiY2NdXRpw13AeZD2ZIhi99957Cg0NVfXq1eXj4yMfHx/VqFFDJUqU0Lvvvuvq8iDph1V/6qc1u7T/0AntO3RcIyd/r/gLl1S1fDEF+PuoQ5PqGjRxvlZu+ktbdh9WtxFfqnrFUFUtFyJJqlu9jMoUz6dOr87Q9r+OaunaXRr94WJ1b1FTnh6sPIis6buvZihXnrzqOXCESpS+T8H5C6pClWrKV6CQfZ+ajzXUM891VbnKVV1YKXB3zZwxTc2eaaEmTZ9WaIkSGjpilHx8fPTt/HmuLg24azgPsp5MEYxy5MihhQsXKioqSnPnztXcuXMVFRWlBQsWONz0FZmDm5tNzevdr2y+XtqwPVqVyhSRl6eHfl0fZd/nr4P/6lDMKT1Yvpgk6cHyxfTnvn8cptb9vG63ArP7Kjw0/11/DUB6+P23VSpeqowmjh6krs0f06AerbXshwWuLgtwqcuJidq9a6eqVY+wt7m5ualatQht37bFhZUBdw/nQdaUKa4xWrNmjR566CGVLFlSJUuWdOrYS5cu6dKlSw5tJjlJNjdGIdJb2RIFtGLGS/Lx8lB8wiW1fOlT7TlwTBVKFdKlxMs6G5/gsP/x2DjlzXV1SlHeXAE6Hut4HdnxU3FXt+UOkKIEZDnHY47q5+/nqeHTbdS0dUftj9qlaZPHy8PDU7Uef9LV5QEucfrMaSUlJSlXrlwO7bly5VJ09AEXVQXcXZwHWVOmGDGqXbu2ihUrpldeeUW7du1y6tixY8cqMDDQ4XHl3z8yqFJr++vgv3rw2bGq2W68Pv1mjT4d/ZxK//81RoAVJZtkFStZWq0691KxEqVVt2Ez1WnQRD8vYpoEAABZTaYIRv/8849eeuklrVy5Uvfdd58qVqyot99+W0eOHLntsUOGDNHZs2cdHh55778LVVvP5StJOnD4pLbsPqzh73+nHX8dVa9Wj+hYbJy8vTwV6O/rsH9wrgD9G3t1VOjf2DgF58ruuD3n1dGkf0/G3Z0XAKSzoJy5VbBIMYe2gkWK6eTxYy6qCHC9oBxBcnd3T3GBeWxsrHLnzu2iqoC7i/Mga8oUwSh37tzq3bu31q5dq/3796t58+aaMWOGQkJCVLt27Vse6+3trYCAAIcH0+juDjebTd5eHtqy+5ASL1/Row/+bwnikkWDVSR/Tm3YHi1J2rA9WveVKKA8Qf72fepUK62z5xK0+wAfIpE1hZWtoJgjfzu0xRz5W3nyct0crMvTy0tlwstqw/rf7G3JycnasOE3la9QyYWVAXcP50HWlCmuMbpesWLFNHjwYFWoUEHDhg3TypUrXV0SJI1+oZF+WrtTh2NOK3s2H7V8oopqVimpp3p+qLj4i5r+7W9686VmOnX2vM6dv6iJg5pr/bYD2rjjoCTpl992a/eBY/p8THu9+u63ypsrQCN6PakpX69S4uUrrn1xQBo1eLq1hvftpAWzpqp6rce0L2qnlv2wQF1ffNW+T3zcWZ08fkynY09Ikv75/yCVI2cu5cjJXw1xb3qufUcNe2WQypa9T/eVK68vZ85QQkKCmjRt5urSgLuG8yDrsRljjKuLuGbt2rWKjIzU3LlzdfHiRTVu3Fht2rRR/fr1nerHt1LvDKrQuj4a0VqPVg1TvtwBOht/UX/uPaoJ037Rrxv2SPrfDV5b1P//G7yu262+Y7/Sv9ctuFAkf5DefeVZ1by/pM5fvKTI7zdq6HsLucFrBvlt4VhXl2AJf6xfrdmff6BjRw8rT74CevKZNqrToKl9+4qfvtdH40elOO6Z57qqebvud7NUSypdIPvtd0KGmB35pf3GlmGly2jQK0NVvnwFV5cF3FWcB5mDTyqHgjJFMBoyZIjmzJmjf/75R4899pjatGmjxo0by8/PL039EYwAghEgEYwAAKkPRpliKt2qVas0cOBAtWjRggvSAAAAANx1mSIYjR07VhEREfLwcCznypUrWrdunWrWrOmiygAAAABYQaZYle7RRx/VqVOnUrSfPXtWjz76qAsqAgAAAGAlmSIYGWNks9lStMfGxipbtmwuqAgAAACAlbh0Kl2zZleXK7TZbOrQoYO8vb3t25KSkrR9+3ZFRES4qjwAAAAAFuHSYBQYGCjp6ohR9uzZ5evra9/m5eWlatWqqWvXrq4qDwAAAIBFuCwY9e/fXx988IGyZcumgwcP6rPPPpO/v7+rygEAAABgYS67xuj9999XfHy8pKvLdV+4cMFVpQAAAACwOJeNGIWEhOi9997T448/LmOMfvvtNwUFBd1wX5brBgAAAJCRbMYY44on/vbbb9WjRw8dP35cNptNNyvDZrMpKSnJqb59K/VOjxKBLO23hWNdXQLgcqULZHd1CQAAF/NJ5VCQy0aMmjRpoiZNmig+Pl4BAQGKiopScHCwq8oBAAAAYGEuXZVOkvz9/bV8+XIVK1ZMHh4uLwcAAACABWWKG7zWqlVLf//9t4YOHapWrVrp+PHjkqQff/xRO3fudHF1AAAAAO51mSIYrVy5UuXKldOGDRs0f/58+2p127Zt04gRI1xcHQAAAIB7XaYIRoMHD9aYMWP0888/y8vLy95eu3ZtrV+/3oWVAQAAALCCTBGMduzYoaZNm6ZoDw4O1smTJ11QEQAAAAAryRTBKEeOHIqJiUnRvmXLFhUsWNAFFQEAAACwkkwRjJ599lkNGjRIx44dk81mU3JystauXasBAwaoXbt2ri4PAAAAwD0uUwSjN954Q6VLl1bhwoUVHx+v8PBwPfzww4qIiNDQoUNdXR4AAACAe5zNGGNcXcQ1hw8f1o4dOxQfH69KlSqpZMmSaerHt1LvdK4MyHp+WzjW1SUALle6QHZXlwAAcDGfVN4q1WV3VO3fv/8tt1+/Gt3EiRMzuhwAAAAAFuayYLRly5ZU7Wez2TK4EgAAAABW57JgtHz5clc9NQAAAAA4yBSLLwAAAACAKxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFiezRhjXF1Eevt66z+uLgFwufZD57u6BMDlTn/b29UlAABczMcjdfsxYgQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8jJNMFq9erXatm2r6tWr6+jRo5KkmTNnas2aNS6uDAAAAMC9LlMEo3nz5qlevXry9fXVli1bdOnSJUnS2bNn9cYbb7i4OgAAAAD3ukwRjMaMGaOPP/5Yn376qTw9Pe3tNWrU0ObNm11YGQAAAAAryBTBKCoqSjVr1kzRHhgYqDNnztz9ggAAAABYSqYIRvny5dO+fftStK9Zs0bFixd3QUUAAAAArCRTBKOuXbuqb9++2rBhg2w2m/755x9FRkZqwIABev75511dHgAAAIB7nIerC5CkwYMHKzk5WXXq1NGFCxdUs2ZNeXt7a8CAAXrhhRdcXR4AAACAe5zNGGNcXcQ1iYmJ2rdvn+Lj4xUeHi5/f/809fP11n/SuTIg62k/dL6rSwBc7vS3vV1dAgDAxXxSORSUKabSderUSefOnZOXl5fCw8NVtWpV+fv76/z58+rUqZOrywMAAABwj8sUwWjGjBlKSEhI0Z6QkKAvvvjCBRUBAAAAsBKXXmMUFxcnY4yMMTp37px8fHzs25KSkvTDDz8oODjYhRUCAAAAsAKXBqMcOXLIZrPJZrOpVKlSKbbbbDaNGjXKBZUBAAAAsBKXBqPly5fLGKPatWtr3rx5ypkzp32bl5eXihYtqgIFCriwQgAAAABW4NJgVKtWLUlSdHS0ChcuLDe3THHJEwAAAACLyRT3MSpatKjOnDmjzz//XLt375YklS1bVp06dVJgYKCLqwMAAABwr8sUQzS///67QkND9c477+jUqVM6deqUJk6cqNDQUG3evNnV5QEAAAC4x2WKEaN+/fqpUaNG+vTTT+XhcbWkK1euqEuXLnrxxRe1atUqF1cIAAAA4F6WKYLR77//7hCKJMnDw0Mvv/yyqlSp4sLKAAAAAFhBpphKFxAQoEOHDqVoP3z4sLJnz+6CigAAAABYSaYIRi1btlTnzp311Vdf6fDhwzp8+LDmzJmjLl26qFWrVq4uDwAAAMA9LlNMpRs/frxsNpvatWunK1euSJI8PT31/PPPa9y4cS6uDgAAAMC9zmaMMa4u4poLFy5o//79kqTQ0FD5+fmlqZ+vt/6TnmUBWVL7ofNdXQLgcqe/7e3qEgAALuaTyqGgTDFi9OWXX6pZs2by8/NTuXLlXF0ObuLgrm1a8/1X+if6L507HatWA15T+AMP2bcbY/TrN9P0+7LFung+XkXC7lOjLv2UK38hSdLp48e0Yv4XOvDnFsWfOaXsOXOrwkN1VatZW3l4eLrqZQGpNqD5/WpSvbhKFQpSQuIVbdh9TK9OX6e9R8847Pdg6Xwa+Vw1PRCWV0nJRtsPnNBTw7/TxcQkSVLF0Dwa06G67i95dfu36/Zr0GdrdP7iZRe8KiBjzJkVqRnTPtfJkydUKqy0Br8yTOXKl3d1WcBdxXmQtWSKa4z69eun4OBgtW7dWj/88IOSkpJcXRJuIPHSReUrGqonO/W94fbV383R+h/nq1GXfur++ofy8vHRjDde1uXEREnSyX8OySQbNe7aXy9MmKYn2vXUpl++1y+zP7ubLwNIs4fvK6CPF+9QrQFz9eSwhfLwcNOi1xrJz/t/f2N6sHQ+LRz1lJZtOaSH+3+jh/p9rY8X7VBy8tXB+fw5s2nxmMbaH3NWNV/6Ro1HfKfwIjn1ab86rnpZQLpb8uMPGv/WWHXv2UtzvlmgsLDSer57Z8XGxrq6NOCu4TzIejJFMIqJidGcOXNks9nUokUL5c+fX7169dK6detcXRquU6rSg6r7bGeFV304xTZjjH77Ya5qNXtOZR54SPmKhurpXkN07vRJ7d60RpJUsmJVNes5SCUqPKCceQuoTJUaeujJFtq1cfXdfilAmjQe8b2+XLZHuw+d0o7oWHV75xcVCQ5QpRLB9n3e6vKQPvx+u8bP3azdh05p79EzmrdmnxKvJEuSnnggRJevJOvFj1Zq79Ez+mPvcb0weYWa1iih4vkDXfXSgHQ1c8Y0NXumhZo0fVqhJUpo6IhR8vHx0bfz57m6NOCu4TzIejJFMPLw8NCTTz6pyMhIHT9+XO+8844OHjyoRx99VKGhoa4uD6lw+niM4s+cUmi5++1tPn7+KlSijA7v3XnT4y5eOC9ff5ZkR9YUkM1bknQ6/qIkKU+gr6qWzqcTZxK0/O2ndXBmJy0d21QR4fntx3h7uuvylSRdf3VnQuLVRWeu3w/Iqi4nJmr3rp2qVj3C3ubm5qZq1SK0fdsWF1YG3D2cB1lTpghG1/Pz81O9evX0xBNPqGTJkjp48KCrS0IqxJ85JUnyDwxyaM8WGGTf9l+xx45q/ZIFeqDuUxleH5DebDbp7a4Pa93Of7Tr76vv8WL5AiRJr7auqqk/7VLjEd9p6/4T+uH1JgotcHU0aMX2I8ob5Kd+zSrJ08NNObJ5a0yH6pKkfDmzuebFAOno9JnTSkpKUq5cuRzac+XKpZMnT7qoKuDu4jzImjJNMLpw4YIiIyPVoEEDFSxYUJMmTVLTpk21c+fNRxsk6dKlS4qLi3N4XE68dJeqRlrFnTqhL954WfdVq6UqdZ50dTmA0yY9X0tli+ZUu7d+sre52WySpM+X/KmZv+zWtgMn9fJna/TXkdNq/1i4JGn3oVPq+s4y9WlaUafm9dDBLzvp4LE4HTt9XiY50ywSCgCA5WSKVemeffZZLVq0SH5+fmrRooWGDRum6tWrp+rYsWPHatSoUQ5tz3Tvr+Y9XsqIUnET/jlySpLiz55W9qD//XXk/NnTyhdSwmHfuFMnNXV0fxUuVVaNuvHvhKznnR411eCBENUdPF9HY8/b22NOX/169yHHUdKow6dVOI+//fuvVv6lr1b+peAcvjp/8YqMMerTpKKij529Oy8AyEBBOYLk7u6e4gLz2NhY5c6d20VVAXcX50HWlClGjNzd3fX1118rJiZGH3zwQapDkSQNGTJEZ8+edXg06cR9K+62oOD88s+RUwd2bLa3XbxwXkf27VbhkmXtbXGnTmjq6H4qUKyUmvUcJDe3TPEWBFLtnR411ah6cdV/9Vv9/e85h21//3tO/8TGq1QhxymlJQrm0KHjjvtK0vEzCTp/8bKeqVlSFy8nadnWwxlaO3A3eHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8yJoyxYhRvXr1VKdOHbm7uzu0JyYmas6cOWrXrt1Nj/X29pa3t7dDm6dXfIbUaXWXLibo1LGj9u/PHI9RzMF98vXPrhy586p6g2e0YsFM5cxfUEHB+bXsq6nKHpRbZf7/Xkdxp07o81H9lCN3XtV/rofOx/3vr+PZ/3/ECcjMJj1fSy1rlVLzMYsVf+Gy8ua4ehPqsxcu2e9R9M68LRrapqp2RJ/UtgMn1bZOaYUVClLrsT/a++nxZDmt331M8QmXVadSYb3RMULDZvyms+cTXfK6gPT2XPuOGvbKIJUte5/uK1deX86coYSEBDVp2szVpQF3DedB1mMzxrh8Uru7u7tiYmIUHBzs0B4bG6vg4GCn72v09dZ/0rM8/L/onVs1dXS/FO2VatVTs56D/3eD118W6eKFeBUJK6enOr+o3AUKS5I2r1iiBR+9ecO+X/tqeYbWbkXth853dQn3nIRFNx6N7vrOL/py2R779wOeqazuDcspKLuPdkSf1KvT1mndrhj79s/611X9KiHy9/VU1JHTmjR/i2Yvj8rw+q3o9LfMIHCV2ZFf2m9sGVa6jAa9MlTly1dwdVnAXcV5kDn4pHIoKE3BaPXq1ZoyZYr279+vuXPnqmDBgpo5c6aKFSumhx56yNnu5Obmpn///Vd58uRxaN+2bZseffRRnTp141XNboZgBBCMAIlgBABIfTByeirdvHnz9Nxzz6lNmzbasmWLLl26ugLc2bNn9cYbb+iHH35IdV+VKlWSzWaTzWZTnTp15OHxv3KSkpIUHR2t+vXrO1siAAAAADjF6WA0ZswYffzxx2rXrp3mzJljb69Ro4bGjBnjVF9NmjSRJG3dulX16tWTv///Vm3y8vJSSEiInn76aWdLBAAAAACnOB2MoqKiVLNmzRTtgYGBOnPmjFN9jRgxQpIUEhKili1bysfHx9lyAAAAAOCOOR2M8uXLp3379ikkJMShfc2aNSpevHiaimjfvr2kq6vQHT9+XMnJyQ7bixQpkqZ+AQAAACA1nA5GXbt2Vd++fTV16lTZbDb9888/+u233zRgwAANGzYsTUXs3btXnTp10rp16xzajTGy2WxOr0oHAAAAAM5wOhgNHjxYycnJqlOnji5cuKCaNWvK29tbAwYM0AsvvJCmIjp06CAPDw8tWrRI+fPnl81mS1M/AAAAAJAWab6PUWJiovbt26f4+HiFh4c7LJzgrGzZsumPP/5Q6dKl09zH9ViuG2C5bkBiuW4AQOqX63ZztuNOnTrp3Llz8vLyUnh4uKpWrSp/f3+dP39enTp1crY7SVJ4eLhOnjyZpmMBAAAA4E45HYxmzJihhISEFO0JCQn64osvUt1PXFyc/fHmm2/q5Zdf1ooVKxQbG+uwLS4uztkSAQAAAMApqb7GKC4uTsYYGWN07tw5h6W1k5KS9MMPPyg4ODjVT5wjRw6Ha4mMMapTp47DPiy+AAAAAOBuSHUwuhZkbDabSpUqlWK7zWbTqFGjUv3Ey5cvT/W+AAAAAJCRUh2Mli9fLmOMateurXnz5ilnzpz2bV5eXipatKgKFCiQ6ieuVauWc5UCAAAAQAZJdTC6FmSio6NVuHBhubk5fXnSTW3fvv2G7TabTT4+PipSpIi8vb3T7fkAAAAA4HpO38eoaNGiOnPmjD7//HPt3r1bklS2bFl16tRJgYGBaSqiYsWKt7x3kaenp1q2bKkpU6Y4XNsEAAAAAOnB6WGf33//XaGhoXrnnXd06tQpnTp1ShMnTlRoaKg2b96cpiIWLFigkiVL6pNPPtHWrVu1detWffLJJwoLC9OsWbP0+eef69dff9XQoUPT1D8AAAAA3IrTN3h9+OGHVaJECX366afy8Lg64HTlyhV16dJFBw4c0KpVq5wuomrVqnrttddUr149h/affvpJw4YN08aNG/Xtt9/qpZde0v79+2/bHzd4BbjBKyBxg1cAQOpv8Or0VLrff//dIRRJkoeHh15++WVVqVLF2e4kSTt27FDRokVTtBctWlQ7duyQdHW6XUxMTJr6BwAAAIBbcXoqXUBAgA4dOpSi/fDhw8qePXuaiihdurTGjRunxMREe9vly5c1btw4lS5dWpJ09OhR5c2bN039AwAAAMCtOD1i1LJlS3Xu3Fnjx49XRESEJGnt2rUaOHCgWrVqlaYiJk+erEaNGqlQoUIqX768pKujSElJSVq0aJEk6cCBA+rZs2ea+gcAAACAW3E6GI0fP142m03t2rXTlStXJF1dNe7555/XuHHj0lRERESEoqOjFRkZqb/++kuS1Lx5c7Vu3do+CvXcc8+lqW8AAAAAuJ1UL75QpUoVdenSRa1bt1ZAQIAuXLhgXwghNDRUfn5+GVqoM1h8AWDxBUBi8QUAQAYsvlChQgW9/PLLeumll/T000+rU6dOeuSRR9JYnvTdd9/piSeekKenp7777rtb7tuoUaM0Pw8AAAAA3I5Ty3VfuHBBX3/9taZPn67Vq1erWLFi6tSpk9q3b6+CBQs69cRubm46duyYgoOD5eZ28zUgbDabkpKSnOqbESOAESNAYsQIAJD6ESOnVqXz8/NThw4dtGLFCv3111969tlnNWXKFIWEhKhhw4aaPz/1H8SSk5MVHBysy5cv65FHHtGePXuUnJyc4uFsKAIAAAAAZzm9XPc1oaGhGjNmjA4ePKjZs2dr/fr1at68udP9eHp6aseOHbccNQIAAACAjHRHaWTFihXq0KGDOnTooKSkJHXt2jVN/bRt21afffbZnZQCAAAAAGnm9HLdR44c0fTp0zV9+nQdOHBADz/8sD788EM1b95cvr6+aSriypUrmjp1qn755Rfdf//9ypYtm8P2iRMnpqlfAAAAAEiNVAejr7/+WlOnTtWyZcsUHBys9u3bq1OnTipRosQdF/Hnn3+qcuXKkmS/j9E1NpvtjvsHAAAAgFtJdTBq27atGjZsqAULFqhBgwbpek3Q8uXL060vAAAAAHBWqoPRkSNHFBwcnJG1AAAAAIBLpHrYh1AEAAAA4F7FGtkAAAAALI9gBAAAAMDyCEYAAAAALM/p+xhdk5iYqOPHjys5OdmhvUiRIndcFAAAAADcTU4Ho71796pTp05at26dQ7sxRjabTUlJSelWHAAAAADcDU4How4dOsjDw0OLFi1S/vz5uQErAAAAgCzP6WC0detW/fHHHypdunRG1AMAAAAAd53Tiy+Eh4fr5MmTGVELAAAAALiE08HozTff1Msvv6wVK1YoNjZWcXFxDg8AAAAAyGqcnkpXt25dSVKdOnUc2ll8AQAAAEBW5XQwWr58eUbUAQAAAAAu43QwqlWrVkbUAQAAAAAuk+YbvF64cEGHDh1SYmKiQ3v58uXvuCgAAAAAuJucDkYnTpxQx44d9eOPP95wO9cYAQAAAMhqnF6V7sUXX9SZM2e0YcMG+fr6asmSJZoxY4ZKliyp7777LiNqBAAAAIAM5fSI0a+//qqFCxeqSpUqcnNzU9GiRfXYY48pICBAY8eOVcOGDTOiTgAAAADIME6PGJ0/f17BwcGSpKCgIJ04cUKSVK5cOW3evDl9qwMAAACAu8DpYBQWFqaoqChJUoUKFTRlyhQdPXpUH3/8sfLnz5/uBQIAAABARnN6Kl3fvn0VExMjSRoxYoTq16+vyMhIeXl5afr06eldHwAAAABkOKeDUdu2be1f33///fr777+1Z88eFSlSRLlz507X4gAAAADgbkjzfYyu8fPzU+XKldOjFgAAAABwCaeDkTFGc+fO1fLly3X8+HElJyc7bJ8/f366FQcAAAAAd4PTwejFF1/UlClT9Oijjypv3ryy2WwZURcAAAAA3DVOB6OZM2dq/vz5atCgQUbUAwAAAAB3ndPLdQcGBqp48eIZUQsAAAAAuITTwWjkyJEaNWqUEhISMqIeAAAAALjrnJ5K16JFC82ePVvBwcEKCQmRp6enw/bNmzenW3EAAAAAcDc4HYzat2+vP/74Q23btmXxBQAAAAD3BKeD0eLFi/XTTz/poYceyoh6AAAAAOCuc/oao8KFCysgICAjagEAAAAAl3A6GE2YMEEvv/yyDh48mAHlAAAAAMDdZzPGGGcOCAoK0oULF3TlyhX5+fmlWHzh1KlT6VpgWly84uoKAAAAAGQGPqm8eMjpa4wmTZrk7CEAAAAAkKk5PWKUFTBiBAAAAEDKwBGjQ4cO3XJ7kSJFnO0SAAAAAFzK6REjNze3W967KCkp6Y6LulOMGAEAAACQMnDEaMuWLQ7fX758WVu2bNHEiRP1+uuvO9sdAAAAALhcul1jtHjxYr399ttasWJFenR3RxgxAgAAACClfsTI6fsY3UxYWJg2bdqUXt0BAAAAwF3j9FS6uLg4h++NMYqJidHIkSNVsmTJdCsMAAAAAO4Wp4NRjhw5Uiy+YIxR4cKFNWfOnHQrDAAAAADuFqeD0fLlyx2+d3NzU548eVSiRAl5eDjdHQAAAAC4HDd4BQAAAHDPSvflur/77rtU7deoUaPUdgkAAAAAmUKqR4zc3BwXsLPZbPrvoTabjRu8AgAAAMg00n257uTkZIeHn5+f9u3b59CWGUIRAAAAADgr3e5jBAAAAABZFcEIAAAAgOURjAAAAABYXpqDkc1mS3GjVwAAAADIilK9Kl1QUJBDEDpz5owCAgJSrFZ36tSp9K0wDViVDgAAAICUAfcxmjRpUhpLAQAAAIDMLdUjRlkJI0YAAAAApAy4jxEAAAAA3KsIRgAAAAAsj2AEAAAAwPIIRgAAAAAsj2AEAAAAwPJSvVz39Y4cOaLvvvtOhw4dUmJiosO2iRMnpkthAAAAAHC3OB2Mli1bpkaNGql48eLas2eP7rvvPh08eFDGGFWuXDkjagQAAACADOX0VLohQ4ZowIAB2rFjh3x8fDRv3jwdPnxYtWrVUvPmzTOiRgAAAADIUE7f4DV79uzaunWrQkNDFRQUpDVr1qhs2bLatm2bGjdurIMHD2ZQqanHDV4BAAAASBl4g9ds2bLZryvKnz+/9u/fb9928uRJZ7sDAAAAAJdz+hqjatWqac2aNSpTpowaNGigl156STt27ND8+fNVrVq1jKgRAAAAADKU01PpDhw4oPj4eJUvX17nz5/XSy+9pHXr1qlkyZKaOHGiihYtmlG1phpT6QAAAABIqZ9K53QwygoIRgAAAACk1AejNN3HSJISExN1/PhxJScnO7QXKVIkrV0CAAAAgEs4HYz++usvde7cWevWrXNoN8bIZrMpKSkp3YoDAAAAgLvB6WDUsWNHeXh4aNGiRcqfP79sNltG1AUAAAAAd43T1xhly5ZNf/zxh0qXLp1RNd0xrjECAAAAIGXgfYzCw8O5XxEAAACAe4rTwejNN9/Uyy+/rBUrVig2NlZxcXEODwAAAADIapyeSufmdjVL/ffaosy0+AJT6QAAAABIGbhc9/Lly509BAAAAAAyNW7wCgAAAOCelWGLL0jS6tWr1bZtW0VEROjo0aOSpJkzZ2rNmjVp6Q4AAAAAXMrpYDRv3jzVq1dPvr6+2rx5sy5duiRJOnv2rN544410LxAAAAAAMprTwWjMmDH6+OOP9emnn8rT09PeXqNGDW3evDldiwMAAACAu8HpYBQVFaWaNWumaA8MDNSZM2fSVMTMmTNVo0YNFShQQH///bckadKkSVq4cGGa+gMAAAAAZzgdjPLly6d9+/alaF+zZo2KFy/udAEfffSR+vfvrwYNGujMmTP25b5z5MihSZMmOd0fAAAAADjL6WDUtWtX9e3bVxs2bJDNZtM///yjyMhIDRgwQM8//7zTBbz//vv69NNP9eqrr8rd3d3eXqVKFe3YscPp/gAAAADAWU7fx2jw4MFKTk5WnTp1dOHCBdWsWVPe3t4aMGCAXnjhBacLiI6OVqVKlVK0e3t76/z58073BwAAAADOcioYJSUlae3aterVq5cGDhyoffv2KT4+XuHh4fL3909TAcWKFdPWrVtVtGhRh/YlS5aoTJkyaeoTAAAAAJzhVDByd3fX448/rt27dytHjhwKDw+/4wL69++vXr166eLFizLGaOPGjZo9e7bGjh2rzz777I77BwAAAIDbcXoq3X333acDBw6oWLFi6VJAly5d5Ovrq6FDh+rChQtq3bq1ChQooHfffVfPPvtsujwHAAAAANyKzRhjnDlgyZIlGjJkiF577TXdf//9ypYtm8P2gICANBdz4cIFxcfHKzg4OM19SNLFK3d0OAAAAIB7hE8qh4KcDkZubv9byM5ms9m/NsbIZrPZl9tOrdq1a2v+/PnKkSOHQ3tcXJyaNGmiX3/91an+JIIRAAAAgKtSG4ycnkq3fPlyZw+5pRUrVigxMTFF+8WLF7V69ep0fS4AAAAAuBGng1GtWrVuuu3PP/9MdT/bt2+3f71r1y4dO3bM/n1SUpKWLFmiggULOlseAAAAADjN6al0/3Xu3DnNnj1bn332mf74449UT6Vzc3OzT8W7UQm+vr56//331alTJ6drYiodAAAAACkDp9Jds2rVKn3++eeaN2+eChQooGbNmmny5MmpPj46OlrGGBUvXlwbN25Unjx57Nu8vLwUHBwsd3f3tJYHAAAAAKnmVDA6duyYpk+frs8//1xxcXFq0aKFLl26pG+//dbpexpdu6FrcnKyU8cBAAAAQHpzu/0uVz311FMKCwvT9u3bNWnSJP3zzz96//3306WI/fv364UXXlDdunVVt25d9enTR/v370+XvgEAAADgdlIdjH788Ud17txZo0aNUsOGDdNtmttPP/2k8PBwbdy4UeXLl1f58uW1YcMGlS1bVj///HO6PAfurjmzIvXEY7X1QKVyavNsc+24bqENwCo4D2B1nAMA50FWk+pgtGbNGp07d07333+/HnzwQX3wwQc6efLkHRcwePBg9evXTxs2bNDEiRM1ceJEbdiwQS+++KIGDRp0x/3j7lry4w8a/9ZYde/ZS3O+WaCwsNJ6vntnxcbGuro04K7hPIDVcQ4AnAdZkdOr0p0/f15fffWVpk6dqo0bNyopKUkTJ05Up06dlD17dqcL8PHx0Y4dO1SyZEmH9r/++kvly5fXxYsXne6TVelcp82zzVX2vnJ6ZehwSVevIXu8Ti21av2cOnft5uLqgLuD8wBWxzkAcB5kJqldlS7VI0bXZMuWTZ06ddKaNWu0Y8cOvfTSSxo3bpyCg4PVqFEjZ7tTnjx5tHXr1hTtW7duVXBwsNP9wXUuJyZq966dqlY9wt7m5uamatUitH3bFhdWBtw9nAewOs4BgPMgq3I6GF0vLCxMb731lo4cOaLZs2enqY+uXbuqW7duevPNN7V69WqtXr1a48aNU/fu3dW1a9fbHn/p0iXFxcU5PC5dupSmWnBnTp85raSkJOXKlcuhPVeuXOky7RLICjgPYHWcAwDnQVaV5vsYXc/d3V1NmjRRkyZNnD522LBhyp49uyZMmKAhQ4ZIkgoUKKCRI0eqT58+tz1+7NixGjVqlEPbq8NGaOjwkU7XAgAAAMCa0iUY3QmbzaZ+/fqpX79+OnfunCQ5da3SkCFD1L9/f4c24+6drjUidYJyBMnd3T3FRYWxsbHKnTu3i6oC7i7OA1gd5wDAeZBV3dFUuvQwZswYRUdHS7oaiJxdwMHb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1mVy4PRN998oxIlSigiIkIffvgh8y6zuOfad9T8uV/ru28X6MD+/RozeqQSEhLUpGkzV5cG3DWcB7A6zgGA8yArcnq57oywc+dORUZGas6cOTpy5Igee+wxtWnTRk2aNJGfn5/T/bFct2vNjvxSM6Z9rpMnTyisdBkNemWoypev4OqygLuK8wBWxzkAcB5kFqldrjtTBKPrrV27VrNmzdI333yjixcvKi4uzuk+CEYAAAAApAy8j1FGy5Ytm3x9feXl5aXLly+7uhwAAAAAFpApglF0dLRef/11lS1bVlWqVNGWLVs0atQoHTt2zNWlAQAAALAAly/XXa1aNW3atEnly5dXx44d1apVKxUsWNDVZQEAAACwEJcHozp16mjq1KkKDw93dSkAAAAALCrTLL6QmJio6OhohYaGysPjzvIaiy8AAAAAkLLQ4gsJCQnq3Lmz/Pz8VLZsWR06dEiS9MILL2jcuHEurg4AAACAFbg8GA0ePFjbtm3TihUr5OPjY2+vW7euvvrqKxdWBgAAAMAqXH6N0bfffquvvvpK1apVk81ms7eXLVtW+/fvd2FlAAAAAKzC5SNGJ06cUHBwcIr28+fPOwQlAAAAAMgoLg9GVapU0eLFi+3fXwtDn332mapXr+6qsgAAAABYiMun0r3xxht64okntGvXLl25ckXvvvuudu3apXXr1mnlypWuLg8AAACABbh8xOihhx7S1q1bdeXKFZUrV05Lly5VcHCwfvvtN91///2uLg8AAACABWSa+xilJ+5jBAAAAEBK/X2MXDKVLi4uTgEBAfavb+XafgAAAACQUVwyYuTu7q6YmBgFBwfLzc3thqvPGWNks9mUlJTkdP+MGAEAAACQMvmI0a+//qqcOXNKkqZNm6bChQvL3d3dYZ/k5GQdOnTIFeUBAAAAsBiXX2N0/ejR9WJjYxUcHMyIEQAAAIA0S+2IkctXpbs2Ze6/4uPj5ePj44KKAAAAAFiNy+5j1L9/f0lXb+g6bNgw+fn52bclJSVpw4YNqlixoouqAwAAAGAlLgtGW7ZskXR1xGjHjh3y8vKyb/Py8lKFChU0YMAAV5UHAAAAwEJcfo1Rx44d9e6776brstxcYwQAAABASv01Ri4PRhmBYAQAAABAykKLLwAAAACAqxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5WW6YHTx4kVXlwAAAADAYjJFMEpOTtZrr72mggULyt/fXwcOHJAkDRs2TJ9//rmLqwMAAABwr8sUwWjMmDGaPn263nrrLXl5ednb77vvPn322WcurAwAAACAFWSKYPTFF1/ok08+UZs2beTu7m5vr1Chgvbs2ePCygAAAABYQaYIRkePHlWJEiVStCcnJ+vy5csuqAgAAACAlWSKYBQeHq7Vq1enaJ87d64qVarkgooAAAAAWImHqwuQpOHDh6t9+/Y6evSokpOTNX/+fEVFRemLL77QokWLXF0eAAAAgHuczRhjXF2EJK1evVqjR4/Wtm3bFB8fr8qVK2v48OF6/PHHne7r4pUMKBAAAABAluOTyqGgTBOM0hPBCAAAAICU+mCUKa4xKl68uGJjY1O0nzlzRsWLF3dBRQAAAACsJFMEo4MHDyopKSlF+6VLl3T06FEXVAQAAADASly6+MJ3331n//qnn35SYGCg/fukpCQtW7ZMISEhLqgMAAAAgJW49BojN7erA1Y2m03/LcPT01MhISGaMGGCnnzySaf65RojAAAAAFLqrzFy6YhRcnKyJKlYsWLatGmTcufO7cpyAAAAAFgUq9IBAAAAuGdlqVXpJGnlypV66qmnVKJECZUoUUKNGjXS6tWrXV0WAAAAAAvIFMHoyy+/VN26deXn56c+ffqoT58+8vX1VZ06dTRr1ixXlwcAAADgHpcpptKVKVNG3bp1U79+/RzaJ06cqE8//VS7d+92qj+m0gEAAACQUj+VLlMEI29vb+3cuVMlSpRwaN+3b5/uu+8+Xbx40an+CEYAAAAApCx2jVHhwoW1bNmyFO2//PKLChcu7IKKAAAAAFiJS5frvuall15Snz59tHXrVkVEREiS1q5dq+nTp+vdd991cXUAAAAA7nWZYiqdJC1YsEATJkywX09UpkwZDRw4UI0bN3a6L6bSAQAAAJCy2DVG6Y1gBAAAAEDKYtcYdenSRStWrHB1GQAAAAAsKlMEoxMnTqh+/foqXLiwBg4cqK1bt7q6JAAAAAAWkmmm0p0+fVrffPONZs2apdWrV6t06dJq06aNWrdurZCQEKf6YiodAAAAACmLX2N05MgRzZ49W1OnTtXevXt15YpzSYdgBAAAAEDKYtcYXe/y5cv6/ffftWHDBh08eFB58+Z1dUkAAAAA7nGZJhgtX75cXbt2Vd68edWhQwcFBARo0aJFOnLkiKtLAwAAAHCPyxRT6QoWLKhTp06pfv36atOmjZ566il5e3unuT+m0gEAAACQstg1Rp9++qmaN2+uHDly3HK/I0eOqECBAnJzu/VAF8EIAAAAgJTFglFqBQQEaOvWrSpevPgt9yMYAQAAAJCy8OILt5KFMhwAAACALCRLBSMAAAAAyAgEIwAAAACWRzACAAAAYHlZKhjZbDZXlwAAAADgHpSlghGLLwAAAADICFlque7Dhw+rQIECcnd3v+V+LNcNAAAAQMoC9zFq1qxZqvedP3++U30TjFxrzqxIzZj2uU6ePKFSYaU1+JVhKle+vKvLAu4qzgNYHecAwHmQWWT6+xgFBgbaHwEBAVq2bJl+//13+/Y//vhDy5YtU2BgoKtKRBos+fEHjX9rrLr37KU53yxQWFhpPd+9s2JjY11dGnDXcB7A6jgHAM6DrChTTKUbNGiQTp06pY8//tg+TS4pKUk9e/ZUQECA3n77baf6Y8TIddo821xl7yunV4YOlyQlJyfr8Tq11Kr1c+rctZuLqwPuDs4DWB3nAMB5kJlk+hGj602dOlUDBgxwuHbI3d1d/fv319SpU11YGZxxOTFRu3ftVLXqEfY2Nzc3VasWoe3btriwMuDu4TyA1XEOAJwHWVWmCEZXrlzRnj17UrTv2bNHycnJtzz20qVLiouLc3hcunQpo0rFLZw+c1pJSUnKlSuXQ3uuXLl08uRJF1UF3F2cB7A6zgGA8yCryhTBqGPHjurcubMmTpyoNWvWaM2aNZowYYK6dOmijh073vLYsWPHOlyvFBgYqLffHHuXKgcAAABwL0jljLuMNX78eOXLl08TJkxQTEyMJCl//vwaOHCgXnrppVseO2TIEPXv39+hzbh7Z1ituLmgHEFyd3dPcVFhbGyscufO7aKqgLuL8wBWxzkAcB5kVZlixMjNzU0vv/yyjh49qjNnzujMmTM6evSoXn755dves8jb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1lVphgxul5AQICrS8AdeK59Rw17ZZDKlr1P95Urry9nzlBCQoKaNE39fauArI7zAFbHOQBwHmRFmSYYzZ07V19//bUOHTqkxMREh22bN292UVVwVv0nGuj0qVP68IP3dPLkCYWVLqMPp3ymXAwbw0I4D2B1nAMA50FWlCnuY/Tee+/p1VdfVYcOHfTJJ5+oY8eO2r9/vzZt2qRevXrp9ddfd6o/7mMEAAAAQEr9fYwyRTAqXbq0RowYoVatWil79uzatm2bihcvruHDh+vUqVP64IMPnOqPYAQAAABAymI3eD106JAiIq7eAMvX11fnzp2TJD333HOaPXu2K0sDAAAAYAGZIhjly5dPp06dkiQVKVJE69evlyRFR0crEwxoAQAAALjHZYpgVLt2bX333XeSrt7stV+/fnrsscfUsmVLNW3a1MXVAQAAALjXZYprjJKTk5WcnCwPj6sTAOfMmaN169apZMmS6t69u7y8vJzqj2uMAAAAAEhZbPGF9EYwAgAAACClPhhlmvsYnT59Wp9//rl2794tSQoPD1fHjh2VM2dOF1cGAAAA4F6XKUaMVq1apUaNGikgIEBVqlSRJP3xxx86c+aMvv/+e9WsWdOp/hgxAgAAACBlsal05cqVU/Xq1fXRRx/J3d1dkpSUlKSePXtq3bp12rFjh1P9EYwAAAAASFksGPn6+mrr1q0KCwtzaI+KilLFihWVkJDgVH8EIwAAAABSFrvBa+XKle3XFl1v9+7dqlChggsqAgAAAGAlLlt8Yfv27fav+/Tpo759+2rfvn2qVq2aJGn9+vWaPHmyxo0b56oSAQAAAFiEy6bSubm5yWaz6XZPb7PZlJSU5FTfTKUDAAAAIGWB5bqjo6Nd9dQAAAAA4CBTLL6Q3hgxAgAAACBlgRGj/9q7d6+WL1+u48ePKzk52WHb8OHDXVQVAAAAACvIFCNGn376qZ5//nnlzp1b+fLlk81ms2+z2WzavHmzU/0xYgQAAABAymL3MSpatKh69uypQYMGpUt/BCMAAAAAUha7j9Hp06fVvHlzV5cBAAAAwKIyRTBq3ry5li5d6uoyAAAAAFiUy6bSvffee/avz58/r4kTJ6phw4YqV66cPD09Hfbt06ePU30zlQ4AAACAlAWuMSpWrFiq9rPZbDpw4IBTfROMAAAAAEhZIBhlJIIRAAAAACmLLb4wevRoXbhwIUV7QkKCRo8e7YKKAAAAAFhJphgxcnd3V0xMjIKDgx3aY2NjFRwcrKSkJKf6Y8QIAAAAgJTFRoyMMQ43db1m27ZtypkzpwsqAgAAAGAlqcxPGSMoKEg2m002m02lSpVyCEdJSUmKj49Xjx49XFghAAAAACtw6VS6GTNmyBijTp06adKkSQoMDLRv8/LyUkhIiKpXr+50v0ylAwAAACBlsVXpVq5cqYiIiBT3L0orghEAAAAAKYsFI+nq1Llvv/1Wu3fvliSVLVtWjRo1kru7u9N9EYwAAAAASFksGO3bt08NGjTQ0aNHFRYWJkmKiopS4cKFtXjxYoWGhjrVH8EIAAAAgJTFglGDBg1kjFFkZKR9FbrY2Fi1bdtWbm5uWrx4sVP9EYwAAAAASFksGGXLlk3r169XuXLlHNq3bdumGjVqKD4+3qn+CEYAAAAApCx2HyNvb2+dO3cuRXt8fLy8vLxcUBEAAAAAK8kUwejJJ59Ut27dtGHDBhljZIzR+vXr1aNHDzVq1MjV5QEAAAC4x2WKqXRnzpxR+/bt9f3339uX7L58+bIaN26s6dOnO9zfKDWYSgcAAABAymLXGF2zb98+7dq1S5IUHh6uEiVKpKkfghEAAAAAKfXBKJW7ZbzPP/9c77zzjvbu3StJKlmypF588UV16dLFxZUBAAAAuNdlimA0fPhwTZw4US+88IKqV68uSfrtt9/Ur18/HTp0SKNHj3ZxhQAAAADuZZliKl2ePHn03nvvqVWrVg7ts2fP1gsvvKCTJ0861R9T6QAAAABIWWy57suXL6tKlSop2u+//35duULKAQAAAJCxMkUweu655/TRRx+laP/kk0/Upk0bF1QEAAAAwEpcdo1R//797V/bbDZ99tlnWrp0qapVqyZJ2rBhgw4dOqR27dq5qkQAAAAAFuGya4weffTRVO1ns9n066+/OtU31xgBAAAAkLLofYzSC8EIAAAAgJTFFl8AAAAAAFciGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMuzGWOMq4vAveXSpUsaO3ashgwZIm9vb1eXA7gE5wHAeQBInAdZCcEI6S4uLk6BgYE6e/asAgICXF0O4BKcBwDnASBxHmQlTKUDAAAAYHkEIwAAAACWRzACAAAAYHkEI6Q7b29vjRgxggsMYWmcBwDnASBxHmQlLL4AAAAAwPIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjC3rkkUf04osvpnr/b7/9ViVKlJC7u7tTxwGZmc1m07fffpvq/VesWCGbzaYzZ85kWE1AZsH7HZmRs59fUiskJESTJk266faDBw/KZrNp69at6f7cHTp0UJMmTdK9X6QNwQi31b17dz3zzDM6fPiwXnvttQw7iadPn64cOXKke7/AjcTExOiJJ55I1z5HjhypihUrpmufEh9SkfH++4EzIiJCMTExCgwMvGs1ZOSHT0Dic4bk/B8FrcbD1QUgc4uPj9fx48dVr149FShQwNXlAOkiMTFR+fLlc3UZQKbl5eXFOQLAchgxsrhLly5pwIABKliwoLJly6YHH3xQK1askHT1r9TZs2eXJNWuXVs2m02PPPKIZsyYoYULF8pms8lms2nFihWKiIjQoEGDHPo+ceKEPD09tWrVKknS6dOn1a5dOwUFBcnPz09PPPGE9u7da3+ujh076uzZs/Z+R44cedsagdR45JFH1Lt3b7344ovKnTu36tWrl+KvZuvWrVPFihXl4+OjKlWq6Ntvv73hX6//+OMPValSRX5+foqIiFBUVJSkq3+JHDVqlLZt22Z/D0+fPl2tW7dWy5YtHfq4fPmycufOrS+++ELS1fd4nz59FBwcLB8fHz300EPatGmTpKt/RX/00UclSUFBQbLZbOrQoYMkKTk5WWPHjlWxYsXk6+urChUqaO7cuRnwE8S9rEOHDlq5cqXeffddh/fuf0cp16xZo4cffli+vr4qXLiw+vTpo/Pnz0uSXnnlFT344IMp+q5QoYJGjx4t6er7dfTo0SpUqJC8vb1VsWJFLVmyxL5vsWLFJEmVKlWy/39zzWeffaYyZcrIx8dHpUuX1ocffpgBPwlkBVeuXFHv3r0VGBio3Llza9iwYbp255m0fs6QpAsXLqhTp07Knj27ihQpok8++STFc+/Zs0cRERHy8fHRfffdp5UrV9q3JSUlqXPnzvbfx2FhYXr33Xcdjk9KSlL//v2VI0cO5cqVSy+//LJSe9ecTz75RAUKFFBycrJDe+PGjdWpUyf79x999JFCQ0Pl5eWlsLAwzZw5074tJCREktS0aVPZbDb795K0cOFCVa5cWT4+PipevLhGjRqlK1eupKq2e4qB5dSqVcv07dvXGGNMly5dTEREhFm1apXZt2+fefvtt423t7f566+/zKVLl0xUVJSRZObNm2diYmLM2bNnTYsWLUz9+vVNTEyMiYmJMZcuXTIffPCBKVKkiElOTrY/z/vvv+/Q1qhRI1OmTBmzatUqs3XrVlOvXj1TokQJk5iYaC5dumQmTZpkAgIC7P2eO3futjUCqVGrVi3j7+9vBg4caPbs2WP27NljJJkFCxYYY4w5e/asyZkzp2nbtq3ZuXOn+eGHH0ypUqWMJLNlyxZjjDHLly83ksyDDz5oVqxYYXbu3GkefvhhExERYYwx5sKFC+all14yZcuWtb+HL1y4YBYtWmR8fX3t72djjPn++++Nr6+viYuLM8YY06dPH1OgQAHzww8/mJ07d5r27duboKAgExsba65cuWLmzZtnJJmoqCgTExNjzpw5Y4wxZsyYMaZ06dJmyZIlZv/+/WbatGnG29vbrFix4u79cJHlnTlzxlSvXt107drV/t795ZdfjCRz+vRpY4wx+/btM9myZTPvvPOO+euvv8zatWtNpUqVTIcOHYwxxvz5559Gktm3b5+932tte/fuNcYYM3HiRBMQEGBmz55t9uzZY15++WXj6elp/12+ceNGI8n88ssvJiYmxsTGxhpjjPnyyy9N/vz5zbx588yBAwfMvHnzTM6cOc306dPv4k8JmcG13+V9+/Y1e/bsMV9++aXx8/Mzn3zyiTEm7Z8zihYtanLmzGkmT55s9u7da8aOHWvc3NzMnj17jDHGREdHG0mmUKFCZu7cuWbXrl2mS5cuJnv27ObkyZPGGGMSExPN8OHDzaZNm8yBAwfstX311Vf2+t98800TFBRk5s2bZ3bt2mU6d+5ssmfPbho3bnzb137q1Cnj5eVlfvnlF3tbbGysQ9v8+fONp6enmTx5somKijITJkww7u7u5tdffzXGGHP8+HEjyUybNs3ExMSY48ePG2OMWbVqlQkICDDTp083+/fvN0uXLjUhISFm5MiRd/gvlvUQjCzoWjD6+++/jbu7uzl69KjD9jp16pghQ4YYY4w5ffq0kWSWL19u396+ffsUJ/Hx48eNh4eHWbVqlb2tevXqZtCgQcYYY/766y8jyaxdu9a+/eTJk8bX19d8/fXXxhhjpk2bZgIDAx36TU2NwO3UqlXLVKpUyaHt+mD00UcfmVy5cpmEhAT79k8//fSGwej6/5QWL15sJNmPGzFihKlQoYLD81y+fNnkzp3bfPHFF/a2Vq1amZYtWxpjjImPjzeenp4mMjLSvj0xMdEUKFDAvPXWWw7Pfe1DqjHGXLx40fj5+Zl169Y5PF/nzp1Nq1atnPjpAI5/MDMm5Xuuc+fOplu3bg7HrF692ri5udnf/xUqVDCjR4+2bx8yZIh58MEH7d8XKFDAvP766w59PPDAA6Znz57GmP99+Lx2zl0TGhpqZs2a5dD22muvmerVq6fptSLrqlWrlilTpozDH2EHDRpkypQpk+bPGcZcDUZt27a1f5+cnGyCg4PNRx99ZIz533tz3Lhx9n0uX75sChUqZN58882b1turVy/z9NNP27/Pnz+//ff69X2kJhgZY0zjxo1Np06d7N9PmTLFFChQwCQlJRljjImIiDBdu3Z1OKZ58+amQYMG9u+v/7/vmjp16pg33njDoW3mzJkmf/78qarrXsJUOgvbsWOHkpKSVKpUKfn7+9sfK1eu1P79+53qK0+ePHr88ccVGRkpSYqOjtZvv/2mNm3aSJJ2794tDw8Ph6kWuXLlUlhYmHbv3n1XaoS13X///TfdFhUVpfLly8vHx8feVrVq1RvuW758efvX+fPnlyQdP378pn17eHioRYsW9nPj/PnzWrhwof3c2L9/vy5fvqwaNWrYj/H09FTVqlVveW7s27dPFy5c0GOPPeZwbnzxxRecG0h327Zt0/Tp0x3ea/Xq1VNycrKio6MlSW3atNGsWbMkScYYzZ492/4+j4uL0z///OPwPpekGjVq3PJ9fv78ee3fv1+dO3d2eO4xY8bwPreoatWqyWaz2b+vXr269u7dq127dqXpc8Y11/9ut9lsypcvX4rf7dWrV7d/7eHhoSpVqjj0PXnyZN1///3KkyeP/P399cknn+jQoUOSpLNnzyomJsahvmt9pFabNm00b948Xbp0SZIUGRmpZ599Vm5uVz/O79692+lzTLp6fo8ePdrhHOvatatiYmJ04cKFVNd3L2DxBQuLj4+Xu7u7/vjjD7m7uzts8/f3d7q/Nm3aqE+fPnr//fc1a9YslStXTuXKlctUNcK6smXLli79eHp62r++9p/zf+d8/1ebNm1Uq1YtHT9+XD///LN8fX1Vv379O6ojPj5ekrR48WIVLFjQYZu3t/cd9Q38V3x8vLp3764+ffqk2FakSBFJUqtWrTRo0CBt3rxZCQkJOnz4cIrr69LyvJL06aefpriG6b//JwB34vrf7dLV3++3+91+vTlz5mjAgAGaMGGCqlevruzZs+vtt9/Whg0b0q3Gp556SsYYLV68WA888IBWr16td9555477jY+P16hRo9SsWbMU267/g6EVEIwsrFKlSkpKStLx48f18MMPp/o4Ly8vJSUlpWhv3LixunXrpiVLlmjWrFlq166dfVuZMmV05coVbdiwQREREZKk2NhYRUVFKTw8/Kb9prVGwBlhYWH68ssvdenSJXuouLb4gTNudm5ERESocOHC+uqrr/Tjjz+qefPm9v+Er10ku3btWhUtWlTS1cUZNm3aZF8+2cvLS5Ic+g4PD5e3t7cOHTqkWrVqOV0rcL2bvXevqVy5snbt2qUSJUrcdJ9ChQqpVq1aioyMVEJCgh577DEFBwdLkgICAlSgQAGtXbvW4f26du1a++jsjd7nefPmVYECBXTgwAH76BOs7b9BY/369SpZsqTCw8PT9DnDGevXr1fNmjUlXV0E4o8//lDv3r0lXX0vR0REqGfPnvb9rx/VDAwMVP78+bVhw4YUfVSuXDlVz+/j46NmzZopMjJS+/btU1hYmMOxZcqU0dq1a9W+fXt729q1a+2vX7oaAP/7M6hcubKioqJueX5bBcHIwkqVKqU2bdqoXbt2mjBhgipVqqQTJ05o2bJlKl++vBo2bHjD40JCQvTTTz8pKipKuXLlUmBgoDw9PZUtWzY1adJEw4YN0+7du9WqVSv7MSVLllTjxo3VtWtXTZkyRdmzZ9fgwYNVsGBBNW7c2N5vfHy8li1bpgoVKsjPzy/NNQLOaN26tV599VV169ZNgwcP1qFDhzR+/HhJcpiycTshISGKjo7W1q1bVahQIWXPnt0etFq3bq2PP/5Yf/31l5YvX24/Jlu2bHr++ec1cOBA5cyZU0WKFNFbb72lCxcuqHPnzpKkokWLymazadGiRWrQoIF8fX2VPXt2DRgwQP369VNycrIeeughnT17VmvXrlVAQIDDf4zA7YSEhGjDhg06ePCg/P39U/ylfNCgQapWrZp69+6tLl26KFu2bNq1a5d+/vlnffDBB/b92rRpoxEjRigxMTHFX7IHDhyoESNGKDQ0VBUrVtS0adO0detW+zTT4OBg+fr6asmSJSpUqJB8fHwUGBioUaNGqU+fPgoMDFT9+vV16dIl/f777zp9+rT69++f8T8cZCqHDh1S//791b17d23evFnvv/++JkyYkObPGX5+fql+7smTJ6tkyZIqU6aM3nnnHZ0+fdq+IlzJkiX1xRdf6KefflKxYsU0c+ZMbdq0yb7aoiT17dtX48aNU8mSJVW6dGlNnDjR6fvTtWnTRk8++aR27typtm3bOmwbOHCgWrRooUqVKqlu3br6/vvvNX/+fP3yyy/2fUJCQrRs2TLVqFFD3t7eCgoK0vDhw/Xkk0+qSJEieuaZZ+Tm5qZt27bpzz//1JgxY5yqL8tz9UVOuPuuv8j22ioqISEhxtPT0+TPn980bdrUbN++3Rhz48UXjh8/bh577DHj7++fYtsPP/xgJJmaNWumeN5Tp06Z5557zgQGBhpfX19Tr169FCvL9ejRw+TKlctIMiNGjEhVjcDt/PfCcmNSXoC6du1aU758eePl5WXuv/9+M2vWLCPJvirRjRZA2LJli5FkoqOjjTFXF0R4+umnTY4cOewr/1yza9cuI8kULVrU4cJhY4xJSEgwL7zwgsmdO7fx9vY2NWrUMBs3bnTYZ/To0SZfvnzGZrOZ9u3bG2OuXiA8adIkExYWZjw9PU2ePHlMvXr1zMqVK+/o5wXriYqKMtWqVTO+vr729+5/3+8bN260/+7Pli2bKV++fIrFFE6fPm28vb2Nn5+fw0qMxhiTlJRkRo4caQoWLGg8PT1NhQoVzI8//uiwz6effmoKFy5s3NzcTK1ateztkZGRpmLFisbLy8sEBQWZmjVrmvnz56f7zwGZW61atUzPnj1Njx49TEBAgAkKCjKvvPKK/XdqWj9nFC1a1LzzzjsO+1WoUMG+/driC7NmzTJVq1Y1Xl5eJjw83L7amzFXf/936NDBBAYGmhw5cpjnn3/eDB482GFBnsuXL5u+ffuagIAAkyNHDtO/f3/Trl27VC++YMzV8yh//vxGktm/f3+K7R9++KEpXry48fT0NKVKlXJY+McYY7777jtTokQJ4+HhYYoWLWpvX7JkiYmIiDC+vr4mICDAVK1a1b7an5XYjEnlAuoAYCGRkZH2e174+vq6uhwAAJDBmEoHAJK++OILFS9eXAULFtS2bds0aNAgtWjRglAEAIBFEIwAQNKxY8c0fPhwHTt2TPnz51fz5s31+uuvu7osAIAFHDp0yGGRhP/atWuXfQVIZBym0gEAAAAudOXKFR08ePCm20NCQuThwXhGRiMYAQAAALA8N1cXAAAAAACuRjACAAAAYHkEIwAAAACWRzACAAAAYHkEIwAAAACWRzACAKTZsWPH1LdvX5UoUUI+Pj7KmzevatSooY8++kgXLlyw7xcSEiKbzSabzaZs2bKpcuXK+uabb1Jsu9GjQ4cOt6xh7dq18vDwUMWKFe1tTz31lOrXr3/D/VevXi2bzabt27en2FauXDn16NHjhsfNnDlT3t7eOnny5G1+KpLNZtO333572/0AAJkHwQgAkCYHDhxQpUqVtHTpUr3xxhvasmWLfvvtN7388statGiRfvnlF4f9R48erZiYGG3ZskUPPPCAWrZsqXXr1mnTpk2KiYlRTEyM5s2bJ0mKioqyt7377rs3reHMmTNq166d6tSp49DeuXNn/fzzzzpy5EiKY6ZNm6YqVaqofPnyKbZ17txZc+bMUUJCwg2Pa9SokXLnzp2qnw8AIGshGAEA0qRnz57y8PDQ77//rhYtWqhMmTIqXry4GjdurMWLF+upp55y2D979uzKly+fSpUqpcmTJ8vX11fff/+98uTJo3z58ilfvnzKmTOnJCk4ONjeFhgYeNMaevToodatW6t69eoO7U8++aTy5Mmj6dOnO7THx8frm2++UefOnW/YX9u2bZWQkGAPaNdER0drxYoV9uM++ugjhYaGysvLS2FhYZo5c6Z935CQEElS06ZNZbPZ7N9L0sKFC1W5cmX5+PioePHiGjVqlK5cuXLT1wcAuHsIRgAAp8XGxmrp0qXq1auXsmXLdsN9bDbbTY/38PCQp6enEhMT01zDtGnTdODAAY0YMeKG/bdr107Tp0/X9fcx/+abb5SUlKRWrVrdsM/cuXOrcePGmjp1qkP79OnTVahQIT3++ONasGCB+vbtq5deekl//vmnunfvro4dO2r58uWSpE2bNtnri4mJsX+/evVqtWvXTn379tWuXbs0ZcoUTZ8+Xa+//nqafwYAgPRDMAIAOG3fvn0yxigsLMyhPXfu3PL395e/v78GDRp0w2MTExM1duxYnT17VrVr107T8+/du1eDBw/Wl19+KQ8Pjxvu06lTJ+3fv18rV660t02bNk1PP/30LUehOnfurBUrVig6OlqSZIzRjBkz1L59e7m5uWn8+PHq0KGDevbsqVKlSql///5q1qyZxo8fL0nKkyePJClHjhzKly+f/ftRo0Zp8ODBat++vYoXL67HHntMr732mqZMmZKmnwEAIH0RjAAA6Wbjxo3aunWrypYtq0uXLjlsGzRokPz9/eXn56c333xT48aNU8OGDW/b57Wg5e/vrx49eigpKUmtW7fWqFGjVKpUqZseV7p0aUVERNhHf/bt26fVq1ffdBrdNY899pgKFSqkadOmSZKWLVumQ4cOqWPHjpKk3bt3q0aNGg7H1KhRQ7t3775lv9u2bdPo0aMdXk/Xrl0VExPjsFAFAMA1bvxnNgAAbqFEiRKy2WyKiopyaC9evLgkydfXN8UxAwcOVIcOHeTv76+8efPecqrd9bZu3Wr/OiAgQOfOndPvv/+uLVu2qHfv3pKk5ORkGWPk4eGhpUuX2keiOnfurBdeeEGTJ0/WtGnTFBoaqlq1at3y+dzc3NShQwfNmDFDI0eO1LRp0/Too4/aX1taxcfHa9SoUWrWrFmKbT4+PnfUNwDgzjFiBABwWq5cufTYY4/pgw8+0Pnz51N1TO7cuVWiRAnly5cv1aFIuhrCrj2Cg4MVEBCgHTt2aOvWrfZHjx49FBYWpq1bt+rBBx+0H9uiRQu5ublp1qxZ+uKLL9SpU6dUPXfHjh11+PBhzZ8/XwsWLHAYZSpTpozWrl3rsP/atWsVHh5u/97T01NJSUkO+1SuXFlRUVEOr+faw82N/44BwNUYMQIApMmHH36oGjVqqEqVKho5cqTKly8vNzc3bdq0SXv27NH999+fIc/r5uam++67z6EtODhYPj4+Kdr9/f3VsmVLDRkyRHFxcbe9J9I1xYoVU+3atdWtWzd5e3s7jPIMHDhQLVq0UKVKlVS3bl19//33mj9/vsPy5CEhIVq2bJlq1Kghb29vBQUFafjw4XryySdVpEgRPfPMM3Jzc9O2bdv0559/asyYMWn/gQAA0gV/ogIApEloaKi2bNmiunXrasiQIapQoYKqVKmi999/XwMGDNBrr73m6hIlXZ1Od/r0adWrV08FChRw+rjWrVs7THVr0qSJ3n33XY0fP15ly5bVlClTNG3aND3yyCP2fSZMmKCff/5ZhQsXVqVKlSRJ9erV06JFi7R06VI98MADqlatmt555x0VLVo03V4rACDtbOb6dUwBAAAAwIIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgef8HWA2Jk61Jt7wAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 1000x700 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "import seaborn as sns\n",
-    "import matplotlib.pyplot as plt\n",
-    "from sklearn.metrics import confusion_matrix\n",
-    "import pandas as pd\n",
-    "\n",
-    "# Assuming df is your DataFrame\n",
-    "\n",
-    "# True labels and predictions\n",
-    "y_true = filtered_df[\"type\"]\n",
-    "y_pred = filtered_df[\"gpt_vote\"]\n",
-    "\n",
-    "# Compute the confusion matrix\n",
-    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
-    "\n",
-    "# Create a pandas DataFrame from the confusion matrix\n",
-    "conf_matrix_df = pd.DataFrame(conf_matrix, index=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"], columns=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
-    "\n",
-    "# Plotting the heatmap\n",
-    "plt.figure(figsize=(10, 7))\n",
-    "sns.heatmap(conf_matrix_df, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False)\n",
-    "plt.title(\"Arena Human vs GPT-4V Confusion Matrix\")\n",
-    "plt.xlabel(\"GPT-4V Vote\")\n",
-    "plt.ylabel(\"Arena Human Vote\")\n",
-    "plt.show()\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Accuracy: 0.5842911877394636\n",
-      "F1 Score (Macro): 0.514392348541452\n",
-      "F1 Score (Micro): 0.5842911877394636\n",
-      "F1 Score (Weighted): 0.5536668839130223\n"
-     ]
-    }
-   ],
-   "source": [
-    "from sklearn.metrics import accuracy_score, f1_score\n",
-    "\n",
-    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
-    "y_true = filtered_df['type']\n",
-    "y_pred = filtered_df['gpt_vote']\n",
-    "\n",
-    "# Calculate accuracy\n",
-    "accuracy = accuracy_score(y_true, y_pred)\n",
-    "print(f'Accuracy: {accuracy}')\n",
-    "\n",
-    "# Calculate F1 score, here using 'macro' average to treat all classes equally\n",
-    "f1 = f1_score(y_true, y_pred, average='macro')\n",
-    "print(f'F1 Score (Macro): {f1}')\n",
-    "\n",
-    "# If you want to calculate F1 score with other averages, for example 'micro' or 'weighted', you can do:\n",
-    "f1_micro = f1_score(y_true, y_pred, average='micro')\n",
-    "print(f'F1 Score (Micro): {f1_micro}')\n",
-    "\n",
-    "f1_weighted = f1_score(y_true, y_pred, average='weighted')\n",
-    "print(f'F1 Score (Weighted): {f1_weighted}')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Cohen's Kappa Score: 0.3442144615665177\n"
-     ]
-    }
-   ],
-   "source": [
-    "from sklearn.metrics import cohen_kappa_score\n",
-    "\n",
-    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
-    "y_true = filtered_df['type']\n",
-    "y_pred = filtered_df['gpt_vote']\n",
-    "\n",
-    "# Calculate Cohen's Kappa score\n",
-    "kappa = cohen_kappa_score(y_true, y_pred)\n",
-    "print(f'Cohen\\'s Kappa Score: {kappa}')\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 48,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Accuracy Score: 0.5842911877394636\n"
-     ]
-    }
-   ],
-   "source": [
-    "from sklearn.metrics import accuracy_score\n",
-    "accuracy = accuracy_score(y_true, y_pred)\n",
-    "print(f'Accuracy Score: {accuracy}')\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 49,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Pearson Correlation Coefficient: 0.2880096104357029\n"
-     ]
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "\n",
-    "# Assuming filtered_df is your DataFrame and it contains 'type' and 'gpt_vote' columns\n",
-    "# Convert 'type' and 'gpt_vote' to categorical codes\n",
-    "filtered_df['type_int'] = pd.factorize(filtered_df['type'])[0]\n",
-    "filtered_df['gpt_vote_int'] = pd.factorize(filtered_df['gpt_vote'])[0]\n",
-    "\n",
-    "# Now you can calculate Pearson correlation between these new integer columns\n",
-    "pearson_correlation = filtered_df['type_int'].corr(filtered_df['gpt_vote_int'])\n",
-    "print(f'Pearson Correlation Coefficient: {pearson_correlation}')\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "otask",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.13"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

arena_elo/get_latest_data.sh

@@ -1,17 +0,0 @@
-
-# set LOGDIR to default if not set before
-if [ -z "$LOGDIR" ]; then
-    export LOGDIR="./vision-arena-logs"
-fi
-mkdir -p results
-
-
-# # for battle data
-python -m elo_rating.clean_battle_data --model_infos_file "./model_infos.json" --mode conv_release
-battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Battle data last updated on $battle_cutoff_date"
-
-mkdir -p ./results/latest
-mkdir -p ./results/$battle_cutoff_date && mv ./clean_battle_conv_$battle_cutoff_date.json ./results/$battle_cutoff_date/clean_battle_conv.json
-cp ./results/$battle_cutoff_date/clean_battle_conv.json ./results/latest/clean_battle_conv.json
-
-echo "Battle data last updated on $battle_cutoff_date" >> ./results/latest/latest_updated_date.txt

arena_elo/pyproject.toml

@@ -1,28 +0,0 @@
-[build-system]
-requires = ["setuptools>=61.0"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "arena_elo"
-version = "0.2.35"
-description = "Elo rating system for WildVision Bench Arena"
-readme = "README.md"
-requires-python = ">=3.9"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: Apache Software License",
-]
-dependencies = [
-    "numpy", "prompt_toolkit>=3.0.0", "uvicorn","polyglot", "pyicu", "pycld2", "morfessor", "scikit-learn",
-    "pytz", "tqdm", "pandas", "plotly", "fire", "Pillow"
-]
-
-[project.urls]
-"Homepage" = "https://github.com/WildVision-Bench/Arena-Elo"
-"Bug Tracker" = "https://github.com/WildVision-Bench/Arena-Elo/issues"
-
-[tool.setuptools.packages.find]
-exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
-
-[tool.wheel]
-exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]

arena_elo/requirements.txt

@@ -1,28 +0,0 @@
--e git+https://github.com/WildVision-Bench/Arena-Elo.git@9dc2fa8543a2e9eda3d5bc01c2212fdfcdd4bfb5#egg=arena_elo
-click==8.1.7
-fire==0.5.0
-h11==0.14.0
-joblib==1.3.2
-Morfessor==2.0.6
-numpy==1.26.4
-packaging==23.2
-pandas==2.2.0
-pillow==10.2.0
-plotly==5.18.0
-polyglot==16.7.4
-prompt-toolkit==3.0.43
-pycld2==0.41
-PyICU==2.12
-python-dateutil==2.8.2
-pytz==2024.1
-scikit-learn==1.4.0
-scipy==1.12.0
-six==1.16.0
-tenacity==8.2.3
-termcolor==2.4.0
-threadpoolctl==3.2.0
-tqdm==4.66.2
-typing_extensions==4.9.0
-tzdata==2024.1
-uvicorn==0.27.1
-wcwidth==0.2.13

arena_elo/results/20240220/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f41023a65a4dc1831a482dfa6098ccd528af9de297a1ea518881d49ce2885f0e
-size 57121

arena_elo/results/20240220/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f383f3920ef3834f6e6ec213691a955992b4ce49ccaf7658c0b35ff72a2219d3
-size 54505

arena_elo/results/20240220/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1252.820838097007,1216.6489026518666,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-PNP,PNP,1175.6261555831445,1171.3279007979363,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-InstructPix2Pix,InstructPix2Pix,1155.8431458813104,1142.6827834982837,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-MagicBrush,MagicBrush,1051.428411953954,1089.4499296239383,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,955.5903260059122,929.2296611307636,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,771.4360186105207,753.4930725653142,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,637.2551038681513,697.1677497318974,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240220/t2i_generation_leaderboard.csv

@@ -1,7 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGroundV2,PlayGroundV2,1151.1834096302248,1150.901721636401,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-PixArtAlpha,PixArtAlpha,1078.3583466674136,1069.815012597113,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,1027.258044463298,1035.47732509915,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,972.0904914158416,969.4933207298967,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,921.3424873878607,906.3184453708288,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,849.7672204353615,868.2154196730218,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240315/clean_battle_image_editing.json

@@ -1,794 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  }
-]

arena_elo/results/20240315/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
-size 57091

arena_elo/results/20240315/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1227.5508595026165,1182.7966878908553,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-InstructPix2Pix,InstructPix2Pix,1160.2057367236093,1104.6020920372373,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-PNP,PNP,1142.693603173293,1163.4165011464634,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-MagicBrush,MagicBrush,1053.1728944865915,1104.2259166081321,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,918.6047552604578,899.6573774461386,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,865.0529105743963,820.8487760803515,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,632.7192402790356,724.4526487908217,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240327/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
-size 62414

arena_elo/results/20240327/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240328/clean_battle_image_editing.json

@@ -1,890 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711610477.1213
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711629129.3894
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711629705.2246
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711630362.5575
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711631112.5207
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1711631690.5127
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711633200.2923
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711633594.9922
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711635443.3071
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711635899.3088
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1711639015.428
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711646372.1201
-  }
-]

arena_elo/results/20240328/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1430e6703dd6fc1e5b8ce06b11bb3a47516763a33edaf99e4c8547da5d9a8516
-size 57064

arena_elo/results/20240328/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1227.5508595026165,1158.5510681980204,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-InstructPix2Pix,InstructPix2Pix,1160.2057367236093,1071.0628993075604,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-PNP,PNP,1142.693603173293,1165.4957550490212,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-MagicBrush,MagicBrush,1053.1728944865915,1130.5422054860635,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,918.6047552604578,960.3217617445996,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,865.0529105743963,813.4794423328381,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,632.7192402790356,700.546867881897,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240330/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e963f9d4b66d29c2f05a3923eff56cebd1f09b07223ac069456e08dc6143cda8
-size 66894

arena_elo/results/20240330/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1236.5076527218755,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1062.0980902577003,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1087.3576445526567,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1019.4526672266176,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,1001.5090282446616,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,969.8928133531979,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,914.9478831930971,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,835.4563491411935,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,812.962889153237,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240408/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd88783d1cf752a0977152f7e16e88b54759173cbb04fb55e9392703ff4819f5
-size 66931

arena_elo/results/20240408/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1233.8616648345985,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1031.1844458387527,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1093.6921447327898,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1004.2360415152086,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,999.6264863931511,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,975.3460583905047,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,927.1873122981513,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,848.6657236271969,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,828.5108951096241,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240411/clean_battle_image_editing.json

@@ -1,906 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711610477.1213
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711629129.3894
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711629705.2246
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711630362.5575
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711631112.5207
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1711631690.5127
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711633200.2923
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711633594.9922
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711635443.3071
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711635899.3088
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1711639015.428
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711646372.1201
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1712873850.0636
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1712876598.7667
-  }
-]

arena_elo/results/20240411/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d66a54af51d2ecf89f461dbb4e15090d084638596952d3541ce369798a525ff3
-size 57096

arena_elo/results/20240411/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:caf98f560387fa9d6b8c233e9915807adad62315cfdd6d4a5e7c9fda30140eb8
-size 62422