Init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/duck.jpg filter=lfs diff=lfs merge=lfs -text

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "GenAI-Arena-hf-logs"]
+	path = GenAI-Arena-hf-logs
+	url = https://github.com/jdf-prog/GenAI-Arena-hf-logs.git

README.md

@@ -1,11 +1,13 @@
 ---
-title: GenAI-Arena
-emoji: 🚀
-colorFrom: indigo
-colorTo: yellow
-sdk: static
+title: GenAI Arena
+emoji: 📈
+colorFrom: purple
+colorTo: pink
+sdk: gradio
+sdk_version: 4.21.0
+app_file: app.py
 pinned: false
-header: mini
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,86 @@
+import gradio as gr
+import os
+from serve.gradio_web import *
+from serve.gradio_web_image_editing import *
+from serve.leaderboard import build_leaderboard_tab
+from model.model_manager import ModelManager
+from pathlib import Path
+from serve.constants import SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR
+
+def build_combine_demo(models, elo_results_file, leaderboard_table_file):
+
+    with gr.Blocks(
+        title="Play with Open Vision Models",
+        theme=gr.themes.Default(),
+        css=block_css,
+    ) as demo:
+        with gr.Tabs() as tabs_combine:
+            with gr.Tab("Image Generation", id=0):
+                with gr.Tabs() as tabs_ig:
+                    with gr.Tab("Generation Arena (battle)", id=0):
+                        build_side_by_side_ui_anony(models)
+
+                    with gr.Tab("Generation Arena (side-by-side)", id=1):
+                        build_side_by_side_ui_named(models)
+
+                    with gr.Tab("Generation Direct Chat", id=2):
+                        build_single_model_ui(models, add_promotion_links=True)
+                    if elo_results_file:
+                        with gr.Tab("Generation Leaderboard", id=3):
+                            build_leaderboard_tab(elo_results_file['t2i_generation'], leaderboard_table_file['t2i_generation'])
+                    with gr.Tab("About Us", id=4):
+                        build_about()
+            
+            with gr.Tab("Image Edition", id=5):
+                with gr.Tabs() as tabs_ie:
+                    with gr.Tab("Edition Arena (battle)", id=5):
+                        build_side_by_side_ui_anony_ie(models)
+
+                    with gr.Tab("Edition Arena (side-by-side)", id=6):
+                        build_side_by_side_ui_named_ie(models)
+
+                    with gr.Tab("Edition Direct Chat", id=7):
+                        build_single_model_ui_ie(models, add_promotion_links=True)
+                    if elo_results_file:
+                        with gr.Tab("Edition Leaderboard", id=8):
+                            build_leaderboard_tab(elo_results_file['image_editing'], leaderboard_table_file['image_editing'])
+                    with gr.Tab("About Us", id=9):
+                        build_about()
+
+    return demo
+
+
+def load_elo_results(elo_results_dir):
+    from collections import defaultdict
+    elo_results_file = defaultdict(lambda: None)
+    leaderboard_table_file = defaultdict(lambda: None)
+    if elo_results_dir is not None:
+        elo_results_dir = Path(elo_results_dir)
+        elo_results_file = {}
+        leaderboard_table_file = {}
+        for file in elo_results_dir.glob('elo_results_*.pkl'):
+            if 't2i_generation' in file.name:
+                elo_results_file['t2i_generation'] = file
+            elif 'image_editing' in file.name:
+                elo_results_file['image_editing'] = file
+            else:
+                raise ValueError(f"Unknown file name: {file.name}")
+        for file in elo_results_dir.glob('*_leaderboard.csv'):
+            if 't2i_generation' in file.name:
+                leaderboard_table_file['t2i_generation'] = file
+            elif 'image_editing' in file.name:
+                leaderboard_table_file['image_editing'] = file
+            else:
+                raise ValueError(f"Unknown file name: {file.name}")
+            
+    return elo_results_file, leaderboard_table_file
+    
+if __name__ == "__main__":
+    server_port = SERVER_PORT
+    root_path = ROOT_PATH
+    elo_results_dir = ELO_RESULTS_DIR
+    models = ModelManager()
+    
+    elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
+    demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
+    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)

arena_elo/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 WildVision-Bench
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

arena_elo/README.md

@@ -0,0 +1,46 @@
+## Computing the Elo Ratings
+
+
+```bash
+apt-get -y install pkg-config
+pip install -r requirements.txt
+```
+
+
+### to update the leaderboard
+
+```bash
+export LOGDIR="/path/to/your/logdir"
+bash update_elo_rating.sh
+```
+
+### to inspect the leaderboard status
+```bash
+python -m elo_rating.inspect_elo_rating_pkl
+```
+
+### to inspect the collected data status and cost
+```bash
+export LOGDIR="/path/to/your/logdir"
+python -m elo_rating.inspect_cost
+```
+
+### to upload the battle data to hugging face🤗 
+```bash
+export HUGGINGFACE_TOKEN="your_huggingface_token"
+bash get_latest_data.sh
+python -m elo_rating.upload_battle_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
+```
+
+### to upload the chat data to hugging face🤗 
+```bash
+export HUGGINGFACE_TOKEN="your_huggingface_token"
+bash get_latest_data.sh
+python -m elo_rating.upload_chat_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
+```
+
+
+### to get the collected data
+```bash
+python -m 
+

arena_elo/edition_model_info.json

@@ -0,0 +1,37 @@
+{
+    "CycleDiffusion": {
+        "Link": "https://github.com/ChenWu98/cycle-diffusion",
+        "License": "X11",
+        "Organization": "Carnegie Mellon University"
+    },
+    "PNP": {
+        "Link": "https://github.com/MichalGeyer/plug-and-play",
+        "License": "-",
+        "Organization": "Weizmann Institute of Science"
+    },
+    "InstructPix2Pix": {
+        "Link": "https://www.timothybrooks.com/instruct-pix2pix",
+        "License": "Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros",
+        "Organization": "University of California, Berkeley"
+    },
+    "Pix2PixZero": {
+        "Link": "https://pix2pixzero.github.io",
+        "License": "MIT License",
+        "Organization": "Carnegie Mellon University, Adobe Research"
+    },
+    "MagicBrush": {
+        "Link": "https://osu-nlp-group.github.io/MagicBrush",
+        "License": "CC-BY-4.0",
+        "Organization": "The Ohio State University, University of Waterloo"
+    },
+    "Prompt2prompt": {
+        "Link": "https://prompt-to-prompt.github.io",
+        "License": "Apache-2.0",
+        "Organization": "Google, Tel Aviv University"
+    },
+    "SDEdit": {
+        "Link": "https://sde-image-editing.github.io",
+        "License": "MIT License",
+        "Organization": "Stanford University"
+    }
+}

arena_elo/elo_rating/basic_stats.py

@@ -0,0 +1,227 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd  # pandas>=2.0.3
+import plotly.express as px
+import plotly.graph_objects as go
+from tqdm import tqdm
+
+NUM_SERVERS = 1
+LOG_ROOT_DIR = os.getenv("LOGDIR", None)
+if LOG_ROOT_DIR is None:
+    raise ValueError("LOGDIR environment variable not set, please set it by `export LOGDIR=...`")
+
+def get_log_files(max_num_files=None):
+    log_root = os.path.expanduser(LOG_ROOT_DIR)
+    filenames = []
+    if NUM_SERVERS == 1:
+        for filename in os.listdir(log_root):
+            if filename.endswith("-conv.json"):
+                filepath = f"{log_root}/{filename}"
+                name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                filenames.append(name_tstamp_tuple)
+    else:
+        for i in range(NUM_SERVERS):
+            for filename in os.listdir(f"{log_root}/server{i}"):
+                if filename.endswith("-conv.json"):
+                    filepath = f"{log_root}/server{i}/{filename}"
+                    name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                    filenames.append(name_tstamp_tuple)
+    # sort by tstamp
+    filenames = sorted(filenames, key=lambda x: x[1])
+    filenames = [x[0] for x in filenames]
+
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def load_log_files(filename):
+    data = []
+    for retry in range(5):
+        try:
+            lines = open(filename).readlines()
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+
+    for l in lines:
+        row = json.loads(l)
+        data.append(
+            dict(
+                type=row["type"],
+                tstamp=row["tstamp"],
+                model=row.get("model", ""),
+                models=row.get("models", ["", ""]),
+            )
+        )
+    return data
+
+
+def load_log_files_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(load_log_files, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+
+def get_anony_vote_df(df):
+    anony_vote_df = df[
+        df["type"].isin(["leftvote", "rightvote", "tievote", "bothbad_vote"])
+    ]
+    anony_vote_df = anony_vote_df[anony_vote_df["models"].apply(lambda x: x[0] == "")]
+    return anony_vote_df
+
+
+def merge_counts(series, on, names):
+    ret = pd.merge(series[0], series[1], on=on)
+    for i in range(2, len(series)):
+        ret = pd.merge(ret, series[i], on=on)
+    ret = ret.reset_index()
+    old_names = list(ret.columns)[-len(series) :]
+    rename = {old_name: new_name for old_name, new_name in zip(old_names, names)}
+    ret = ret.rename(columns=rename)
+    return ret
+
+
+def report_basic_stats(log_files):
+    df_all = load_log_files_parallel(log_files)
+    df_all = pd.DataFrame(df_all)
+    now_t = df_all["tstamp"].max()
+    df_1_hour = df_all[df_all["tstamp"] > (now_t - 3600)]
+    df_1_day = df_all[df_all["tstamp"] > (now_t - 3600 * 24)]
+    anony_vote_df_all = get_anony_vote_df(df_all)
+
+    # Chat trends
+    chat_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in df_all[df_all["type"] == "chat"]["tstamp"]
+    ]
+    chat_dates_counts = pd.value_counts(chat_dates)
+    vote_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in anony_vote_df_all["tstamp"]
+    ]
+    vote_dates_counts = pd.value_counts(vote_dates)
+    chat_dates_bar = go.Figure(
+        data=[
+            go.Bar(
+                name="Anony. Vote",
+                x=vote_dates_counts.index,
+                y=vote_dates_counts,
+                text=[f"{val:.0f}" for val in vote_dates_counts],
+                textposition="auto",
+            ),
+            go.Bar(
+                name="Chat",
+                x=chat_dates_counts.index,
+                y=chat_dates_counts,
+                text=[f"{val:.0f}" for val in chat_dates_counts],
+                textposition="auto",
+            ),
+        ]
+    )
+    chat_dates_bar.update_layout(
+        barmode="stack",
+        xaxis_title="Dates",
+        yaxis_title="Count",
+        height=300,
+        width=1200,
+    )
+
+    # Model call counts
+    model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
+    model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()
+    model_hist_1_hour = df_1_hour[df_1_hour["type"] == "chat"]["model"].value_counts()
+    model_hist = merge_counts(
+        [model_hist_all, model_hist_1_day, model_hist_1_hour],
+        on="model",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    model_hist_md = model_hist.to_markdown(index=False, tablefmt="github")
+
+    # Action counts
+    action_hist_all = df_all["type"].value_counts()
+    action_hist_1_day = df_1_day["type"].value_counts()
+    action_hist_1_hour = df_1_hour["type"].value_counts()
+    action_hist = merge_counts(
+        [action_hist_all, action_hist_1_day, action_hist_1_hour],
+        on="type",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    action_hist_md = action_hist.to_markdown(index=False, tablefmt="github")
+
+    # Anony vote counts
+    anony_vote_hist_all = anony_vote_df_all["type"].value_counts()
+    anony_vote_df_1_day = get_anony_vote_df(df_1_day)
+    anony_vote_hist_1_day = anony_vote_df_1_day["type"].value_counts()
+    # anony_vote_df_1_hour = get_anony_vote_df(df_1_hour)
+    # anony_vote_hist_1_hour = anony_vote_df_1_hour["type"].value_counts()
+    anony_vote_hist = merge_counts(
+        [anony_vote_hist_all, anony_vote_hist_1_day],
+        on="type",
+        names=["All", "Last Day"],
+    )
+    anony_vote_hist_md = anony_vote_hist.to_markdown(index=False, tablefmt="github")
+
+    # Last 24 hours
+    chat_1_day = df_1_day[df_1_day["type"] == "chat"]
+    num_chats_last_24_hours = []
+    base = df_1_day["tstamp"].min()
+    for i in range(24, 0, -1):
+        left = base + (i - 1) * 3600
+        right = base + i * 3600
+        num = ((chat_1_day["tstamp"] >= left) & (chat_1_day["tstamp"] < right)).sum()
+        num_chats_last_24_hours.append(num)
+    times = [
+        datetime.datetime.fromtimestamp(
+            base + i * 3600, tz=timezone("US/Pacific")
+        ).strftime("%Y-%m-%d %H:%M:%S %Z")
+        for i in range(24, 0, -1)
+    ]
+    last_24_hours_df = pd.DataFrame({"time": times, "value": num_chats_last_24_hours})
+    last_24_hours_md = last_24_hours_df.to_markdown(index=False, tablefmt="github")
+
+    # Last update datetime
+    last_updated_tstamp = now_t
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    # code.interact(local=locals())
+
+    return {
+        "chat_dates_bar": chat_dates_bar,
+        "model_hist_md": model_hist_md,
+        "action_hist_md": action_hist_md,
+        "anony_vote_hist_md": anony_vote_hist_md,
+        "num_chats_last_24_hours": last_24_hours_md,
+        "last_updated_datetime": last_updated_datetime,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    basic_stats = report_basic_stats(log_files)
+
+    print(basic_stats["action_hist_md"] + "\n")
+    print(basic_stats["model_hist_md"] + "\n")
+    print(basic_stats["anony_vote_hist_md"] + "\n")
+    print(basic_stats["num_chats_last_24_hours"] + "\n")

arena_elo/elo_rating/clean_battle_data.py

@@ -0,0 +1,381 @@
+"""
+Clean chatbot arena battle log.
+
+Usage:
+python3 clean_battle_data.py --mode conv_release
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+from pytz import timezone
+import time
+import PIL
+from PIL import ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+from tqdm import tqdm
+
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date
+
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
+IDENTITY_WORDS = [
+    "vicuna",
+    "lmsys",
+    "koala",
+    "uc berkeley",
+    "open assistant",
+    "laion",
+    "chatglm",
+    "chatgpt",
+    "gpt-4",
+    "openai",
+    "anthropic",
+    "claude",
+    "bard",
+    "palm",
+    "lamda",
+    "google",
+    "llama",
+    "qianwan",
+    "alibaba",
+    "mistral",
+    "zhipu",
+    "KEG lab",
+    "01.AI",
+    "AI2",
+    "Tülu",
+    "Tulu",
+    "NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.",
+    "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES.",
+    "API REQUEST ERROR. Please increase the number of max tokens.",
+    "**API REQUEST ERROR** Reason: The response was blocked.",
+    "**API REQUEST ERROR**",
+]
+
+for i in range(len(IDENTITY_WORDS)):
+    IDENTITY_WORDS[i] = IDENTITY_WORDS[i].lower()
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def to_openai_format(messages):
+    roles = ["user", "assistant"]
+    ret = []
+    for i, x in enumerate(messages):
+        ret.append({"role": roles[i % 2], "content": x[1]})
+    return ret
+
+
+def replace_model_name(old_name, tstamp):
+    replace_dict = {
+        "bard": "palm-2",
+        "claude-v1": "claude-1",
+        "claude-instant-v1": "claude-instant-1",
+        "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
+        "claude-2": "claude-2.0",
+    }
+    if old_name in ["gpt-4", "gpt-3.5-turbo"]:
+        if tstamp > 1687849200:
+            return old_name + "-0613"
+        else:
+            return old_name + "-0314"
+    if old_name in replace_dict:
+        return replace_dict[old_name]
+    return old_name
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def load_image(image_path):
+    try:
+        return PIL.Image.open(image_path)
+    except:
+        return None
+
+def clean_battle_data(
+    log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple", task_name="image_editing"
+):
+    data = read_file_parallel(log_files, num_threads=16)
+
+    convert_type = {
+        "leftvote": "model_a",
+        "rightvote": "model_b",
+        "tievote": "tie",
+        "bothbad_vote": "tie (bothbad)",
+    }
+    
+    all_models = set()
+    all_ips = dict()
+    ct_anony = 0
+    ct_invalid = 0
+    ct_leaked_identity = 0
+    ct_banned = 0
+    battles = []
+    for row in tqdm(data, desc="Cleaning"):
+        if row["models"][0] is None or row["models"][1] is None:
+            continue
+
+        # Resolve model names
+        models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+        if "model_name" in row["states"][0]:
+            models_hidden = [
+                row["states"][0]["model_name"],
+                row["states"][1]["model_name"],
+            ]
+            if models_hidden[0] is None:
+                models_hidden = models_public
+        else:
+            models_hidden = models_public
+
+        if (models_public[0] == "" and models_public[1] != "") or (
+            models_public[1] == "" and models_public[0] != ""
+        ):
+            ct_invalid += 1
+            continue
+
+        if models_public[0] == "" or models_public[0] == "Model A":
+            anony = True
+            models = models_hidden
+            ct_anony += 1
+        else:
+            anony = False
+            models = models_public
+            if not models_public == models_hidden:
+                ct_invalid += 1
+                continue
+
+        # # Detect langauge
+        # state = row["states"][0]
+        # if state["offset"] >= len(state["messages"]):
+        #     ct_invalid += 1
+        #     continue
+        # lang_code = detect_language(state["messages"][state["offset"]][1])
+
+        # # Drop conversations if the model names are leaked
+        # leaked_identity = False
+        # messages = ""
+        # for i in range(2):
+        #     state = row["states"][i]
+        #     for turn_idx, (role, msg) in enumerate(
+        #         state["messages"][state["offset"] :]
+        #     ):
+        #         if msg:
+        #             messages += msg.lower()
+        # for word in IDENTITY_WORDS:
+        #     if word in messages:
+        #         leaked_identity = True
+        #         break
+
+        # if leaked_identity:
+        #     ct_leaked_identity += 1
+        #     continue
+
+        # Replace bard with palm
+        models = [replace_model_name(m, row["tstamp"]) for m in models]
+        if task_name == "image_editing":
+            if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
+                # print(f"Invalid model names: {models}")
+                ct_invalid += 1
+                continue
+            models = [x[len("imagenhub_"):-len("_edition")] for x in models]
+        elif task_name == "t2i_generation":
+            if not all("playground" in x.lower() or (x.startswith("imagenhub_") and x.endswith("_generation")) for x in models):
+                # print(f"Invalid model names: {models}")
+                ct_invalid += 1
+                continue
+            # models = [x[len("imagenhub_"):-len("_generation")] for x in models]
+            for i, model_name in enumerate(models):
+                if model_name.startswith("imagenhub_"):
+                    models[i] = model_name[len("imagenhub_"):-len("_generation")]
+
+        else:
+            raise ValueError(f"Invalid task_name: {task_name}")
+        
+        # Exclude certain models
+        if exclude_model_names and any(x in exclude_model_names for x in models):
+            ct_invalid += 1
+            continue
+        
+        # if models[0] not in model_infos or models[1] not in model_infos:
+        #     continue
+
+        # # Exclude votes before the starting date
+        # if model_infos and (model_infos[models[0]]["starting_from"] > row["tstamp"] or model_infos[models[1]]["starting_from"] > row["tstamp"]):
+        #     print(f"Invalid vote before the valid starting date for {models[0]} and {models[1]}")
+        #     ct_invalid += 1
+        #     continue
+        
+        
+
+        if mode == "conv_release":
+            # assert the two images are the same
+            date = datetime.datetime.fromtimestamp(row["tstamp"], tz=timezone("US/Pacific")).strftime("%Y-%m-%d") # 2024-02-29
+            image_path_format = f"{LOG_ROOT_DIR}/{date}-convinput_images/input_image_"
+            image_path_0 = image_path_format + str(row["states"][0]["conv_id"]) + ".png"
+            image_path_1 = image_path_format + str(row["states"][1]["conv_id"]) + ".png"
+            if not os.path.exists(image_path_0) or not os.path.exists(image_path_1):
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            
+            image_0 = load_image(image_path_0)
+            image_1 = load_image(image_path_1)
+            if image_0 is None or image_1 is None:
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            if image_0.tobytes() != image_1.tobytes():
+                print(f"Image not the same for {image_path_0} and {image_path_1}")
+                ct_invalid += 1
+                continue
+        
+        
+        question_id = row["states"][0]["conv_id"]
+        # conversation_a = to_openai_format(
+        #     row["states"][0]["messages"][row["states"][0]["offset"] :]
+        # )
+        # conversation_b = to_openai_format(
+        #     row["states"][1]["messages"][row["states"][1]["offset"] :]
+        # )
+
+        ip = row["ip"]
+        if ip not in all_ips:
+            all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
+        all_ips[ip]["count"] += 1
+        if sanitize_ip:
+            user_id = f"arena_user_{all_ips[ip]['sanitized_id']}"
+        else:
+            user_id = f"{all_ips[ip]['ip']}"
+
+        if ban_ip_list is not None and ip in ban_ip_list:
+            ct_banned += 1
+            continue
+
+        # Save the results
+        battles.append(
+            dict(
+                question_id=question_id,
+                model_a=models[0],
+                model_b=models[1],
+                winner=convert_type[row["type"]],
+                judge=f"arena_user_{user_id}",
+                # conversation_a=conversation_a,
+                # conversation_b=conversation_b,
+                # turn=len(conversation_a) // 2,
+                anony=anony,
+                # language=lang_code,
+                tstamp=row["tstamp"],
+            )
+        )
+
+        all_models.update(models_hidden)
+    battles.sort(key=lambda x: x["tstamp"])
+    last_updated_tstamp = battles[-1]["tstamp"]
+
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    print(
+        f"#votes: {len(data)}, #invalid votes: {ct_invalid}, "
+        f"#leaked_identity: {ct_leaked_identity} "
+        f"#banned: {ct_banned} "
+    )
+    print(f"#battles: {len(battles)}, #anony: {ct_anony}")
+    print(f"#models: {len(all_models)}, {all_models}")
+    print(f"last-updated: {last_updated_datetime}")
+
+    if ban_ip_list is not None:
+        for ban_ip in ban_ip_list:
+            if ban_ip in all_ips:
+                del all_ips[ban_ip]
+    print("Top 30 IPs:")
+    print(sorted(all_ips.values(), key=lambda x: x["count"], reverse=True)[:30])
+    return battles
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument(
+        "--mode", type=str, choices=["simple", "conv_release"], default="simple"
+    )
+    parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation"])
+    parser.add_argument("--exclude-model-names", type=str, nargs="+")
+    parser.add_argument("--ban-ip-file", type=str)
+    parser.add_argument("--sanitize-ip", action="store_true", default=False)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None
+
+    battles = clean_battle_data(
+        log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, args.mode, args.task_name
+    )
+    last_updated_tstamp = battles[-1]["tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+    if args.mode == "simple":
+        for x in battles:
+            for key in [
+                "conversation_a",
+                "conversation_b",
+                "question_id",
+            ]:
+                if key in x:
+                    del x[key]
+        print("Samples:")
+        for i in range(min(4, len(battles))):
+            print(battles[i])
+        output = f"clean_battle_{args.task_name}_{cutoff_date}.json"
+    elif args.mode == "conv_release":
+        # new_battles = []
+        # for x in battles:
+        #     if not x["anony"]:
+        #         continue
+        #     for key in []:
+        #         del x[key]
+        #     new_battles.append(x)
+        # battles = new_battles
+        output = f"clean_battle_{args.task_name}_conv_{cutoff_date}.json"
+
+    with open(output, "w") as fout:
+        json.dump(battles, fout, indent=2, ensure_ascii=False)
+    print(f"Write cleaned data to {output}")
+    
+    with open("cut_off_date.txt", "w") as fout:
+        fout.write(cutoff_date)

arena_elo/elo_rating/elo_analysis.py

@@ -0,0 +1,378 @@
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import math
+import pickle
+from pytz import timezone
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+from tqdm import tqdm
+
+from .model_registry import get_model_info
+from .basic_stats import get_log_files
+from .clean_battle_data import clean_battle_data
+        
+pd.options.display.float_format = "{:.2f}".format
+
+
+def compute_elo(battles, K=4, SCALE=400, BASE=10, INIT_RATING=1000):
+    rating = defaultdict(lambda: INIT_RATING)
+
+    for rd, model_a, model_b, winner in battles[
+        ["model_a", "model_b", "winner"]
+    ].itertuples():
+        ra = rating[model_a]
+        rb = rating[model_b]
+        ea = 1 / (1 + BASE ** ((rb - ra) / SCALE))
+        eb = 1 / (1 + BASE ** ((ra - rb) / SCALE))
+        if winner == "model_a":
+            sa = 1
+        elif winner == "model_b":
+            sa = 0
+        elif winner == "tie" or winner == "tie (bothbad)":
+            sa = 0.5
+        else:
+            raise Exception(f"unexpected vote {winner}")
+        rating[model_a] += K * (sa - ea)
+        rating[model_b] += K * (1 - sa - eb)
+
+    return dict(rating)
+
+
+def get_bootstrap_result(battles, func_compute_elo, num_round=1000):
+    rows = []
+    for i in tqdm(range(num_round), desc="bootstrap"):
+        tmp_battles = battles.sample(frac=1.0, replace=True)
+        rows.append(func_compute_elo(tmp_battles))
+    df = pd.DataFrame(rows)
+    return df[df.median().sort_values(ascending=False).index]
+
+
+def compute_elo_mle_with_tie(df, SCALE=400, BASE=10, INIT_RATING=1000):
+    from sklearn.linear_model import LogisticRegression
+
+    models = pd.concat([df["model_a"], df["model_b"]]).unique()
+    models = pd.Series(np.arange(len(models)), index=models)
+
+    # duplicate battles
+    df = pd.concat([df, df], ignore_index=True)
+    p = len(models.index)
+    n = df.shape[0]
+
+    X = np.zeros([n, p])
+    X[np.arange(n), models[df["model_a"]]] = +math.log(BASE)
+    X[np.arange(n), models[df["model_b"]]] = -math.log(BASE)
+
+    # one A win => two A win
+    Y = np.zeros(n)
+    Y[df["winner"] == "model_a"] = 1.0
+
+    # one tie => one A win + one B win
+    # find tie + tie (both bad) index
+    tie_idx = (df["winner"] == "tie") | (df["winner"] == "tie (bothbad)")
+    tie_idx[len(tie_idx) // 2 :] = False
+    Y[tie_idx] = 1.0
+
+    lr = LogisticRegression(fit_intercept=False)
+    lr.fit(X, Y)
+
+    elo_scores = SCALE * lr.coef_[0] + INIT_RATING
+    # calibrate llama-13b to 800 if applicable
+    if "llama-13b" in models.index:
+        elo_scores += 800 - elo_scores[models["llama-13b"]]
+    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
+
+
+def get_median_elo_from_bootstrap(bootstrap_df):
+    median = dict(bootstrap_df.quantile(0.5))
+    median = {k: int(v + 0.5) for k, v in median.items()}
+    return median
+
+
+def compute_pairwise_win_fraction(battles, model_order, limit_show_number=None):
+    # Times each model wins as Model A
+    a_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_a"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting times each model wins as Model B
+    b_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_b"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting number of A-B pairs
+    num_battles_ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+
+    # Computing the proportion of wins for each model as A and as B
+    # against all other models
+    row_beats_col_freq = (a_win_ptbl + b_win_ptbl.T) / (
+        num_battles_ptbl + num_battles_ptbl.T
+    )
+
+    if model_order is None:
+        prop_wins = row_beats_col_freq.mean(axis=1).sort_values(ascending=False)
+        model_order = list(prop_wins.keys())
+
+    if limit_show_number is not None:
+        model_order = model_order[:limit_show_number]
+
+    # Arrange ordering according to proprition of wins
+    row_beats_col = row_beats_col_freq.loc[model_order, model_order]
+    return row_beats_col
+
+
+def visualize_leaderboard_table(rating):
+    models = list(rating.keys())
+    models.sort(key=lambda k: -rating[k])
+
+    emoji_dict = {
+        1: "🥇",
+        2: "🥈",
+        3: "🥉",
+    }
+
+    md = ""
+    md += "| Rank | Model | Elo Rating | Description |\n"
+    md += "| --- | --- | --- | --- |\n"
+    for i, model in enumerate(models):
+        rank = i + 1
+        minfo = get_model_info(model)
+        emoji = emoji_dict.get(rank, "")
+        md += f"| {rank} | {emoji} [{model}]({minfo.link}) | {rating[model]:.0f} | {minfo.description} |\n"
+
+    return md
+
+
+def visualize_pairwise_win_fraction(battles, model_order):
+    row_beats_col = compute_pairwise_win_fraction(battles, model_order)
+    fig = px.imshow(
+        row_beats_col,
+        color_continuous_scale="RdBu",
+        text_auto=".2f",
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
+    )
+
+    return fig
+
+
+def visualize_battle_count(battles, model_order):
+    ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+    battle_counts = ptbl + ptbl.T
+    fig = px.imshow(
+        battle_counts.loc[model_order, model_order],
+        text_auto=True,
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
+    )
+    return fig
+
+
+def visualize_average_win_rate(battles, limit_show_number):
+    row_beats_col_freq = compute_pairwise_win_fraction(
+        battles, None, limit_show_number=limit_show_number
+    )
+    fig = px.bar(
+        row_beats_col_freq.mean(axis=1).sort_values(ascending=False),
+        text_auto=".2f",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(
+        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False
+    )
+    return fig
+
+
+def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
+    bars = (
+        pd.DataFrame(
+            dict(
+                lower=df.quantile(0.025),
+                rating=df_final,
+                upper=df.quantile(0.975),
+            )
+        )
+        .reset_index(names="model")
+        .sort_values("rating", ascending=False)
+    )
+    bars = bars[:limit_show_number]
+    bars["error_y"] = bars["upper"] - bars["rating"]
+    bars["error_y_minus"] = bars["rating"] - bars["lower"]
+    bars["rating_rounded"] = np.round(bars["rating"], 2)
+    fig = px.scatter(
+        bars,
+        x="model",
+        y="rating",
+        error_y="error_y",
+        error_y_minus="error_y_minus",
+        text="rating_rounded",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
+    return fig
+
+
+def report_elo_analysis_results(battles_json, rating_system="bt", num_bootstrap=100, anony_only=True):
+    battles = pd.DataFrame(battles_json)
+    battles = battles.sort_values(ascending=True, by=["tstamp"])
+    # Only use anonymous votes
+    if anony_only:
+        battles = battles[battles["anony"]].reset_index(drop=True)
+    battles_no_ties = battles[~battles["winner"].str.contains("tie")]
+
+    # Online update
+    elo_rating_online = compute_elo(battles)
+
+    if rating_system == "bt":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo_mle_with_tie, num_round=num_bootstrap
+        )
+        elo_rating_final = compute_elo_mle_with_tie(battles)
+    elif rating_system == "elo":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo, num_round=num_bootstrap
+        )
+        elo_rating_median = get_median_elo_from_bootstrap(bootstrap_df)
+        elo_rating_final = elo_rating_median
+
+    model_order = list(elo_rating_final.keys())
+    model_order.sort(key=lambda k: -elo_rating_final[k])
+
+    limit_show_number = 25  # limit show number to make plots smaller
+    model_order = model_order[:limit_show_number]
+
+    # leaderboard_table_df: elo rating, variance, 95% interval, number of battles
+    leaderboard_table_df = pd.DataFrame(
+        {
+            "rating": elo_rating_final,
+            "variance": bootstrap_df.var(),
+            "rating_q975": bootstrap_df.quantile(0.975),
+            "rating_q025": bootstrap_df.quantile(0.025),
+            "num_battles": battles["model_a"].value_counts()
+            + battles["model_b"].value_counts(),
+        }
+    )
+
+    # Plots
+    leaderboard_table = visualize_leaderboard_table(elo_rating_final)
+    win_fraction_heatmap = visualize_pairwise_win_fraction(battles_no_ties, model_order)
+    battle_count_heatmap = visualize_battle_count(battles_no_ties, model_order)
+    average_win_rate_bar = visualize_average_win_rate(
+        battles_no_ties, limit_show_number
+    )
+    bootstrap_elo_rating = visualize_bootstrap_elo_rating(
+        bootstrap_df, elo_rating_final, limit_show_number
+    )
+
+    last_updated_tstamp = battles["tstamp"].max()
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    return {
+        "rating_system": rating_system,
+        "elo_rating_online": elo_rating_online,
+        "elo_rating_final": elo_rating_final,
+        "leaderboard_table": leaderboard_table,
+        "win_fraction_heatmap": win_fraction_heatmap,
+        "battle_count_heatmap": battle_count_heatmap,
+        "average_win_rate_bar": average_win_rate_bar,
+        "bootstrap_elo_rating": bootstrap_elo_rating,
+        "last_updated_datetime": last_updated_datetime,
+        "last_updated_tstamp": last_updated_tstamp,
+        "bootstrap_df": bootstrap_df,
+        "leaderboard_table_df": leaderboard_table_df,
+    }
+
+
+def pretty_print_elo_rating(rating):
+    model_order = list(rating.keys())
+    model_order.sort(key=lambda k: -rating[k])
+    for i, model in enumerate(model_order):
+        print(f"{i+1:2d}, {model:25s}, {rating[model]:.0f}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--clean-battle-file", type=str)
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument("--num-bootstrap", type=int, default=100)
+    parser.add_argument(
+        "--rating-system", type=str, choices=["bt", "elo"], default="bt"
+    )
+    parser.add_argument("--exclude-tie", action="store_true", default=False)
+    args = parser.parse_args()
+
+    np.random.seed(42)
+
+    if args.clean_battle_file:
+        # Read data from a cleaned battle files
+        battles = pd.read_json(args.clean_battle_file)
+    else:
+        # Read data from all log files
+        log_files = get_log_files(args.max_num_files)
+        battles = clean_battle_data(log_files)
+
+    anony_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=True
+    )
+    full_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=False
+    )
+    
+
+    print("# Online Elo")
+    pretty_print_elo_rating(anony_results["elo_rating_online"])
+    print("# Median")
+    pretty_print_elo_rating(anony_results["elo_rating_final"])
+    print(f"last update : {anony_results['last_updated_datetime']}")
+
+    last_updated_tstamp = anony_results["last_updated_tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+
+    results = {
+        "anony": anony_results,
+        "full": full_results,
+    }
+    with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
+        pickle.dump(results, fout)

arena_elo/elo_rating/generate_leaderboard.py

@@ -0,0 +1,71 @@
+import fire
+import json
+import pandas as pd
+import pickle
+
+
+def main(
+        model_info_file: str,
+        elo_rating_pkl: str,
+        output_csv: str
+):
+    model_info = json.load(open(model_info_file))
+
+    with open(elo_rating_pkl, "rb") as fin:
+        elo_rating_results = pickle.load(fin)
+
+    anony_elo_rating_results = elo_rating_results["anony"]
+    full_elo_rating_results = elo_rating_results["full"]
+    anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
+    full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
+
+    # Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
+    fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
+    # set Organization and license to empty for now
+    all_models = anony_leaderboard_data.index.tolist()
+
+    for model in all_models:
+        if not model in model_info:
+            model_info[model] = {}
+            model_info[model]["License"] = "N/A"
+            model_info[model]["Organization"] = "N/A"
+            model_info[model]["Link"] = "N/A"
+        model_info[model]["Model"] = model
+        model_info[model]["key"] = model
+
+        if model in anony_leaderboard_data.index:
+            model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (anony)"] = 0
+
+        if model in full_elo_rating_results["leaderboard_table_df"].index:
+            model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (full)"] = 0
+        # if model in anony_leaderboard_data.index:
+        #     model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
+        # else:
+        #     model_info[model]["Arena Elo rating"] = 0
+
+    final_model_info = {}
+    for model in model_info:
+        if "Model" in model_info[model]:
+            final_model_info[model] = model_info[model]
+    model_info = final_model_info
+
+    exclude_keys = ['starting_from']
+    for key in exclude_keys:
+        for model in model_info:
+            if key in model_info[model]:
+                del model_info[model][key]
+    df = pd.DataFrame(model_info).T
+    df = df[fields]
+    # sort by anony rating
+    df = df.sort_values(by=["Arena Elo rating (anony)"], ascending=False)
+    df.to_csv(output_csv, index=False)
+    print("Leaderboard data saved to", output_csv)
+    print(df)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_conv_rating.py

@@ -0,0 +1,234 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd
+from tqdm import tqdm
+import csv
+
+import base64
+from icecream import ic
+from openai import OpenAI
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+
+def get_gpt4v_response(client, img_bs64=None, text_prompt="", use_vision=False):
+    if use_vision:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{img_bs64}"
+                        }
+                    },
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    else:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    return response.choices[0].message.content
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    ic(log_files)
+    data = []
+    total_vote = 0
+    correct_vote = 0
+        
+    client = OpenAI()
+    with open('all_pairvote_log_wgpt_prtchatbot.csv', 'w', newline='') as csvfile:
+        # fieldnames = ['tstamp', 'type', 'model_1', 'model_2', 'template_name_1', 'template_name_2', 'system_message_1', 'system_message_2', 'role_1', 'role_2', 'instruction_1', 'instruction_2', 'message_1', 'message_2', 'offset_1', 'offset_2', 'conv_id_1', 'conv_id_2', 'model_name_1', 'model_name_2', 'ip']
+        fieldnames = ['tstamp', 'type', 'models', 'states', 'ip', 'gpt_vote']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        
+        # Write the header
+        writer.writeheader()
+
+        for filename in tqdm(log_files, desc="read files"):
+            for retry in range(5):
+                try:
+                    lines = open(filename).readlines()
+                    break
+                except FileNotFoundError:
+                    time.sleep(2)
+            
+            for l in lines:
+                row = json.loads(l)
+                
+                if "states" not in row:
+                    continue
+                if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                    continue
+                
+                model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+                
+
+                # Iterate through each state and write the relevant information
+                if not len(row["states"][0]['messages']): continue
+                # ic(row["states"][0]['messages'][1][1])
+
+                if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+                total_vote += 1
+                # row = {
+                #     'tstamp': row['tstamp'],
+                #     'type': row['type'],
+                #     'model_1': row['models'][0],
+                #     'model_2': row['models'][1],
+                #     'template_name_1': row["states"][0]['template_name'],
+                #     'system_message_1': row["states"][0]['system_message'],
+                #     'template_name_2': row["states"][1]['template_name'],
+                #     'system_message_2': row["states"][1]['system_message'],
+                #     'role_1': row["states"][0]['roles'],
+                #     'role_2': row["states"][1]['roles'],
+                #     'instruction_1': row["states"][0]['messages'][0][1],
+                #     'instruction_2': row["states"][1]['messages'][0][1],
+                #     'message_1': row["states"][0]['messages'][1][1],
+                #     'message_2': row["states"][1]['messages'][1][1],
+                #     'offset_1': row["states"][0]['offset'],
+                #     'offset_2': row["states"][1]['offset'],
+                #     'conv_id_1': row["states"][0]['conv_id'],
+                #     'conv_id_2': row["states"][1]['conv_id'],
+                #     'model_name_1': row["states"][0]['model_name'],
+                #     'model_name_2': row["states"][1]['model_name'],
+                #     'ip': row['ip']
+                # }
+                # writer.writerow(row)
+                # Convert complex objects to JSON strings
+                # TODO: check two image are the same
+                conv_id = row["states"][0]['conv_id']
+                image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+                if not os.path.exists(image_path):
+                    response = "NA"
+                    ic(image_path)
+                else:
+                    base64_image = encode_image(image_path)
+                    left_response = row["states"][0]['messages'][1][1]
+                    right_response = row["states"][1]['messages'][1][1]
+                    sep = "-" * 20
+                    instruction = row["states"][0]['messages'][0][1]
+                    generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+                    text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+                    # ic(text_prompt)
+                    try:
+                        response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    except:
+                        ic(">>> skip")
+                        response = "NA"
+                    
+                    # response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    ic(row['type'], response)
+                    if response.strip() not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                        response = "NA"
+                    # ic(generated_sentence)
+                    
+                    # if row['type'] == "leftvote":
+                    #     row['type'] = "A"
+                    # elif row['type'] == "rightvote":
+                    #     row['type'] = "B"
+                    # elif row['type'] in ["bothbad_vote", "tievote"]:
+                    #     row['type'] = "C"
+                    if row['type'] == response.strip():
+                        correct_vote += 1
+                row['models'] = json.dumps(row['models'])
+                row['states'] = json.dumps(row['states'], ensure_ascii=False)
+                row['gpt_vote'] = response
+                
+                # Write the modified row to the CSV file
+                writer.writerow(row)
+                # if row["type"] == "leftvote":
+                #     winner, loser = model_names[0], model_names[1]
+                #     winner_conv, loser_conv = row["states"][0], row["states"][1]
+                # elif row["type"] == "rightvote":
+                #     loser, winner = model_names[0], model_names[1]
+                #     loser_conv, winner_conv = row["states"][0], row["states"][1]
+
+                # if loser == "llava-v1.5-13b" and winner == "llava-v1.5-13b":
+                #     print("=" * 20)
+                #     print(f"Winner: {winner}")
+                #     pretty_print_conversation(winner_conv["messages"])
+                #     print(f"Loser: {loser}")
+                #     pretty_print_conversation(loser_conv["messages"])
+                #     print("=" * 20)
+                #     input()
+                # if row['type'] == 'bothbad_vote':
+                #     from icecream import ic
+                #     ic(model_names)
+                # if row["type"] == "bothbad_vote" and "gpt-4-vision-preview" in model_names:
+                #    print("=" * 20)
+                #    print(f"Model A: {model_names[0]}")
+                #    pretty_print_conversation(row["states"][0]["messages"])
+                #    print(f"Model B: {model_names[1]}")
+                #    pretty_print_conversation(row["states"][1]["messages"])
+                #    print("=" * 20)
+                #    input()
+                # if correct_vote >= 300: break
+    ic(total_vote, correct_vote)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)

arena_elo/elo_rating/inspect_cost.py

@@ -0,0 +1,177 @@
+import fire
+import time
+import json
+from collections import defaultdict
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date, get_input_image_path, load_image_from_path
+from tqdm import tqdm
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote", "chat"]
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def num_tokens(s:str):
+    if s is None:
+        return 0
+    return len(s) / 4
+
+def main(
+):
+    log_files = get_log_files()
+    data = read_file_parallel(log_files)
+    
+    all_model_counts = defaultdict(int)
+    all_model_input_tokens_counts = defaultdict(list)
+    all_model_output_tokens_counts = defaultdict(list)
+    all_model_image_sizes = defaultdict(list)
+    chat_battle_counts = defaultdict(int)
+    for row in tqdm(data, desc="counting"):
+        if row['type'] == "chat":
+            chat_battle_counts["chat"] += 1
+            all_model_counts[row['model']] += 1
+            tstamp = row["tstamp"]
+            conv_id = row["state"]["conv_id"]
+            
+            image = load_image_from_path(get_input_image_path(tstamp, conv_id))
+            if image is None:
+                image_size = None
+            else:
+                image_size = load_image_from_path(get_input_image_path(tstamp, conv_id)).size
+            all_model_image_sizes[row['model']].append(image_size)
+            try:
+                for message in row["state"]["messages"][row["state"]["offset"] :: 2]:
+                    all_model_input_tokens_counts[row['model']].append(num_tokens(message[1]))
+                for message in row["state"]["messages"][row["state"]["offset"] + 1 :: 2]:
+                    all_model_output_tokens_counts[row['model']].append(num_tokens(message[1]))
+            except Exception as e:
+                print(row)
+                raise e
+            
+        else:
+            chat_battle_counts[row['type']] += 1
+            if row["models"][0] is None or row["models"][1] is None:
+                continue
+
+            # Resolve model names
+            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+            if "model_name" in row["states"][0]:
+                models_hidden = [
+                    row["states"][0]["model_name"],
+                    row["states"][1]["model_name"],
+                ]
+                if models_hidden[0] is None:
+                    models_hidden = models_public
+            else:
+                models_hidden = models_public
+
+            if (models_public[0] == "" and models_public[1] != "") or (
+                models_public[1] == "" and models_public[0] != ""
+            ):
+                continue
+
+            if models_public[0] == "" or models_public[0] == "Model A":
+                anony = True
+                models = models_hidden
+            else:
+                anony = False
+                models = models_public
+                if not models_public == models_hidden:
+                    continue
+            
+            all_model_counts[models[0]] += 1
+            all_model_counts[models[1]] += 1
+            tstamp = row["tstamp"]
+            conv_id1 = row["states"][0]["conv_id"]
+            conv_id2 = row["states"][1]["conv_id"]
+            
+            image1 = load_image_from_path(get_input_image_path(tstamp, conv_id1))
+            image2 = load_image_from_path(get_input_image_path(tstamp, conv_id2))
+            all_model_image_sizes[models[0]].append(None if image1 is None else image1.size)
+            all_model_image_sizes[models[1]].append(None if image2 is None else image2.size)
+            
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[1]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[1]].append(num_tokens(message[1]))
+
+    print("### Chat battle counts (requests)")
+    print(json.dumps(chat_battle_counts, indent=4))
+    
+    print("### Model counts (requests)")
+    print(json.dumps(all_model_counts, indent=4))
+    
+    print("### Model Avg input tokens counts (tokens)")
+    average_input_tokens_counts = {}
+    for model, counts in all_model_input_tokens_counts.items():
+        average_input_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_input_tokens_counts, indent=4))
+    
+    print("### Model AVg output tokens counts (tokens)")
+    average_output_tokens_counts = {}
+    for model, counts in all_model_output_tokens_counts.items():
+        average_output_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_output_tokens_counts, indent=4))
+    
+    print("### Model Avg image sizes (height, width)")
+    average_image_sizes = {}
+    for model, sizes in all_model_image_sizes.items():
+        avg_height = sum([size[0] for size in sizes if size is not None]) / len(sizes)
+        avg_width = sum([size[1] for size in sizes if size is not None]) / len(sizes)
+        average_image_sizes[model] = (avg_height, avg_width)
+    print(json.dumps(average_image_sizes, indent=4))
+    
+    print("### GPT-4V estimated cost (USD)")
+    gpt_4v_name = "gpt-4-vision-preview"
+    gpt_4v_cost = {}
+    gpt_4v_cost['input'] = sum(all_model_input_tokens_counts[gpt_4v_name]) / 1000 * 0.01
+    gpt_4v_cost['output'] = sum(all_model_output_tokens_counts[gpt_4v_name]) / 1000 * 0.03
+    
+    all_image_cost = 0
+    for size in all_model_image_sizes[gpt_4v_name]:
+        if size is None:
+            continue
+        all_image_tokens = (size[0] // 512 + 1) * (size[1] // 512 + 1) * 170 + 85
+        all_image_cost += all_image_tokens / 1000 * 0.01
+    gpt_4v_cost['image'] = all_image_cost
+    print(json.dumps(gpt_4v_cost, indent=4))
+        
+    
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_elo_rating_pkl.py

@@ -0,0 +1,33 @@
+import pickle
+import plotly.graph_objects as go
+
+def output_figure(data, figure_name="battle_count_heatmap", label="annoy"):
+    fig = data[label][figure_name]
+    fig.update_layout(
+        height=700,
+        width=700,
+        title={'text': f'{figure_name}', 'x': 0.5, 'y': 0.07},
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        # coloraxis_colorscale=[[0.0, '#0d0887'], [1.0, '#f0f921']],
+        margin={'t': 60}
+    )
+    fig.write_image(f"{figure_name}.png")
+
+with open("./results/latest/elo_results.pkl",'rb') as f:
+    data = pickle.load(f)
+    print()
+    df = data["anony"]["leaderboard_table_df"]
+    # sort by rating 
+    print(data["anony"].keys())
+    
+    for figure_name in [ 'win_fraction_heatmap', 'battle_count_heatmap',]:
+        output_figure(data, figure_name, "anony")
+        
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df) 
+    df = data["full"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+    print('done') 

arena_elo/elo_rating/model_registry.py

@@ -0,0 +1,578 @@
+"""Additional information of the models."""
+from collections import namedtuple, OrderedDict
+from typing import List
+
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+
+
+model_info = OrderedDict()
+
+
+def register_model_info(
+    full_names: List[str], simple_name: str, link: str, description: str
+):
+    info = ModelInfo(simple_name, link, description)
+
+    for full_name in full_names:
+        model_info[full_name] = info
+
+
+def get_model_info(name: str) -> ModelInfo:
+    if name in model_info:
+        return model_info[name]
+    else:
+        # To fix this, please use `register_model_info` to register your model
+        return ModelInfo(
+            name, "", "Register the description at arena.model/model_registry.py"
+        )
+
+
+register_model_info(
+    [
+        "IEITYuan/Yuan2-2B-Janus-hf",
+        "IEITYuan/Yuan2-2B-hf",
+        "IEITYuan/Yuan2-51B-hf",
+        "IEITYuan/Yuan2-102B-hf",
+    ],
+    "IEIT-Yuan2",
+    "https://github.com/IEIT-Yuan/Yuan-2.0",
+    "Yuan2.0 is a new generation Fundamental Large Language Model developed by IEIT System.",
+)
+
+register_model_info(
+    ["mixtral-8x7b-instruct-v0.1", "mistral-7b-instruct"],
+    "Mixtral of experts",
+    "https://mistral.ai/news/mixtral-of-experts/",
+    "A Mixture-of-Experts model by Mistral AI",
+)
+
+register_model_info(
+    ["gemini-pro"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["gemini-pro-vision"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["solar-10.7b-instruct-v1.0"],
+    "SOLAR-10.7B-Instruct",
+    "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
+    "A model trained using depth up-scaling by Upstage AI",
+)
+
+register_model_info(
+    ["gpt-4-turbo"],
+    "GPT-4-Turbo",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4-vision-preview"],
+    "gpt-4-vision-preview",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4(V) by OpenAI",
+)
+
+register_model_info(
+    ["gpt-3.5-turbo", "gpt-3.5-turbo-0314", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-1106"],
+    "GPT-3.5",
+    "https://platform.openai.com/docs/models/gpt-3-5",
+    "GPT-3.5-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4", "gpt-4-0314", "gpt-4-0613"],
+    "GPT-4",
+    "https://openai.com/research/gpt-4",
+    "GPT-4 by OpenAI",
+)
+
+register_model_info(
+    ["claude-2.1", "claude-2.0"],
+    "Claude",
+    "https://www.anthropic.com/index/claude-2",
+    "Claude 2 by Anthropic",
+)
+
+register_model_info(
+    ["claude-1"],
+    "Claude",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude 1 by Anthropic",
+)
+
+register_model_info(
+    ["claude-instant-1", "claude-instant-1.2"],
+    "Claude Instant",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude Instant by Anthropic",
+)
+
+register_model_info(
+    ["pplx-70b-online", "pplx-7b-online"],
+    "pplx-online-llms",
+    "https://blog.perplexity.ai/blog/introducing-pplx-online-llms",
+    "Online LLM API by Perplexity AI",
+)
+
+register_model_info(
+    ["openhermes-2.5-mistral-7b"],
+    "OpenHermes-2.5-Mistral-7B",
+    "https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B",
+    "a mistral-based model fine-tuned on 1M GPT-4 outputs",
+)
+
+register_model_info(
+    ["starling-lm-7b-alpha"],
+    "Starling-LM-7B-alpha",
+    "https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha",
+    "an open model trained using RLAIF by Berkeley",
+)
+
+register_model_info(
+    ["tulu-2-dpo-70b"],
+    "Tulu 2",
+    "https://huggingface.co/allenai/tulu-2-dpo-70b",
+    "an instruction and RLHF model by UW/AllenAI",
+)
+
+register_model_info(
+    ["yi-34b-chat", "yi-6b-chat"],
+    "Yi-Chat",
+    "https://huggingface.co/01-ai/Yi-34B-Chat",
+    "A large language model by 01 AI",
+)
+
+register_model_info(
+    ["llama-2-70b-chat", "llama-2-34b-chat", "llama-2-13b-chat", "llama-2-7b-chat"],
+    "Llama 2",
+    "https://ai.meta.com/llama/",
+    "open foundation and fine-tuned chat models by Meta",
+)
+
+register_model_info(
+    [
+        "vicuna-33b",
+        "vicuna-33b-v1.3",
+        "vicuna-13b",
+        "vicuna-13b-v1.3",
+        "vicuna-7b",
+        "vicuna-7b-v1.3",
+    ],
+    "Vicuna",
+    "https://lmsys.org/blog/2023-03-30-vicuna/",
+    "a chat assistant fine-tuned on user-shared conversations by LMSYS",
+)
+
+register_model_info(
+    ["chatglm3-6b", "chatglm2-6b", "chatglm-6b"],
+    "ChatGLM",
+    "https://chatglm.cn/blog",
+    "an open bilingual dialogue language model by Tsinghua University",
+)
+
+register_model_info(
+    ["openchat-3.5"],
+    "OpenChat 3.5",
+    "https://github.com/imoneoi/openchat",
+    "an open model fine-tuned on Mistral-7B using C-RLFT",
+)
+
+register_model_info(
+    ["tenyxchat-7b-v1"],
+    "TenyxChat-7B",
+    "https://huggingface.co/tenyx/TenyxChat-7B-v1",
+    "an open model DPO trained on top of OpenChat-3.5 using Tenyx fine-tuning",
+)
+
+register_model_info(
+    ["zephyr-7b-beta", "zephyr-7b-alpha"],
+    "Zephyr",
+    "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha",
+    "a chatbot fine-tuned from Mistral by Hugging Face",
+)
+
+register_model_info(
+    ["notus-7b-v1"],
+    "Notus",
+    "https://huggingface.co/argilla/notus-7b-v1",
+    "a chatbot fine-tuned from Zephyr SFT by Argilla",
+)
+
+register_model_info(
+    ["catppt"],
+    "CatPPT",
+    "https://huggingface.co/rishiraj/CatPPT",
+    "a chatbot fine-tuned from a SLERP merged model by Rishiraj Acharya",
+)
+
+register_model_info(
+    ["TinyLlama"],
+    "TinyLlama",
+    "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "The TinyLlama project is an open endeavor to pretrain a 1.1B Llama model on 3 trillion tokens.",
+)
+
+register_model_info(
+    ["qwen-14b-chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-14B-Chat",
+    "a large language model by Alibaba Cloud",
+)
+
+register_model_info(
+    ["codellama-34b-instruct", "codellama-13b-instruct", "codellama-7b-instruct"],
+    "Code Llama",
+    "https://ai.meta.com/blog/code-llama-large-language-model-coding/",
+    "open foundation models for code by Meta",
+)
+
+register_model_info(
+    ["wizardlm-70b", "wizardlm-30b", "wizardlm-13b"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM",
+    "an instruction-following LLM using evol-instruct by Microsoft",
+)
+
+register_model_info(
+    ["wizardcoder-15b-v1.0"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder",
+    "Empowering Code Large Language Models with Evol-Instruct",
+)
+
+register_model_info(
+    ["mpt-7b-chat", "mpt-30b-chat"],
+    "MPT-Chat",
+    "https://www.mosaicml.com/blog/mpt-30b",
+    "a chatbot fine-tuned from MPT by MosaicML",
+)
+
+register_model_info(
+    ["guanaco-33b", "guanaco-65b"],
+    "Guanaco",
+    "https://github.com/artidoro/qlora",
+    "a model fine-tuned with QLoRA by UW",
+)
+
+register_model_info(
+    ["gpt4all-13b-snoozy"],
+    "GPT4All-Snoozy",
+    "https://github.com/nomic-ai/gpt4all",
+    "a finetuned LLaMA model on assistant style data by Nomic AI",
+)
+
+register_model_info(
+    ["koala-13b"],
+    "Koala",
+    "https://bair.berkeley.edu/blog/2023/04/03/koala",
+    "a dialogue model for academic research by BAIR",
+)
+
+register_model_info(
+    ["RWKV-4-Raven-14B"],
+    "RWKV-4-Raven",
+    "https://huggingface.co/BlinkDL/rwkv-4-raven",
+    "an RNN with transformer-level LLM performance",
+)
+
+register_model_info(
+    ["alpaca-13b"],
+    "Alpaca",
+    "https://crfm.stanford.edu/2023/03/13/alpaca.html",
+    "a model fine-tuned from LLaMA on instruction-following demonstrations by Stanford",
+)
+
+register_model_info(
+    ["oasst-pythia-12b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["oasst-sft-7-llama-30b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["palm-2"],
+    "PaLM 2 Chat",
+    "https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023",
+    "PaLM 2 for Chat (chat-bison@001) by Google",
+)
+
+register_model_info(
+    ["llama-7b", "llama-13b"],
+    "LLaMA",
+    "https://arxiv.org/abs/2302.13971",
+    "open and efficient foundation language models by Meta",
+)
+
+register_model_info(
+    ["open-llama-7b-v2-open-instruct", "open-llama-7b-open-instruct"],
+    "Open LLaMa (Open Instruct)",
+    "https://medium.com/vmware-data-ml-blog/starter-llm-for-the-enterprise-instruction-tuning-openllama-7b-d05fc3bbaccc",
+    "Open LLaMa fine-tuned on instruction-following data by VMware",
+)
+
+register_model_info(
+    ["dolly-v2-12b"],
+    "Dolly",
+    "https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm",
+    "an instruction-tuned open large language model by Databricks",
+)
+
+register_model_info(
+    ["stablelm-tuned-alpha-7b"],
+    "StableLM",
+    "https://github.com/stability-AI/stableLM",
+    "Stability AI language models",
+)
+
+register_model_info(
+    ["codet5p-6b"],
+    "CodeT5p-6b",
+    "https://huggingface.co/Salesforce/codet5p-6b",
+    "Code completion model released by Salesforce",
+)
+
+register_model_info(
+    ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],
+    "FastChat-T5",
+    "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0",
+    "a chat assistant fine-tuned from FLAN-T5 by LMSYS",
+)
+
+register_model_info(
+    ["phoenix-inst-chat-7b"],
+    "Phoenix-7B",
+    "https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b",
+    "a multilingual chat assistant fine-tuned from Bloomz to democratize ChatGPT across languages by CUHK(SZ)",
+)
+
+register_model_info(
+    ["realm-7b-v1"],
+    "ReaLM",
+    "https://github.com/FreedomIntelligence/ReaLM",
+    "A chatbot fine-tuned from LLaMA2 with data generated via iterative calls to UserGPT and ChatGPT by CUHK(SZ) and SRIBD.",
+)
+
+register_model_info(
+    ["billa-7b-sft"],
+    "BiLLa-7B-SFT",
+    "https://huggingface.co/Neutralzz/BiLLa-7B-SFT",
+    "an instruction-tuned bilingual LLaMA with enhanced reasoning ability by an independent researcher",
+)
+
+register_model_info(
+    ["h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2"],
+    "h2oGPT-GM-7b",
+    "https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
+    "an instruction-tuned OpenLLaMA with enhanced conversational ability by H2O.ai",
+)
+
+register_model_info(
+    ["baize-v2-7b", "baize-v2-13b"],
+    "Baize v2",
+    "https://github.com/project-baize/baize-chatbot#v2",
+    "A chatbot fine-tuned from LLaMA with ChatGPT self-chat data and Self-Disillation with Feedback (SDF) by UCSD and SYSU.",
+)
+
+register_model_info(
+    [
+        "airoboros-l2-7b-2.1",
+        "airoboros-l2-13b-2.1",
+        "airoboros-c34b-2.1",
+        "airoboros-l2-70b-2.1",
+    ],
+    "airoboros",
+    "https://huggingface.co/jondurbin/airoboros-l2-70b-2.1",
+    "an instruction-tuned LlaMa model tuned with 100% synthetic instruction-response pairs from GPT4",
+)
+
+register_model_info(
+    [
+        "spicyboros-7b-2.2",
+        "spicyboros-13b-2.2",
+        "spicyboros-70b-2.2",
+    ],
+    "spicyboros",
+    "https://huggingface.co/jondurbin/spicyboros-70b-2.2",
+    "de-aligned versions of the airoboros models",
+)
+
+register_model_info(
+    ["Robin-7b-v2", "Robin-13b-v2", "Robin-33b-v2"],
+    "Robin-v2",
+    "https://huggingface.co/OptimalScale/robin-7b-v2-delta",
+    "A chatbot fine-tuned from LLaMA-7b, achieving competitive performance on chitchat, commonsense reasoning and instruction-following tasks, by OptimalScale, HKUST.",
+)
+
+register_model_info(
+    ["manticore-13b-chat"],
+    "Manticore 13B Chat",
+    "https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg",
+    "A chatbot fine-tuned from LlaMa across several CoT and chat datasets.",
+)
+
+register_model_info(
+    ["redpajama-incite-7b-chat"],
+    "RedPajama-INCITE-7B-Chat",
+    "https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat",
+    "A chatbot fine-tuned from RedPajama-INCITE-7B-Base by Together",
+)
+
+register_model_info(
+    [
+        "falcon-7b",
+        "falcon-7b-instruct",
+        "falcon-40b",
+        "falcon-40b-instruct",
+        "falcon-180b",
+        "falcon-180b-chat",
+    ],
+    "Falcon",
+    "https://huggingface.co/tiiuae/falcon-180B",
+    "TII's flagship series of large language models",
+)
+
+register_model_info(
+    ["tigerbot-7b-sft"],
+    "Tigerbot",
+    "https://huggingface.co/TigerResearch/tigerbot-7b-sft",
+    "TigerBot is a large-scale language model (LLM) with multiple languages and tasks.",
+)
+
+register_model_info(
+    ["internlm-chat-7b", "internlm-chat-7b-8k"],
+    "InternLM",
+    "https://huggingface.co/internlm/internlm-chat-7b",
+    "InternLM is a multi-language large-scale language model (LLM), developed by SHLAB.",
+)
+
+register_model_info(
+    ["Qwen-7B-Chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-7B-Chat",
+    "Qwen is a multi-language large-scale language model (LLM), developed by Damo Academy.",
+)
+
+register_model_info(
+    ["Llama2-Chinese-13b-Chat", "LLama2-Chinese-13B"],
+    "Llama2-Chinese",
+    "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat",
+    "Llama2-Chinese is a multi-language large-scale language model (LLM), developed by FlagAlpha.",
+)
+
+register_model_info(
+    ["Chinese-Alpaca-2-7B", "Chinese-Alpaca-2-13B"],
+    "Chinese-Alpaca",
+    "https://huggingface.co/hfl/chinese-alpaca-2-13b",
+    "New extended Chinese vocabulary beyond Llama-2, open-sourcing the Chinese LLaMA-2 and Alpaca-2 LLMs.",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Instruct", "Vigogne-2-13B-Instruct"],
+    "Vigogne-Instruct",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-instruct",
+    "Vigogne-Instruct is a French large language model (LLM) optimized for instruction-following, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Chat", "Vigogne-2-13B-Chat"],
+    "Vigogne-Chat",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-chat",
+    "Vigogne-Chat is a French large language model (LLM) optimized for instruction-following and multi-turn dialogues, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["stable-vicuna-13B-HF"],
+    "stable-vicuna",
+    "https://huggingface.co/TheBloke/stable-vicuna-13B-HF",
+    "StableVicuna is a Vicuna model fine-tuned using RLHF via PPO on various conversational and instructional datasets.",
+)
+
+register_model_info(
+    ["deluxe-chat-v1", "deluxe-chat-v1.1", "deluxe-chat-v1.2"],
+    "DeluxeChat",
+    "",
+    "Deluxe Chat",
+)
+
+register_model_info(
+    [
+        "Xwin-LM-7B-V0.1",
+        "Xwin-LM-13B-V0.1",
+        "Xwin-LM-70B-V0.1",
+        "Xwin-LM-7B-V0.2",
+        "Xwin-LM-13B-V0.2",
+    ],
+    "Xwin-LM",
+    "https://github.com/Xwin-LM/Xwin-LM",
+    "Chat models developed by Xwin-LM team",
+)
+
+register_model_info(
+    ["lemur-70b-chat"],
+    "Lemur-Chat",
+    "https://huggingface.co/OpenLemur/lemur-70b-chat-v1",
+    "an openly accessible language model optimized for both natural language and coding capabilities ",
+)
+
+register_model_info(
+    ["Mistral-7B-OpenOrca"],
+    "Open-Orca",
+    "https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca",
+    "A fine-tune of [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) using [OpenOrca dataset](https://huggingface.co/datasets/Open-Orca/OpenOrca)",
+)
+
+register_model_info(
+    ["dolphin-2.2.1-mistral-7b"],
+    "dolphin-mistral",
+    "https://huggingface.co/ehartford/dolphin-2.2.1-mistral-7b",
+    "An uncensored fine-tuned Mistral 7B",
+)
+
+register_model_info(
+    [
+        "AquilaChat-7B",
+        "AquilaChat2-7B",
+        "AquilaChat2-34B",
+    ],
+    "Aquila-Chat",
+    "https://huggingface.co/BAAI/AquilaChat2-34B",
+    "Chat models developed by BAAI team",
+)
+
+register_model_info(
+    ["xDAN-L1-Chat-RL-v1"],
+    "xDAN-L1-Chat",
+    "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
+    "A large language chat model created by xDAN-AI.",
+)
+
+register_model_info(
+    ["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
+    "MetaMath",
+    "https://huggingface.co/meta-math",
+    "MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
+)
+
+register_model_info(
+    ["Yuan2-2B-hf", "Yuan2-51B-hf", "Yuan2-102B-hf"],
+    "IEIYuan",
+    "https://huggingface.co/IEITYuan",
+    "Yuan2 is a Basemodel developed by IEI.",
+)

arena_elo/elo_rating/upload_battle_data.py

@@ -0,0 +1,193 @@
+import fire
+import json
+import os
+import datasets
+import datetime
+from pathlib import Path
+from datetime import datetime
+from PIL import Image
+
+datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
+def create_hf_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model": datasets.Value("string"),
+                "conversation": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+
+def create_hf_battle_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model_a": datasets.Value("string"),
+                "model_b": datasets.Value("string"),
+                "conversation_a": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "conversation_b": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+                "anony": datasets.Value("bool"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+                           
+                                            
+
+
+def load_image(path:str):
+    try:
+        return Image.open(path)
+    except Exception as e:
+        print(f"Error loading image {path}: {e}")
+        return None
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d")
+    return date_str
+
+def load_battle_image(battle, log_dir):
+    image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
+    return load_image(image_path)
+    
+
+def main(
+    data_file: str = "./results/latest/clean_battle_conv.json",
+    repo_id: str = "DongfuTingle/wildvision-bench",
+    log_dir: str = os.getenv("LOGDIR", "./vision-arena-logs/"),
+    mode="battle",
+    token = os.environ.get("HUGGINGFACE_TOKEN", None)
+):
+    with open(data_file, "r") as f:
+        data = json.load(f)
+    
+    
+    
+    has_image_stats = {
+        "has_image": 0,
+        "no_image": 0,
+    }
+    if mode == "keep_bad_only":
+        # anony only
+        data = [d for d in data if d["anony"]]
+        
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                # we don't keep the data without image
+                continue
+            has_image_stats["has_image"] += 1
+            
+            if battle["winner"] in ["model_a", "model_b"]:
+                if battle["winner"] == "model_a":
+                    worse_model = "model_b"
+                    worse_conv = "conversation_b"
+                if battle["winner"] == "model_b":
+                    worse_model = "model_a"
+                    worse_conv = "conversation_a"
+                    
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle[worse_model],
+                    "conversation": battle[worse_conv],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+            elif battle["winner"] == "tie (bothbad)":
+                
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_a"],
+                    "conversation": battle["conversation_a"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_b"],
+                    "conversation": battle["conversation_b"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+                
+        split = "test"
+        hf_dataset = create_hf_dataset(new_data, "test")
+    
+    elif mode == "battle":
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                continue
+            has_image_stats["has_image"] += 1
+            new_data.append({
+                "question_id": battle["question_id"],
+                "model_a": battle["model_a"],
+                "model_b": battle["model_b"],
+                "conversation_a": battle["conversation_a"],
+                "conversation_b": battle["conversation_b"],
+                "language": battle["language"],
+                "image": image,
+                "turn": battle["turn"],
+                "anony": battle["anony"],
+            })
+        split = "test"
+        hf_dataset = create_hf_battle_dataset(new_data, "test")
+    else:
+        raise ValueError(f"Invalid mode: {mode}")
+
+    print(f"Stats: {has_image_stats}")
+    print(hf_dataset)
+    print(f"Uploading to part {repo_id}:{split}...")
+    hf_dataset.push_to_hub(
+        repo_id=repo_id,
+        config_name=mode,
+        split=split,
+        token=token,
+        commit_message=f"Add vision-arena {split} dataset",
+    )
+    
+    print("Done!")
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/utils.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+import pytz
+import PIL
+import os 
+
+def detect_language(text: str) -> str:
+    """Detect the langauge of a string."""
+    import polyglot  # pip3 install polyglot pyicu pycld2
+    from polyglot.detect import Detector
+    from polyglot.detect.base import logger as polyglot_logger
+    import pycld2
+
+    polyglot_logger.setLevel("ERROR")
+
+    try:
+        lang_code = Detector(text).language.name
+    except (pycld2.error, polyglot.detect.base.UnknownLanguage):
+        lang_code = "unknown"
+    return lang_code
+
+
+def get_time_stamp_from_date(date_str:str):
+    """
+    Convert a date string to a Unix timestamp
+    Args:
+        date_str (str): The input date string in the format 'YYYY-MM-DD-HH:MM-TZ', e.g. '2024-02-10-14:00-PT'
+    """
+    
+    # Convert the date string into a format that Python's datetime can understand
+    # and specify the correct timezone for PT, which is 'US/Pacific'
+    date_format = "%Y-%m-%d-%H:%M-%Z"
+
+    # Parse the date string into a datetime object
+    # Note: PT is not directly recognized by pytz, so we manually map it to 'US/Pacific'
+    timezone_map = {
+        "PT": "US/Pacific",
+    }
+
+    # Extract the timezone abbreviation
+    tz_abbr = date_str.split("-")[-1]
+    # Map the abbreviation to a pytz timezone
+    tz_info = pytz.timezone(timezone_map[tz_abbr])
+
+    # Remove the timezone abbreviation for parsing
+    date_str_parsed = date_str.rsplit("-", 1)[0]
+
+    # Create a datetime object with the corresponding timezone
+    dt = datetime.strptime(date_str_parsed, "%Y-%m-%d-%H:%M").replace(tzinfo=tz_info)
+
+    # Convert the datetime object to a Unix timestamp
+    unix_timestamp = dt.timestamp()
+    return unix_timestamp
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d %H:%M:%S %Z")
+    return date_str
+
+
+def get_input_image_path(tstamp, conv_id):
+    # from tstamp to date e.g. 2024-02-10
+    date_str = datetime.fromtimestamp(tstamp, tz=pytz.timezone("US/Pacific")).strftime("%Y-%m-%d")
+    LOGDIR = os.getenv("LOGDIR")
+    return f"{LOGDIR}/{date_str}-convinput_images/input_image_{conv_id}.png"
+
+def load_image_from_path(image_path):
+    # Load the image from the specified
+    # path using the Python Imaging Library (PIL)
+    try:
+        image = PIL.Image.open(image_path)
+        return image
+    except FileNotFoundError:
+        print(f"Image not found at path: {image_path}")
+        return None
+    except PIL.UnidentifiedImageError:
+        print(f"Unidentified image format at path: {image_path}")
+        return None
+    
+
+    

arena_elo/evaluator/convert_to_evaluator_data.py

@@ -0,0 +1,134 @@
+import argparse
+import json
+import os
+import time
+from pytz import timezone
+from tqdm import tqdm
+import base64
+from icecream import ic
+from PIL import Image
+
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    json_data = []
+
+    ic(log_files)
+    total_vote = 0
+
+    for filename in tqdm(log_files, desc="read files"):
+        for retry in range(5):
+            try:
+                lines = open(filename).readlines()
+                break
+            except FileNotFoundError:
+                time.sleep(2)
+        
+        for l in lines:
+            row = json.loads(l)
+            
+            if "states" not in row:
+                continue
+            if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                continue
+            
+            model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+            
+
+            # Iterate through each state and write the relevant information
+            if not len(row["states"][0]['messages']): continue
+            # ic(row["states"][0]['messages'][1][1])
+
+            if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+            total_vote += 1
+
+            conv_id = row["states"][0]['conv_id']
+            image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+            if not os.path.exists(image_path) :
+                continue
+            try:
+                image = Image.open(image_path).convert("RGB")
+            except:
+                continue
+
+            left_response = row["states"][0]['messages'][1][1]
+            right_response = row["states"][1]['messages'][1][1]
+            instruction = row["states"][0]['messages'][0][1]
+            generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+            text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+            
+            user_input = text_prompt
+            # Create the conversation structure
+            conversation = [
+                {
+                    "from": "human",
+                    "value": user_input
+                },
+                {
+                    "from": "gpt",
+                    "value": row["type"]
+                }
+            ]
+            
+            # Create the JSON object for each row
+            json_obj = {
+                "id": conv_id,
+                "image": image_path,
+                "conversations": conversation
+            }
+            
+            # Append the JSON object to the list
+            json_data.append(json_obj)
+
+        # Write the JSON data to a file
+        with open('output_evaluator_data.json', 'w') as json_file:
+            json.dump(json_data, json_file, indent=2)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)
+
+
+

arena_elo/evaluator/rating_analysis.ipynb

@@ -0,0 +1,321 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1338\n",
+      "1044\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "\n",
+    "# Replace 'your_file_name.csv' with the path to your CSV file\n",
+    "file_name = 'all_pairvote_log_wgpt.csv'\n",
+    "\n",
+    "# Load the CSV file into a DataFrame\n",
+    "df = pd.read_csv(file_name)\n",
+    "\n",
+    "# Define a function to parse JSON data\n",
+    "def parse_json(data):\n",
+    "    try:\n",
+    "        # Parse the JSON data\n",
+    "        return json.loads(data)\n",
+    "    except ValueError as e:\n",
+    "        # Return None or an empty dictionary if the data cannot be parsed\n",
+    "        return None\n",
+    "\n",
+    "# Apply the parse_json function to the 'models' and 'states' columns\n",
+    "df['models'] = df['models'].apply(parse_json)\n",
+    "df['states'] = df['states'].apply(parse_json)\n",
+    "# row[\"states\"][0]['messages'][0][1]\n",
+    "\n",
+    "# Now df contains the parsed JSON data in the 'models' and 'states' columns\n",
+    "# print(df.head())\n",
+    "print(len(df))\n",
+    "# filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"])]#, \"tievote\", \"bothbad_vote\"\n",
+    "# \\#1\n",
+    "filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])]\n",
+    "# \\#2\n",
+    "# filter_vote_df = df\n",
+    "filter_vote_df.loc[~filter_vote_df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]), \"gpt_vote\"] = \"tie\"\n",
+    "filter_vote_df.loc[~filter_vote_df[\"type\"].isin([\"leftvote\", \"rightvote\"]), \"type\"] = \"tie\"\n",
+    "# \\#3\n",
+    "#[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]) & df[\"type\"].isin([\"leftvote\", \"rightvote\"])]\n",
+    "filtered_df = filter_vote_df[filter_vote_df[\"states\"].apply(lambda x: len(x[0]['messages'][0][1]) > 10)]\n",
+    "print(len(filtered_df))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Confusion Matrix:\n",
+      "[[300  61  34]\n",
+      " [102 269  27]\n",
+      " [ 99 111  41]]\n",
+      "\n",
+      "Accuracy: 0.5842911877394636\n"
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming df is your DataFrame\n",
+    "\n",
+    "# True labels\n",
+    "y_true = filtered_df[\"type\"]\n",
+    "\n",
+    "# Predictions\n",
+    "y_pred = filtered_df[\"gpt_vote\"]\n",
+    "\n",
+    "# Compute the confusion matrix\n",
+    "# conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tie\"])\n",
+    "\n",
+    "# Compute the accuracy\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "\n",
+    "print(\"Confusion Matrix:\")\n",
+    "print(conf_matrix)\n",
+    "\n",
+    "print(\"\\nAccuracy:\", accuracy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0YAAAJwCAYAAACtcHEcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB0yUlEQVR4nO3deZyN5f/H8feZfcaYMZaxMwyGkTWJURSKKFshS/YtREQoe4oKaVFpsaRBZUlRUrLLUtYsk2VkaYSxjGEYZq7fH37O12ksc8aMM+N+PR+P83jMXPd9X+dzxrnHec913ddtM8YYAQAAAICFubm6AAAAAABwNYIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAWsWTJElWsWFE+Pj6y2Ww6c+ZMuvY/ffp02Ww2HTx4MF37zcpsNptGjhzp6jIApALBCECafPjhh7LZbHrwwQddXcodu/Zh7vfff7/h9kceeUT33XffXa7q3hQdHa3evXurVKlS8vPzk5+fn8LDw9WrVy9t377dYd+RI0fKZrPZH9f2HTp0qOLi4iTJYfutHitWrLhtbZcvX1Z4eLhsNpvGjx9vb+/Tp49sNpv27dt302NfffVV2Wy2FK/hRlasWKFmzZopX7588vLyUnBwsJ566inNnz//tsfeidjYWLVo0UK+vr6aPHmyZs6cqWzZsmXoc95NISEhstlsqlu37g23f/rpp/b3w83O9VtZt26dRo4cme5hEkDm4eHqAgBkTZGRkQoJCdHGjRu1b98+lShRwtUlIZNbtGiRWrZsKQ8PD7Vp00YVKlSQm5ub9uzZo/nz5+ujjz5SdHS0ihYt6nDcRx99JH9/f8XHx2vp0qV6/fXX9euvv2rt2rWaOXOmw75ffPGFfv755xTtZcqUuW1977//vg4dOpSivU2bNnr//fc1a9YsDR8+/IbHzp49W+XKlVP58uVv+RwjRozQ6NGjVbJkSXXv3l1FixZVbGysfvjhBz399NOKjIxU69atb1trWmzatEnnzp3Ta6+9dtPwcKeee+45Pfvss/L29s6Q/m/Hx8dHy5cv17Fjx5QvXz6HbZGRkfLx8dHFixfT1Pe6des0atQodejQQTly5Ej1cQkJCfLw4OMWkCUYAHDSgQMHjCQzf/58kydPHjNy5MhUHXf58mVz6dKlDK7OedOmTTOSzKZNm264vVatWqZs2bJ3uap7y759+0y2bNlMmTJlzD///JNi++XLl827775rDh06ZG8bMWKEkWROnDjhsG+zZs2MJLNu3boU/fTq1cuk5b+2f//91wQGBprRo0cbSebtt9922F6iRAlTunTpGx67bt06I8mMGzfuls/xzTffGEnmmWeeMYmJiSm2L1myxHz//fdO155aM2bMuOX7PKsrWrSoqVOnjgkICDCTJk1y2Hb48GHj5uZmnn766TT/DN5++20jyURHR99236SkJJOQkOD0cwBwLabSAXBaZGSkgoKC1LBhQz3zzDOKjIxMsc/BgwftU5ImTZqk0NBQeXt7a9euXZKkPXv26JlnnlHOnDnl4+OjKlWq6LvvvnPo49oUt7Vr16p///7KkyePsmXLpqZNm+rEiRMO+y5cuFANGzZUgQIF5O3trdDQUL322mtKSkpK99d/7bVNnz49xbb/Xk9wbTrYX3/9pbZt2yowMFB58uTRsGHDZIzR4cOH1bhxYwUEBChfvnyaMGGCQ3+JiYkaPny47r//fgUGBipbtmx6+OGHtXz58hvWNH78eH3yySf2n/cDDzygTZs23fL1/P7777LZbJoxY0aKbT/99JNsNpsWLVokSTp37pxefPFFhYSEyNvbW8HBwXrssce0efPmWz7HW2+9pfPnz2vatGnKnz9/iu0eHh7q06ePChcufMt+JKl27dqSrk7LSy+DBw9WWFiY2rZte8Ptbdq00Z49e274OmfNmiWbzaZWrVrd8jmGDRumnDlzaurUqfL09EyxvV69enryySft3x8/flydO3dW3rx55ePjowoVKqT4N0rtv/sjjzyi9u3bS5IeeOAB2Ww2dejQQdLVKWjXvr7eI488okceecSh7f3331fZsmXl5+enoKAgValSRbNmzbJvv9k1Rh9++KHKli0rb29vFShQQL169UoxJe3alNVdu3bp0UcflZ+fnwoWLKi33nrrZj/SFHx8fNSsWTOHmqSrI3pBQUGqV69eimO2b9+uDh06qHjx4vLx8VG+fPnUqVMnxcbG2vcZOXKkBg4cKEkqVqyYfUretddps9nUu3dvRUZG2l/nkiVL7Nuu/U5ISEhQ6dKlVbp0aSUkJNj7P3XqlPLnz6+IiIgM+Z0FIHUY2wXgtMjISDVr1kxeXl5q1aqVPvroI23atEkPPPBAin2nTZumixcvqlu3bvL29lbOnDm1c+dO1ahRQwULFtTgwYOVLVs2ff3112rSpInmzZunpk2bOvTxwgsvKCgoSCNGjNDBgwc1adIk9e7dW1999ZV9n+nTp8vf31/9+/eXv7+/fv31Vw0fPlxxcXF6++23U/W6zp49q5MnT6Zov3z5spM/oZRatmypMmXKaNy4cVq8eLHGjBmjnDlzasqUKapdu7befPNNRUZGasCAAXrggQdUs2ZNSVJcXJw+++wztWrVSl27dtW5c+f0+eefq169etq4caMqVqzo8DyzZs3SuXPn1L17d9lsNr311ltq1qyZDhw4cMMP45JUpUoVFS9eXF9//bX9w/M1X331lcMHyh49emju3Lnq3bu3wsPDFRsbqzVr1mj37t2qXLnyTV//okWLVKJEiXS5Jm3//v2SpFy5ct1xX5K0ceNGzZgxQ2vWrJHNZrvhPm3atNGoUaM0a9Ysh9eZlJSkr7/+Wg8//LCKFCly0+fYu3ev9uzZo06dOil79uy3rSkhIUGPPPKI9u3bp969e6tYsWL65ptv1KFDB505c0Z9+/Z12P92/+6vvvqqwsLC9Mknn2j06NEqVqyYQkNDU/kTuurTTz9Vnz599Mwzz6hv3766ePGitm/frg0bNtxy+t/IkSM1atQo1a1bV88//7yioqLsvzPWrl3r8L48ffq06tevr2bNmqlFixaaO3euBg0apHLlyumJJ55IVZ2tW7fW448/rv3799tf46xZs/TMM8/c8Bz4+eefdeDAAXXs2FH58uXTzp079cknn2jnzp1av369bDabmjVrpr/++kuzZ8/WO++8o9y5c0uS8uTJY+/n119/1ddff63evXsrd+7cCgkJSfFcvr6+mjFjhmrUqKFXX31VEydOlCT16tVLZ8+e1fTp0+Xu7p6q1wkgA7h6yApA1vL7778bSebnn382xhiTnJxsChUqZPr27euwX3R0tJFkAgICzPHjxx221alTx5QrV85cvHjR3pacnGwiIiJMyZIl7W3XprjVrVvXJCcn29v79etn3N3dzZkzZ+xtFy5cSFFr9+7djZ+fn8Pz3Mi157nV4/qpdNde27Rp01L0JcmMGDHC/v216WDdunWzt125csUUKlTI2Gw2h+lXp0+fNr6+vqZ9+/YO+/53+uHp06dN3rx5TadOnVLUlCtXLnPq1Cl7+8KFC42k207RGjJkiPH09HQ49tKlSyZHjhwOzxMYGGh69ep1y77+6+zZs0aSadKkSYptp0+fNidOnLA/rv93vPazi4qKMidOnDDR0dFmypQpxtvb2+TNm9ecP38+RX/OTqVLTk42VatWNa1atTLG/O/n+N+pdMYY88ADD5hChQqZpKQke9uSJUuMJDNlypRbPs+1f4d33nknVXVNmjTJSDJffvmlvS0xMdFUr17d+Pv7m7i4OId6U/PvfrMpo0WLFnV4z11Tq1YtU6tWLfv3jRs3vu2U0mvPcW262fHjx42Xl5d5/PHHHX5uH3zwgZFkpk6d6vB8kswXX3xhb7t06ZLJly+fefrpp2/5vNdeR8OGDc2VK1dMvnz5zGuvvWaMMWbXrl1Gklm5cuUNfwY3+t0xe/ZsI8msWrXK3narqXSSjJubm9m5c+cNt13/O8GYq+ebm5ubWbVqlX2K5X+n/wG4+5hKB8ApkZGRyps3rx599FFJV6eJtGzZUnPmzLnhFJCnn37a4a+qp06d0q+//qoWLVro3LlzOnnypE6ePKnY2FjVq1dPe/fu1dGjRx366Natm8Nf8h9++GElJSXp77//trf5+vrav77W78MPP6wLFy5oz549qXptkydP1s8//5zicbsL6lOjS5cu9q/d3d1VpUoVGWPUuXNne3uOHDkUFhamAwcOOOzr5eUlSUpOTtapU6d05coVValS5YbTulq2bKmgoCD79w8//LAkOfR5Iy1bttTly5cdVkZbunSpzpw5o5YtWzrUuGHDBv3zzz+pfen2FeT8/f1TbHvkkUeUJ08e+2Py5Mkp9gkLC1OePHlUrFgxde/eXSVKlNDixYvl5+eX6hpuZvr06dqxY4fefPPN2+7btm1bHTlyRKtWrbK3zZo1S15eXmrevPktj732M0jNaJEk/fDDD8qXL5/D9DxPT0/16dNH8fHxWrlypcP+af13d0aOHDl05MiR207NvN4vv/yixMREvfjii3Jz+99Hjq5duyogIECLFy922N/f399hOqOXl5eqVq3q1Otwd3dXixYtNHv2bElXf2cVLlzY/jP5r+t/d1y8eFEnT55UtWrVJOm2U0SvV6tWLYWHh6dq35EjR6ps2bJq3769evbsqVq1aqlPnz6pfi4AGYNgBCDVkpKSNGfOHD366KOKjo7Wvn37tG/fPj344IP6999/tWzZshTHFCtWzOH7ffv2yRijYcOGOXwgzpMnj0aMGCHp6rUV1/vvFKVrHwBPnz5tb9u5c6eaNm2qwMBABQQEKE+ePPYPWGfPnk3V66tatarq1q2b4nH9B860+u9rCAwMlI+Pj31KzvXt178uSZoxY4bKly8vHx8f5cqVS3ny5NHixYtv+LpS87O6kQoVKqh06dIO0xO/+uor5c6d235Nj3T1WqE///xThQsXVtWqVTVy5Mjbfmi9Fgbi4+NTbJsyZYp+/vlnffnllzc9ft68efr555+1YsUK7du3T3/++afuv//+Wz7n9eLj43Xs2DH749r1aXFxcRoyZIgGDhyYqmubnn32Wbm7u9uvX7l48aIWLFigJ5544rbvkYCAAElXQ3tq/P333ypZsqRDmJD+t7re9X8UkNL+7+6MQYMGyd/fX1WrVlXJkiXVq1cvrV279pbHXKszLCzMod3Ly0vFixdP8ToKFSqUYjpjUFCQ06+jdevW2rVrl7Zt26ZZs2bp2Wefvek0yVOnTqlv377KmzevfH197SFcSv3vDinl77pb8fLy0tSpUxUdHa1z585p2rRpN60PwN3DNUYAUu3XX39VTEyM5syZozlz5qTYHhkZqccff9yh7fq/xkpXRz0kacCAATe8EFpSiqW/bzbn3hgjSTpz5oxq1aqlgIAAjR49WqGhofLx8dHmzZs1aNAg+3Oml5t9gLnVRdM3eg23e12S9OWXX6pDhw5q0qSJBg4cqODgYLm7u2vs2LH2a22c7fNmWrZsqddff10nT55U9uzZ9d1336lVq1YOSw23aNFCDz/8sBYsWKClS5fq7bff1ptvvqn58+ff9BqQwMBA5c+fX3/++WeKbdeuObrVDUFr1qyZIkA6Y/z48Ro1apT9+6JFi+rgwYMaP368EhMT1bJlS/vzHzlyRNLVQHHw4EEVKFDAPmJ3baGJefPmafLkyfr+++917tw5tWnT5rY1lC5dWpK0Y8eONL+OW7mTf/dbvZ+v77dMmTKKiorSokWLtGTJEs2bN08ffvihhg8f7vDzvRN38jqu9+CDDyo0NFQvvviioqOjb3kNVIsWLbRu3ToNHDhQFStWlL+/v5KTk1W/fn2nfnf893fd7fz000+SrgbsvXv3OhWsAGQMghGAVIuMjFRwcPANpzvNnz9fCxYs0Mcff3zLDwjFixeXdHVaUHrdS2XFihWKjY3V/Pnz7YsWSOm7atn1rv01/r+rav33r9/pYe7cuSpevLjmz5/v8AH22uhaemrZsqVGjRqlefPmKW/evIqLi9Ozzz6bYr/8+fOrZ8+e6tmzp44fP67KlSvr9ddfv+XF8Q0bNtRnn32mjRs3qmrVqule+620a9dODz30kP37a+/PQ4cO6fTp0ypbtmyKY9544w298cYb2rJli8MCF23atNGSJUv0448/atasWQoICNBTTz112xpKlSqlsLAwLVy4UO++++4NpxVer2jRotq+fbuSk5MdRo2uTQv9772e7kRQUNANb1r6999/28/Xa7Jly6aWLVuqZcuWSkxMVLNmzfT6669ryJAh8vHxueHrkKSoqCiHvhITExUdHZ1h91OSpFatWmnMmDEqU6ZMikVKrjl9+rSWLVumUaNGOdyjau/evSn2Tc8Rne3bt2v06NHq2LGjtm7dqi5dumjHjh0KDAxMt+cA4Dym0gFIlYSEBM2fP19PPvmknnnmmRSP3r1769y5cymW3P6v4OBgPfLII5oyZYpiYmJSbP/vMtypce2vzNf/VTkxMVEffvih032lRkBAgHLnzu1wrYmkDHm+G722DRs26Lfffkv35ypTpozKlSunr776Sl999ZXy58/vEDSTkpJSTC0KDg5WgQIFdOnSpVv2/fLLL8vPz0+dOnXSv//+m2K7syMCzihevLjD1MgaNWpIkvr06aMFCxY4PKZMmSJJ6tChgxYsWJDir/hNmjSRn5+fPvzwQ/34449q1qzZDQPBjYwaNUqxsbHq0qWLrly5kmL70qVL7cuiN2jQQMeOHXOY2njlyhW9//778vf3V61atdL0s7iR0NBQrV+/XomJifa2RYsW6fDhww77Xb98tXR1Olh4eLiMMTddubFu3bry8vLSe++95/Bv/Pnnn+vs2bNq2LBhur2O/+rSpYtGjBiRYgn8693o/JKkSZMmpdg3W7ZsklL+QcRZly9fVocOHVSgQAG9++67mj59uv7991/169fvjvoFcOcYMQKQKt99953OnTunRo0a3XB7tWrVlCdPHkVGRjpcrH8jkydP1kMPPaRy5cqpa9euKl68uP7991/99ttvOnLkiLZt2+ZUbREREQoKClL79u3Vp08f2Ww2zZw5M0M/bHfp0kXjxo1Tly5dVKVKFa1atUp//fVXuj/Pk08+qfnz56tp06Zq2LChoqOj9fHHHys8PPyG1+zcqZYtW2r48OHy8fFR586dHUYrzp07p0KFCumZZ55RhQoV5O/vr19++UWbNm265YdPSSpZsqRmzZqlVq1aKSwsTG3atFGFChVkjFF0dLRmzZolNzc3FSpUKN1f081Urlw5xRLj16bUlS1bVk2aNElxjL+/v5o0aWK/zig10+iuadmypXbs2KHXX39dW7ZsUatWrVS0aFHFxsZqyZIlWrZsmb3fbt26acqUKerQoYP++OMPhYSEaO7cuVq7dq0mTZqU6kUcUqNLly6aO3eu6tevrxYtWmj//v368ssvUyzn/fjjjytfvnyqUaOG8ubNq927d+uDDz5Qw4YNb1pPnjx5NGTIEI0aNUr169dXo0aNFBUVpQ8//FAPPPDATe8blR6KFi3qcE+xGwkICFDNmjX11ltv6fLlyypYsKCWLl16w9Hma9e1vfrqq3r22Wfl6empp556yh6YUmvMmDHaunWrli1bpuzZs6t8+fIaPny4hg4dqmeeeUYNGjRwqj8A6YdgBCBVIiMj5ePjo8cee+yG293c3NSwYUNFRkam+Mvyf4WHh+v333/XqFGjNH36dMXGxio4OFiVKlVymM6SWrly5dKiRYv00ksvaejQoQoKClLbtm1Vp06dm17HdKeGDx+uEydOaO7cufr666/1xBNP6Mcff1RwcHC6Pk+HDh107NgxTZkyRT/99JPCw8P15Zdf6ptvvtGKFSvS9bmkqx/ehw4dqgsXLqQIuH5+furZs6eWLl2q+fPnKzk5WSVKlNCHH36o559//rZ9N27cWDt27NCECRO0dOlSTZ06VTabTUWLFlXDhg3Vo0cPVahQId1fU3pr06aNZs2apfz58zssTJEaY8aMUe3atfXee+/po48+0qlTpxQUFKRq1app4cKF9j88+Pr6asWKFRo8eLBmzJihuLg4hYWFadq0aTe8GeudqFevniZMmKCJEyfqxRdfVJUqVezn0/W6d++uyMhITZw4UfHx8SpUqJD69OmjoUOH3rL/kSNHKk+ePPrggw/Ur18/5cyZU926ddMbb7xx03tr3U2zZs3SCy+8oMmTJ8sYo8cff1w//vijChQo4LDfAw88oNdee00ff/yxlixZouTkZEVHRzsVjDZv3qw33nhDvXv3tq/sKV29wfDChQvVtWtX7dy5Uzly5EivlwfACTaTkX9SBQAAAIAsgGuMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFjePXmDV99KvV1dAuByvy0c6+oSAJcrXSC7q0sAALiYTyoTDyNGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACwv0wSj/fv3a+jQoWrVqpWOHz8uSfrxxx+1c+dOF1cGAAAA4F6XKYLRypUrVa5cOW3YsEHz589XfHy8JGnbtm0aMWKEi6sDAAAAcK/LFMFo8ODBGjNmjH7++Wd5eXnZ22vXrq3169e7sDIAAAAAVpApgtGOHTvUtGnTFO3BwcE6efKkCyoCAAAAYCWZIhjlyJFDMTExKdq3bNmiggULuqAiAAAAAFaSKYLRs88+q0GDBunYsWOy2WxKTk7W2rVrNWDAALVr187V5QEAAAC4x2WKYPTGG2+odOnSKly4sOLj4xUeHq6aNWsqIiJCQ4cOdXV5AAAAAO5xNmOMcXUR1xw+fFg7duxQfHy8KlWqpJIlS6apH99KvdO5MiDr+W3hWFeXALhc6QLZXV0CAMDFfDxSt1+mGDEaPXq0Lly4oMKFC6tBgwZq0aKFSpYsqYSEBI0ePdrV5QEAAAC4x2WKESN3d3fFxMQoODjYoT02NlbBwcFKSkpyqj9GjABGjACJESMAQBYbMTLGyGazpWjftm2bcubM6YKKAAAAAFhJKvNTxggKCpLNZpPNZlOpUqUcwlFSUpLi4+PVo0cPF1YIAAAAwApcGowmTZokY4w6deqkUaNGKTAw0L7Ny8tLISEhql69ugsrBAAAAGAFLg1G7du3lyQVK1ZMERER8vT0dGU5AAAAACzKpcHomlq1aikpKUnz5s3T7t27JUlly5ZVo0aN5O7u7uLqAAAAANzrMkUw2rdvnxo0aKCjR48qLCxMkjR27FgVLlxYixcvVmhoqIsrBAAAAHAvyxSr0vXp00ehoaE6fPiwNm/erM2bN+vQoUMqVqyY+vTp4+ryAAAAANzjMsWI0cqVK7V+/XqHpblz5cqlcePGqUaNGi6sDAAAAIAVZIpg5O3trXPnzqVoj4+Pl5eXlwsqwn91bf6Quj7zsIoWuBpedx84pjc++VFL1+6SJHl7eWhc/2ZqXu9+eXt56JffdqvvG1/p+Kn//bsWzhekd19pqVpVSik+4ZIiv9+gYe9/p6SkZJe8JiA9nDp5XJGfva+tG9fp0qWLylegkJ4fMEKhYeGSpA2rf9Uvi+bpwN49ij93Vm9+FKmQEmEurhrIeHNmRWrGtM918uQJlQorrcGvDFO58uVdXRZwV3EeZC2ZYirdk08+qW7dumnDhg0yxsgYo/Xr16tHjx5q1KiRq8uDpKP/ntGw9xcqos1bqtHmba3Y+Je+eaebyhTPJ0l6a8DTaljzPrV5+XM93mWS8ucJ1JwJXezHu7nZNP+95+Xl6aFHO0xQ1+Ez1bbRgxr+fENXvSTgjsWfi9PwFzvL3d1DQ954VxM/+1rPde+nbNkD7PtcupigsPsqqnWXF1xYKXB3LfnxB41/a6y69+ylOd8sUFhYaT3fvbNiY2NdXRpw13AeZD2ZIhi99957Cg0NVfXq1eXj4yMfHx/VqFFDJUqU0Lvvvuvq8iDph1V/6qc1u7T/0AntO3RcIyd/r/gLl1S1fDEF+PuoQ5PqGjRxvlZu+ktbdh9WtxFfqnrFUFUtFyJJqlu9jMoUz6dOr87Q9r+OaunaXRr94WJ1b1FTnh6sPIis6buvZihXnrzqOXCESpS+T8H5C6pClWrKV6CQfZ+ajzXUM891VbnKVV1YKXB3zZwxTc2eaaEmTZ9WaIkSGjpilHx8fPTt/HmuLg24azgPsp5MEYxy5MihhQsXKioqSnPnztXcuXMVFRWlBQsWONz0FZmDm5tNzevdr2y+XtqwPVqVyhSRl6eHfl0fZd/nr4P/6lDMKT1Yvpgk6cHyxfTnvn8cptb9vG63ArP7Kjw0/11/DUB6+P23VSpeqowmjh6krs0f06AerbXshwWuLgtwqcuJidq9a6eqVY+wt7m5ualatQht37bFhZUBdw/nQdaUKa4xWrNmjR566CGVLFlSJUuWdOrYS5cu6dKlSw5tJjlJNjdGIdJb2RIFtGLGS/Lx8lB8wiW1fOlT7TlwTBVKFdKlxMs6G5/gsP/x2DjlzXV1SlHeXAE6Hut4HdnxU3FXt+UOkKIEZDnHY47q5+/nqeHTbdS0dUftj9qlaZPHy8PDU7Uef9LV5QEucfrMaSUlJSlXrlwO7bly5VJ09AEXVQXcXZwHWVOmGDGqXbu2ihUrpldeeUW7du1y6tixY8cqMDDQ4XHl3z8yqFJr++vgv3rw2bGq2W68Pv1mjT4d/ZxK//81RoAVJZtkFStZWq0691KxEqVVt2Ez1WnQRD8vYpoEAABZTaYIRv/8849eeuklrVy5Uvfdd58qVqyot99+W0eOHLntsUOGDNHZs2cdHh55778LVVvP5StJOnD4pLbsPqzh73+nHX8dVa9Wj+hYbJy8vTwV6O/rsH9wrgD9G3t1VOjf2DgF58ruuD3n1dGkf0/G3Z0XAKSzoJy5VbBIMYe2gkWK6eTxYy6qCHC9oBxBcnd3T3GBeWxsrHLnzu2iqoC7i/Mga8oUwSh37tzq3bu31q5dq/3796t58+aaMWOGQkJCVLt27Vse6+3trYCAAIcH0+juDjebTd5eHtqy+5ASL1/Row/+bwnikkWDVSR/Tm3YHi1J2rA9WveVKKA8Qf72fepUK62z5xK0+wAfIpE1hZWtoJgjfzu0xRz5W3nyct0crMvTy0tlwstqw/rf7G3JycnasOE3la9QyYWVAXcP50HWlCmuMbpesWLFNHjwYFWoUEHDhg3TypUrXV0SJI1+oZF+WrtTh2NOK3s2H7V8oopqVimpp3p+qLj4i5r+7W9686VmOnX2vM6dv6iJg5pr/bYD2rjjoCTpl992a/eBY/p8THu9+u63ypsrQCN6PakpX69S4uUrrn1xQBo1eLq1hvftpAWzpqp6rce0L2qnlv2wQF1ffNW+T3zcWZ08fkynY09Ikv75/yCVI2cu5cjJXw1xb3qufUcNe2WQypa9T/eVK68vZ85QQkKCmjRt5urSgLuG8yDrsRljjKuLuGbt2rWKjIzU3LlzdfHiRTVu3Fht2rRR/fr1nerHt1LvDKrQuj4a0VqPVg1TvtwBOht/UX/uPaoJ037Rrxv2SPrfDV5b1P//G7yu262+Y7/Sv9ctuFAkf5DefeVZ1by/pM5fvKTI7zdq6HsLucFrBvlt4VhXl2AJf6xfrdmff6BjRw8rT74CevKZNqrToKl9+4qfvtdH40elOO6Z57qqebvud7NUSypdIPvtd0KGmB35pf3GlmGly2jQK0NVvnwFV5cF3FWcB5mDTyqHgjJFMBoyZIjmzJmjf/75R4899pjatGmjxo0by8/PL039EYwAghEgEYwAAKkPRpliKt2qVas0cOBAtWjRggvSAAAAANx1mSIYjR07VhEREfLwcCznypUrWrdunWrWrOmiygAAAABYQaZYle7RRx/VqVOnUrSfPXtWjz76qAsqAgAAAGAlmSIYGWNks9lStMfGxipbtmwuqAgAAACAlbh0Kl2zZleXK7TZbOrQoYO8vb3t25KSkrR9+3ZFRES4qjwAAAAAFuHSYBQYGCjp6ohR9uzZ5evra9/m5eWlatWqqWvXrq4qDwAAAIBFuCwY9e/fXx988IGyZcumgwcP6rPPPpO/v7+rygEAAABgYS67xuj9999XfHy8pKvLdV+4cMFVpQAAAACwOJeNGIWEhOi9997T448/LmOMfvvtNwUFBd1wX5brBgAAAJCRbMYY44on/vbbb9WjRw8dP35cNptNNyvDZrMpKSnJqb59K/VOjxKBLO23hWNdXQLgcqULZHd1CQAAF/NJ5VCQy0aMmjRpoiZNmig+Pl4BAQGKiopScHCwq8oBAAAAYGEuXZVOkvz9/bV8+XIVK1ZMHh4uLwcAAACABWWKG7zWqlVLf//9t4YOHapWrVrp+PHjkqQff/xRO3fudHF1AAAAAO51mSIYrVy5UuXKldOGDRs0f/58+2p127Zt04gRI1xcHQAAAIB7XaYIRoMHD9aYMWP0888/y8vLy95eu3ZtrV+/3oWVAQAAALCCTBGMduzYoaZNm6ZoDw4O1smTJ11QEQAAAAAryRTBKEeOHIqJiUnRvmXLFhUsWNAFFQEAAACwkkwRjJ599lkNGjRIx44dk81mU3JystauXasBAwaoXbt2ri4PAAAAwD0uUwSjN954Q6VLl1bhwoUVHx+v8PBwPfzww4qIiNDQoUNdXR4AAACAe5zNGGNcXcQ1hw8f1o4dOxQfH69KlSqpZMmSaerHt1LvdK4MyHp+WzjW1SUALle6QHZXlwAAcDGfVN4q1WV3VO3fv/8tt1+/Gt3EiRMzuhwAAAAAFuayYLRly5ZU7Wez2TK4EgAAAABW57JgtHz5clc9NQAAAAA4yBSLLwAAAACAKxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFiezRhjXF1Eevt66z+uLgFwufZD57u6BMDlTn/b29UlAABczMcjdfsxYgQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8jJNMFq9erXatm2r6tWr6+jRo5KkmTNnas2aNS6uDAAAAMC9LlMEo3nz5qlevXry9fXVli1bdOnSJUnS2bNn9cYbb7i4OgAAAAD3ukwRjMaMGaOPP/5Yn376qTw9Pe3tNWrU0ObNm11YGQAAAAAryBTBKCoqSjVr1kzRHhgYqDNnztz9ggAAAABYSqYIRvny5dO+fftStK9Zs0bFixd3QUUAAAAArCRTBKOuXbuqb9++2rBhg2w2m/755x9FRkZqwIABev75511dHgAAAIB7nIerC5CkwYMHKzk5WXXq1NGFCxdUs2ZNeXt7a8CAAXrhhRdcXR4AAACAe5zNGGNcXcQ1iYmJ2rdvn+Lj4xUeHi5/f/809fP11n/SuTIg62k/dL6rSwBc7vS3vV1dAgDAxXxSORSUKabSderUSefOnZOXl5fCw8NVtWpV+fv76/z58+rUqZOrywMAAABwj8sUwWjGjBlKSEhI0Z6QkKAvvvjCBRUBAAAAsBKXXmMUFxcnY4yMMTp37px8fHzs25KSkvTDDz8oODjYhRUCAAAAsAKXBqMcOXLIZrPJZrOpVKlSKbbbbDaNGjXKBZUBAAAAsBKXBqPly5fLGKPatWtr3rx5ypkzp32bl5eXihYtqgIFCriwQgAAAABW4NJgVKtWLUlSdHS0ChcuLDe3THHJEwAAAACLyRT3MSpatKjOnDmjzz//XLt375YklS1bVp06dVJgYKCLqwMAAABwr8sUQzS///67QkND9c477+jUqVM6deqUJk6cqNDQUG3evNnV5QEAAAC4x2WKEaN+/fqpUaNG+vTTT+XhcbWkK1euqEuXLnrxxRe1atUqF1cIAAAA4F6WKYLR77//7hCKJMnDw0Mvv/yyqlSp4sLKAAAAAFhBpphKFxAQoEOHDqVoP3z4sLJnz+6CigAAAABYSaYIRi1btlTnzp311Vdf6fDhwzp8+LDmzJmjLl26qFWrVq4uDwAAAMA9LlNMpRs/frxsNpvatWunK1euSJI8PT31/PPPa9y4cS6uDgAAAMC9zmaMMa4u4poLFy5o//79kqTQ0FD5+fmlqZ+vt/6TnmUBWVL7ofNdXQLgcqe/7e3qEgAALuaTyqGgTDFi9OWXX6pZs2by8/NTuXLlXF0ObuLgrm1a8/1X+if6L507HatWA15T+AMP2bcbY/TrN9P0+7LFung+XkXC7lOjLv2UK38hSdLp48e0Yv4XOvDnFsWfOaXsOXOrwkN1VatZW3l4eLrqZQGpNqD5/WpSvbhKFQpSQuIVbdh9TK9OX6e9R8847Pdg6Xwa+Vw1PRCWV0nJRtsPnNBTw7/TxcQkSVLF0Dwa06G67i95dfu36/Zr0GdrdP7iZRe8KiBjzJkVqRnTPtfJkydUKqy0Br8yTOXKl3d1WcBdxXmQtWSKa4z69eun4OBgtW7dWj/88IOSkpJcXRJuIPHSReUrGqonO/W94fbV383R+h/nq1GXfur++ofy8vHRjDde1uXEREnSyX8OySQbNe7aXy9MmKYn2vXUpl++1y+zP7ubLwNIs4fvK6CPF+9QrQFz9eSwhfLwcNOi1xrJz/t/f2N6sHQ+LRz1lJZtOaSH+3+jh/p9rY8X7VBy8tXB+fw5s2nxmMbaH3NWNV/6Ro1HfKfwIjn1ab86rnpZQLpb8uMPGv/WWHXv2UtzvlmgsLDSer57Z8XGxrq6NOCu4TzIejJFMIqJidGcOXNks9nUokUL5c+fX7169dK6detcXRquU6rSg6r7bGeFV304xTZjjH77Ya5qNXtOZR54SPmKhurpXkN07vRJ7d60RpJUsmJVNes5SCUqPKCceQuoTJUaeujJFtq1cfXdfilAmjQe8b2+XLZHuw+d0o7oWHV75xcVCQ5QpRLB9n3e6vKQPvx+u8bP3azdh05p79EzmrdmnxKvJEuSnnggRJevJOvFj1Zq79Ez+mPvcb0weYWa1iih4vkDXfXSgHQ1c8Y0NXumhZo0fVqhJUpo6IhR8vHx0bfz57m6NOCu4TzIejJFMPLw8NCTTz6pyMhIHT9+XO+8844OHjyoRx99VKGhoa4uD6lw+niM4s+cUmi5++1tPn7+KlSijA7v3XnT4y5eOC9ff5ZkR9YUkM1bknQ6/qIkKU+gr6qWzqcTZxK0/O2ndXBmJy0d21QR4fntx3h7uuvylSRdf3VnQuLVRWeu3w/Iqi4nJmr3rp2qVj3C3ubm5qZq1SK0fdsWF1YG3D2cB1lTpghG1/Pz81O9evX0xBNPqGTJkjp48KCrS0IqxJ85JUnyDwxyaM8WGGTf9l+xx45q/ZIFeqDuUxleH5DebDbp7a4Pa93Of7Tr76vv8WL5AiRJr7auqqk/7VLjEd9p6/4T+uH1JgotcHU0aMX2I8ob5Kd+zSrJ08NNObJ5a0yH6pKkfDmzuebFAOno9JnTSkpKUq5cuRzac+XKpZMnT7qoKuDu4jzImjJNMLpw4YIiIyPVoEEDFSxYUJMmTVLTpk21c+fNRxsk6dKlS4qLi3N4XE68dJeqRlrFnTqhL954WfdVq6UqdZ50dTmA0yY9X0tli+ZUu7d+sre52WySpM+X/KmZv+zWtgMn9fJna/TXkdNq/1i4JGn3oVPq+s4y9WlaUafm9dDBLzvp4LE4HTt9XiY50ywSCgCA5WSKVemeffZZLVq0SH5+fmrRooWGDRum6tWrp+rYsWPHatSoUQ5tz3Tvr+Y9XsqIUnET/jlySpLiz55W9qD//XXk/NnTyhdSwmHfuFMnNXV0fxUuVVaNuvHvhKznnR411eCBENUdPF9HY8/b22NOX/169yHHUdKow6dVOI+//fuvVv6lr1b+peAcvjp/8YqMMerTpKKij529Oy8AyEBBOYLk7u6e4gLz2NhY5c6d20VVAXcX50HWlClGjNzd3fX1118rJiZGH3zwQapDkSQNGTJEZ8+edXg06cR9K+62oOD88s+RUwd2bLa3XbxwXkf27VbhkmXtbXGnTmjq6H4qUKyUmvUcJDe3TPEWBFLtnR411ah6cdV/9Vv9/e85h21//3tO/8TGq1QhxymlJQrm0KHjjvtK0vEzCTp/8bKeqVlSFy8nadnWwxlaO3A3eHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8yJoyxYhRvXr1VKdOHbm7uzu0JyYmas6cOWrXrt1Nj/X29pa3t7dDm6dXfIbUaXWXLibo1LGj9u/PHI9RzMF98vXPrhy586p6g2e0YsFM5cxfUEHB+bXsq6nKHpRbZf7/Xkdxp07o81H9lCN3XtV/rofOx/3vr+PZ/3/ECcjMJj1fSy1rlVLzMYsVf+Gy8ua4ehPqsxcu2e9R9M68LRrapqp2RJ/UtgMn1bZOaYUVClLrsT/a++nxZDmt331M8QmXVadSYb3RMULDZvyms+cTXfK6gPT2XPuOGvbKIJUte5/uK1deX86coYSEBDVp2szVpQF3DedB1mMzxrh8Uru7u7tiYmIUHBzs0B4bG6vg4GCn72v09dZ/0rM8/L/onVs1dXS/FO2VatVTs56D/3eD118W6eKFeBUJK6enOr+o3AUKS5I2r1iiBR+9ecO+X/tqeYbWbkXth853dQn3nIRFNx6N7vrOL/py2R779wOeqazuDcspKLuPdkSf1KvT1mndrhj79s/611X9KiHy9/VU1JHTmjR/i2Yvj8rw+q3o9LfMIHCV2ZFf2m9sGVa6jAa9MlTly1dwdVnAXcV5kDn4pHIoKE3BaPXq1ZoyZYr279+vuXPnqmDBgpo5c6aKFSumhx56yNnu5Obmpn///Vd58uRxaN+2bZseffRRnTp141XNboZgBBCMAIlgBABIfTByeirdvHnz9Nxzz6lNmzbasmWLLl26ugLc2bNn9cYbb+iHH35IdV+VKlWSzWaTzWZTnTp15OHxv3KSkpIUHR2t+vXrO1siAAAAADjF6WA0ZswYffzxx2rXrp3mzJljb69Ro4bGjBnjVF9NmjSRJG3dulX16tWTv///Vm3y8vJSSEiInn76aWdLBAAAAACnOB2MoqKiVLNmzRTtgYGBOnPmjFN9jRgxQpIUEhKili1bysfHx9lyAAAAAOCOOR2M8uXLp3379ikkJMShfc2aNSpevHiaimjfvr2kq6vQHT9+XMnJyQ7bixQpkqZ+AQAAACA1nA5GXbt2Vd++fTV16lTZbDb9888/+u233zRgwAANGzYsTUXs3btXnTp10rp16xzajTGy2WxOr0oHAAAAAM5wOhgNHjxYycnJqlOnji5cuKCaNWvK29tbAwYM0AsvvJCmIjp06CAPDw8tWrRI+fPnl81mS1M/AAAAAJAWab6PUWJiovbt26f4+HiFh4c7LJzgrGzZsumPP/5Q6dKl09zH9ViuG2C5bkBiuW4AQOqX63ZztuNOnTrp3Llz8vLyUnh4uKpWrSp/f3+dP39enTp1crY7SVJ4eLhOnjyZpmMBAAAA4E45HYxmzJihhISEFO0JCQn64osvUt1PXFyc/fHmm2/q5Zdf1ooVKxQbG+uwLS4uztkSAQAAAMApqb7GKC4uTsYYGWN07tw5h6W1k5KS9MMPPyg4ODjVT5wjRw6Ha4mMMapTp47DPiy+AAAAAOBuSHUwuhZkbDabSpUqlWK7zWbTqFGjUv3Ey5cvT/W+AAAAAJCRUh2Mli9fLmOMateurXnz5ilnzpz2bV5eXipatKgKFCiQ6ieuVauWc5UCAAAAQAZJdTC6FmSio6NVuHBhubk5fXnSTW3fvv2G7TabTT4+PipSpIi8vb3T7fkAAAAA4HpO38eoaNGiOnPmjD7//HPt3r1bklS2bFl16tRJgYGBaSqiYsWKt7x3kaenp1q2bKkpU6Y4XNsEAAAAAOnB6WGf33//XaGhoXrnnXd06tQpnTp1ShMnTlRoaKg2b96cpiIWLFigkiVL6pNPPtHWrVu1detWffLJJwoLC9OsWbP0+eef69dff9XQoUPT1D8AAAAA3IrTN3h9+OGHVaJECX366afy8Lg64HTlyhV16dJFBw4c0KpVq5wuomrVqnrttddUr149h/affvpJw4YN08aNG/Xtt9/qpZde0v79+2/bHzd4BbjBKyBxg1cAQOpv8Or0VLrff//dIRRJkoeHh15++WVVqVLF2e4kSTt27FDRokVTtBctWlQ7duyQdHW6XUxMTJr6BwAAAIBbcXoqXUBAgA4dOpSi/fDhw8qePXuaiihdurTGjRunxMREe9vly5c1btw4lS5dWpJ09OhR5c2bN039AwAAAMCtOD1i1LJlS3Xu3Fnjx49XRESEJGnt2rUaOHCgWrVqlaYiJk+erEaNGqlQoUIqX768pKujSElJSVq0aJEk6cCBA+rZs2ea+gcAAACAW3E6GI0fP142m03t2rXTlStXJF1dNe7555/XuHHj0lRERESEoqOjFRkZqb/++kuS1Lx5c7Vu3do+CvXcc8+lqW8AAAAAuJ1UL75QpUoVdenSRa1bt1ZAQIAuXLhgXwghNDRUfn5+GVqoM1h8AWDxBUBi8QUAQAYsvlChQgW9/PLLeumll/T000+rU6dOeuSRR9JYnvTdd9/piSeekKenp7777rtb7tuoUaM0Pw8AAAAA3I5Ty3VfuHBBX3/9taZPn67Vq1erWLFi6tSpk9q3b6+CBQs69cRubm46duyYgoOD5eZ28zUgbDabkpKSnOqbESOAESNAYsQIAJD6ESOnVqXz8/NThw4dtGLFCv3111969tlnNWXKFIWEhKhhw4aaPz/1H8SSk5MVHBysy5cv65FHHtGePXuUnJyc4uFsKAIAAAAAZzm9XPc1oaGhGjNmjA4ePKjZs2dr/fr1at68udP9eHp6aseOHbccNQIAAACAjHRHaWTFihXq0KGDOnTooKSkJHXt2jVN/bRt21afffbZnZQCAAAAAGnm9HLdR44c0fTp0zV9+nQdOHBADz/8sD788EM1b95cvr6+aSriypUrmjp1qn755Rfdf//9ypYtm8P2iRMnpqlfAAAAAEiNVAejr7/+WlOnTtWyZcsUHBys9u3bq1OnTipRosQdF/Hnn3+qcuXKkmS/j9E1NpvtjvsHAAAAgFtJdTBq27atGjZsqAULFqhBgwbpek3Q8uXL060vAAAAAHBWqoPRkSNHFBwcnJG1AAAAAIBLpHrYh1AEAAAA4F7FGtkAAAAALI9gBAAAAMDyCEYAAAAALM/p+xhdk5iYqOPHjys5OdmhvUiRIndcFAAAAADcTU4Ho71796pTp05at26dQ7sxRjabTUlJSelWHAAAAADcDU4How4dOsjDw0OLFi1S/vz5uQErAAAAgCzP6WC0detW/fHHHypdunRG1AMAAAAAd53Tiy+Eh4fr5MmTGVELAAAAALiE08HozTff1Msvv6wVK1YoNjZWcXFxDg8AAAAAyGqcnkpXt25dSVKdOnUc2ll8AQAAAEBW5XQwWr58eUbUAQAAAAAu43QwqlWrVkbUAQAAAAAuk+YbvF64cEGHDh1SYmKiQ3v58uXvuCgAAAAAuJucDkYnTpxQx44d9eOPP95wO9cYAQAAAMhqnF6V7sUXX9SZM2e0YcMG+fr6asmSJZoxY4ZKliyp7777LiNqBAAAAIAM5fSI0a+//qqFCxeqSpUqcnNzU9GiRfXYY48pICBAY8eOVcOGDTOiTgAAAADIME6PGJ0/f17BwcGSpKCgIJ04cUKSVK5cOW3evDl9qwMAAACAu8DpYBQWFqaoqChJUoUKFTRlyhQdPXpUH3/8sfLnz5/uBQIAAABARnN6Kl3fvn0VExMjSRoxYoTq16+vyMhIeXl5afr06eldHwAAAABkOKeDUdu2be1f33///fr777+1Z88eFSlSRLlz507X4gAAAADgbkjzfYyu8fPzU+XKldOjFgAAAABwCaeDkTFGc+fO1fLly3X8+HElJyc7bJ8/f366FQcAAAAAd4PTwejFF1/UlClT9Oijjypv3ryy2WwZURcAAAAA3DVOB6OZM2dq/vz5atCgQUbUAwAAAAB3ndPLdQcGBqp48eIZUQsAAAAAuITTwWjkyJEaNWqUEhISMqIeAAAAALjrnJ5K16JFC82ePVvBwcEKCQmRp6enw/bNmzenW3EAAAAAcDc4HYzat2+vP/74Q23btmXxBQAAAAD3BKeD0eLFi/XTTz/poYceyoh6AAAAAOCuc/oao8KFCysgICAjagEAAAAAl3A6GE2YMEEvv/yyDh48mAHlAAAAAMDdZzPGGGcOCAoK0oULF3TlyhX5+fmlWHzh1KlT6VpgWly84uoKAAAAAGQGPqm8eMjpa4wmTZrk7CEAAAAAkKk5PWKUFTBiBAAAAEDKwBGjQ4cO3XJ7kSJFnO0SAAAAAFzK6REjNze3W967KCkp6Y6LulOMGAEAAACQMnDEaMuWLQ7fX758WVu2bNHEiRP1+uuvO9sdAAAAALhcul1jtHjxYr399ttasWJFenR3RxgxAgAAACClfsTI6fsY3UxYWJg2bdqUXt0BAAAAwF3j9FS6uLg4h++NMYqJidHIkSNVsmTJdCsMAAAAAO4Wp4NRjhw5Uiy+YIxR4cKFNWfOnHQrDAAAAADuFqeD0fLlyx2+d3NzU548eVSiRAl5eDjdHQAAAAC4HDd4BQAAAHDPSvflur/77rtU7deoUaPUdgkAAAAAmUKqR4zc3BwXsLPZbPrvoTabjRu8AgAAAMg00n257uTkZIeHn5+f9u3b59CWGUIRAAAAADgr3e5jBAAAAABZFcEIAAAAgOURjAAAAABYXpqDkc1mS3GjVwAAAADIilK9Kl1QUJBDEDpz5owCAgJSrFZ36tSp9K0wDViVDgAAAICUAfcxmjRpUhpLAQAAAIDMLdUjRlkJI0YAAAAApAy4jxEAAAAA3KsIRgAAAAAsj2AEAAAAwPIIRgAAAAAsj2AEAAAAwPJSvVz39Y4cOaLvvvtOhw4dUmJiosO2iRMnpkthAAAAAHC3OB2Mli1bpkaNGql48eLas2eP7rvvPh08eFDGGFWuXDkjagQAAACADOX0VLohQ4ZowIAB2rFjh3x8fDRv3jwdPnxYtWrVUvPmzTOiRgAAAADIUE7f4DV79uzaunWrQkNDFRQUpDVr1qhs2bLatm2bGjdurIMHD2ZQqanHDV4BAAAASBl4g9ds2bLZryvKnz+/9u/fb9928uRJZ7sDAAAAAJdz+hqjatWqac2aNSpTpowaNGigl156STt27ND8+fNVrVq1jKgRAAAAADKU01PpDhw4oPj4eJUvX17nz5/XSy+9pHXr1qlkyZKaOHGiihYtmlG1phpT6QAAAABIqZ9K53QwygoIRgAAAACk1AejNN3HSJISExN1/PhxJScnO7QXKVIkrV0CAAAAgEs4HYz++usvde7cWevWrXNoN8bIZrMpKSkp3YoDAAAAgLvB6WDUsWNHeXh4aNGiRcqfP79sNltG1AUAAAAAd43T1xhly5ZNf/zxh0qXLp1RNd0xrjECAAAAIGXgfYzCw8O5XxEAAACAe4rTwejNN9/Uyy+/rBUrVig2NlZxcXEODwAAAADIapyeSufmdjVL/ffaosy0+AJT6QAAAABIGbhc9/Lly509BAAAAAAyNW7wCgAAAOCelWGLL0jS6tWr1bZtW0VEROjo0aOSpJkzZ2rNmjVp6Q4AAAAAXMrpYDRv3jzVq1dPvr6+2rx5sy5duiRJOnv2rN544410LxAAAAAAMprTwWjMmDH6+OOP9emnn8rT09PeXqNGDW3evDldiwMAAACAu8HpYBQVFaWaNWumaA8MDNSZM2fSVMTMmTNVo0YNFShQQH///bckadKkSVq4cGGa+gMAAAAAZzgdjPLly6d9+/alaF+zZo2KFy/udAEfffSR+vfvrwYNGujMmTP25b5z5MihSZMmOd0fAAAAADjL6WDUtWtX9e3bVxs2bJDNZtM///yjyMhIDRgwQM8//7zTBbz//vv69NNP9eqrr8rd3d3eXqVKFe3YscPp/gAAAADAWU7fx2jw4MFKTk5WnTp1dOHCBdWsWVPe3t4aMGCAXnjhBacLiI6OVqVKlVK0e3t76/z58073BwAAAADOcioYJSUlae3aterVq5cGDhyoffv2KT4+XuHh4fL3909TAcWKFdPWrVtVtGhRh/YlS5aoTJkyaeoTAAAAAJzhVDByd3fX448/rt27dytHjhwKDw+/4wL69++vXr166eLFizLGaOPGjZo9e7bGjh2rzz777I77BwAAAIDbcXoq3X333acDBw6oWLFi6VJAly5d5Ovrq6FDh+rChQtq3bq1ChQooHfffVfPPvtsujwHAAAAANyKzRhjnDlgyZIlGjJkiF577TXdf//9ypYtm8P2gICANBdz4cIFxcfHKzg4OM19SNLFK3d0OAAAAIB7hE8qh4KcDkZubv9byM5ms9m/NsbIZrPZl9tOrdq1a2v+/PnKkSOHQ3tcXJyaNGmiX3/91an+JIIRAAAAgKtSG4ycnkq3fPlyZw+5pRUrVigxMTFF+8WLF7V69ep0fS4AAAAAuBGng1GtWrVuuu3PP/9MdT/bt2+3f71r1y4dO3bM/n1SUpKWLFmiggULOlseAAAAADjN6al0/3Xu3DnNnj1bn332mf74449UT6Vzc3OzT8W7UQm+vr56//331alTJ6drYiodAAAAACkDp9Jds2rVKn3++eeaN2+eChQooGbNmmny5MmpPj46OlrGGBUvXlwbN25Unjx57Nu8vLwUHBwsd3f3tJYHAAAAAKnmVDA6duyYpk+frs8//1xxcXFq0aKFLl26pG+//dbpexpdu6FrcnKyU8cBAAAAQHpzu/0uVz311FMKCwvT9u3bNWnSJP3zzz96//3306WI/fv364UXXlDdunVVt25d9enTR/v370+XvgEAAADgdlIdjH788Ud17txZo0aNUsOGDdNtmttPP/2k8PBwbdy4UeXLl1f58uW1YcMGlS1bVj///HO6PAfurjmzIvXEY7X1QKVyavNsc+24bqENwCo4D2B1nAMA50FWk+pgtGbNGp07d07333+/HnzwQX3wwQc6efLkHRcwePBg9evXTxs2bNDEiRM1ceJEbdiwQS+++KIGDRp0x/3j7lry4w8a/9ZYde/ZS3O+WaCwsNJ6vntnxcbGuro04K7hPIDVcQ4AnAdZkdOr0p0/f15fffWVpk6dqo0bNyopKUkTJ05Up06dlD17dqcL8PHx0Y4dO1SyZEmH9r/++kvly5fXxYsXne6TVelcp82zzVX2vnJ6ZehwSVevIXu8Ti21av2cOnft5uLqgLuD8wBWxzkAcB5kJqldlS7VI0bXZMuWTZ06ddKaNWu0Y8cOvfTSSxo3bpyCg4PVqFEjZ7tTnjx5tHXr1hTtW7duVXBwsNP9wXUuJyZq966dqlY9wt7m5uamatUitH3bFhdWBtw9nAewOs4BgPMgq3I6GF0vLCxMb731lo4cOaLZs2enqY+uXbuqW7duevPNN7V69WqtXr1a48aNU/fu3dW1a9fbHn/p0iXFxcU5PC5dupSmWnBnTp85raSkJOXKlcuhPVeuXOky7RLICjgPYHWcAwDnQVaV5vsYXc/d3V1NmjRRkyZNnD522LBhyp49uyZMmKAhQ4ZIkgoUKKCRI0eqT58+tz1+7NixGjVqlEPbq8NGaOjwkU7XAgAAAMCa0iUY3QmbzaZ+/fqpX79+OnfunCQ5da3SkCFD1L9/f4c24+6drjUidYJyBMnd3T3FRYWxsbHKnTu3i6oC7i7OA1gd5wDAeZBV3dFUuvQwZswYRUdHS7oaiJxdwMHb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1mVy4PRN998oxIlSigiIkIffvgh8y6zuOfad9T8uV/ru28X6MD+/RozeqQSEhLUpGkzV5cG3DWcB7A6zgGA8yArcnq57oywc+dORUZGas6cOTpy5Igee+wxtWnTRk2aNJGfn5/T/bFct2vNjvxSM6Z9rpMnTyisdBkNemWoypev4OqygLuK8wBWxzkAcB5kFqldrjtTBKPrrV27VrNmzdI333yjixcvKi4uzuk+CEYAAAAApAy8j1FGy5Ytm3x9feXl5aXLly+7uhwAAAAAFpApglF0dLRef/11lS1bVlWqVNGWLVs0atQoHTt2zNWlAQAAALAAly/XXa1aNW3atEnly5dXx44d1apVKxUsWNDVZQEAAACwEJcHozp16mjq1KkKDw93dSkAAAAALCrTLL6QmJio6OhohYaGysPjzvIaiy8AAAAAkLLQ4gsJCQnq3Lmz/Pz8VLZsWR06dEiS9MILL2jcuHEurg4AAACAFbg8GA0ePFjbtm3TihUr5OPjY2+vW7euvvrqKxdWBgAAAMAqXH6N0bfffquvvvpK1apVk81ms7eXLVtW+/fvd2FlAAAAAKzC5SNGJ06cUHBwcIr28+fPOwQlAAAAAMgoLg9GVapU0eLFi+3fXwtDn332mapXr+6qsgAAAABYiMun0r3xxht64okntGvXLl25ckXvvvuudu3apXXr1mnlypWuLg8AAACABbh8xOihhx7S1q1bdeXKFZUrV05Lly5VcHCwfvvtN91///2uLg8AAACABWSa+xilJ+5jBAAAAEBK/X2MXDKVLi4uTgEBAfavb+XafgAAAACQUVwyYuTu7q6YmBgFBwfLzc3thqvPGWNks9mUlJTkdP+MGAEAAACQMvmI0a+//qqcOXNKkqZNm6bChQvL3d3dYZ/k5GQdOnTIFeUBAAAAsBiXX2N0/ejR9WJjYxUcHMyIEQAAAIA0S+2IkctXpbs2Ze6/4uPj5ePj44KKAAAAAFiNy+5j1L9/f0lXb+g6bNgw+fn52bclJSVpw4YNqlixoouqAwAAAGAlLgtGW7ZskXR1xGjHjh3y8vKyb/Py8lKFChU0YMAAV5UHAAAAwEJcfo1Rx44d9e6776brstxcYwQAAABASv01Ri4PRhmBYAQAAABAykKLLwAAAACAqxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5WW6YHTx4kVXlwAAAADAYjJFMEpOTtZrr72mggULyt/fXwcOHJAkDRs2TJ9//rmLqwMAAABwr8sUwWjMmDGaPn263nrrLXl5ednb77vvPn322WcurAwAAACAFWSKYPTFF1/ok08+UZs2beTu7m5vr1Chgvbs2ePCygAAAABYQaYIRkePHlWJEiVStCcnJ+vy5csuqAgAAACAlWSKYBQeHq7Vq1enaJ87d64qVarkgooAAAAAWImHqwuQpOHDh6t9+/Y6evSokpOTNX/+fEVFRemLL77QokWLXF0eAAAAgHuczRhjXF2EJK1evVqjR4/Wtm3bFB8fr8qVK2v48OF6/PHHne7r4pUMKBAAAABAluOTyqGgTBOM0hPBCAAAAICU+mCUKa4xKl68uGJjY1O0nzlzRsWLF3dBRQAAAACsJFMEo4MHDyopKSlF+6VLl3T06FEXVAQAAADASly6+MJ3331n//qnn35SYGCg/fukpCQtW7ZMISEhLqgMAAAAgJW49BojN7erA1Y2m03/LcPT01MhISGaMGGCnnzySaf65RojAAAAAFLqrzFy6YhRcnKyJKlYsWLatGmTcufO7cpyAAAAAFgUq9IBAAAAuGdlqVXpJGnlypV66qmnVKJECZUoUUKNGjXS6tWrXV0WAAAAAAvIFMHoyy+/VN26deXn56c+ffqoT58+8vX1VZ06dTRr1ixXlwcAAADgHpcpptKVKVNG3bp1U79+/RzaJ06cqE8//VS7d+92qj+m0gEAAACQUj+VLlMEI29vb+3cuVMlSpRwaN+3b5/uu+8+Xbx40an+CEYAAAAApCx2jVHhwoW1bNmyFO2//PKLChcu7IKKAAAAAFiJS5frvuall15Snz59tHXrVkVEREiS1q5dq+nTp+vdd991cXUAAAAA7nWZYiqdJC1YsEATJkywX09UpkwZDRw4UI0bN3a6L6bSAQAAAJCy2DVG6Y1gBAAAAEDKYtcYdenSRStWrHB1GQAAAAAsKlMEoxMnTqh+/foqXLiwBg4cqK1bt7q6JAAAAAAWkmmm0p0+fVrffPONZs2apdWrV6t06dJq06aNWrdurZCQEKf6YiodAAAAACmLX2N05MgRzZ49W1OnTtXevXt15YpzSYdgBAAAAEDKYtcYXe/y5cv6/ffftWHDBh08eFB58+Z1dUkAAAAA7nGZJhgtX75cXbt2Vd68edWhQwcFBARo0aJFOnLkiKtLAwAAAHCPyxRT6QoWLKhTp06pfv36atOmjZ566il5e3unuT+m0gEAAACQstg1Rp9++qmaN2+uHDly3HK/I0eOqECBAnJzu/VAF8EIAAAAgJTFglFqBQQEaOvWrSpevPgt9yMYAQAAAJCy8OILt5KFMhwAAACALCRLBSMAAAAAyAgEIwAAAACWRzACAAAAYHlZKhjZbDZXlwAAAADgHpSlghGLLwAAAADICFlque7Dhw+rQIECcnd3v+V+LNcNAAAAQMoC9zFq1qxZqvedP3++U30TjFxrzqxIzZj2uU6ePKFSYaU1+JVhKle+vKvLAu4qzgNYHecAwHmQWWT6+xgFBgbaHwEBAVq2bJl+//13+/Y//vhDy5YtU2BgoKtKRBos+fEHjX9rrLr37KU53yxQWFhpPd+9s2JjY11dGnDXcB7A6jgHAM6DrChTTKUbNGiQTp06pY8//tg+TS4pKUk9e/ZUQECA3n77baf6Y8TIddo821xl7yunV4YOlyQlJyfr8Tq11Kr1c+rctZuLqwPuDs4DWB3nAMB5kJlk+hGj602dOlUDBgxwuHbI3d1d/fv319SpU11YGZxxOTFRu3ftVLXqEfY2Nzc3VasWoe3btriwMuDu4TyA1XEOAJwHWVWmCEZXrlzRnj17UrTv2bNHycnJtzz20qVLiouLc3hcunQpo0rFLZw+c1pJSUnKlSuXQ3uuXLl08uRJF1UF3F2cB7A6zgGA8yCryhTBqGPHjurcubMmTpyoNWvWaM2aNZowYYK6dOmijh073vLYsWPHOlyvFBgYqLffHHuXKgcAAABwL0jljLuMNX78eOXLl08TJkxQTEyMJCl//vwaOHCgXnrppVseO2TIEPXv39+hzbh7Z1ituLmgHEFyd3dPcVFhbGyscufO7aKqgLuL8wBWxzkAcB5kVZlixMjNzU0vv/yyjh49qjNnzujMmTM6evSoXn755dves8jb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1lVphgxul5AQICrS8AdeK59Rw17ZZDKlr1P95Urry9nzlBCQoKaNE39fauArI7zAFbHOQBwHmRFmSYYzZ07V19//bUOHTqkxMREh22bN292UVVwVv0nGuj0qVP68IP3dPLkCYWVLqMPp3ymXAwbw0I4D2B1nAMA50FWlCnuY/Tee+/p1VdfVYcOHfTJJ5+oY8eO2r9/vzZt2qRevXrp9ddfd6o/7mMEAAAAQEr9fYwyRTAqXbq0RowYoVatWil79uzatm2bihcvruHDh+vUqVP64IMPnOqPYAQAAABAymI3eD106JAiIq7eAMvX11fnzp2TJD333HOaPXu2K0sDAAAAYAGZIhjly5dPp06dkiQVKVJE69evlyRFR0crEwxoAQAAALjHZYpgVLt2bX333XeSrt7stV+/fnrsscfUsmVLNW3a1MXVAQAAALjXZYprjJKTk5WcnCwPj6sTAOfMmaN169apZMmS6t69u7y8vJzqj2uMAAAAAEhZbPGF9EYwAgAAACClPhhlmvsYnT59Wp9//rl2794tSQoPD1fHjh2VM2dOF1cGAAAA4F6XKUaMVq1apUaNGikgIEBVqlSRJP3xxx86c+aMvv/+e9WsWdOp/hgxAgAAACBlsal05cqVU/Xq1fXRRx/J3d1dkpSUlKSePXtq3bp12rFjh1P9EYwAAAAASFksGPn6+mrr1q0KCwtzaI+KilLFihWVkJDgVH8EIwAAAABSFrvBa+XKle3XFl1v9+7dqlChggsqAgAAAGAlLlt8Yfv27fav+/Tpo759+2rfvn2qVq2aJGn9+vWaPHmyxo0b56oSAQAAAFiEy6bSubm5yWaz6XZPb7PZlJSU5FTfTKUDAAAAIGWB5bqjo6Nd9dQAAAAA4CBTLL6Q3hgxAgAAACBlgRGj/9q7d6+WL1+u48ePKzk52WHb8OHDXVQVAAAAACvIFCNGn376qZ5//nnlzp1b+fLlk81ms2+z2WzavHmzU/0xYgQAAABAymL3MSpatKh69uypQYMGpUt/BCMAAAAAUha7j9Hp06fVvHlzV5cBAAAAwKIyRTBq3ry5li5d6uoyAAAAAFiUy6bSvffee/avz58/r4kTJ6phw4YqV66cPD09Hfbt06ePU30zlQ4AAACAlAWuMSpWrFiq9rPZbDpw4IBTfROMAAAAAEhZIBhlJIIRAAAAACmLLb4wevRoXbhwIUV7QkKCRo8e7YKKAAAAAFhJphgxcnd3V0xMjIKDgx3aY2NjFRwcrKSkJKf6Y8QIAAAAgJTFRoyMMQ43db1m27ZtypkzpwsqAgAAAGAlqcxPGSMoKEg2m002m02lSpVyCEdJSUmKj49Xjx49XFghAAAAACtw6VS6GTNmyBijTp06adKkSQoMDLRv8/LyUkhIiKpXr+50v0ylAwAAACBlsVXpVq5cqYiIiBT3L0orghEAAAAAKYsFI+nq1Llvv/1Wu3fvliSVLVtWjRo1kru7u9N9EYwAAAAASFksGO3bt08NGjTQ0aNHFRYWJkmKiopS4cKFtXjxYoWGhjrVH8EIAAAAgJTFglGDBg1kjFFkZKR9FbrY2Fi1bdtWbm5uWrx4sVP9EYwAAAAASFksGGXLlk3r169XuXLlHNq3bdumGjVqKD4+3qn+CEYAAAAApCx2HyNvb2+dO3cuRXt8fLy8vLxcUBEAAAAAK8kUwejJJ59Ut27dtGHDBhljZIzR+vXr1aNHDzVq1MjV5QEAAAC4x2WKqXRnzpxR+/bt9f3339uX7L58+bIaN26s6dOnO9zfKDWYSgcAAABAymLXGF2zb98+7dq1S5IUHh6uEiVKpKkfghEAAAAAKfXBKJW7ZbzPP/9c77zzjvbu3StJKlmypF588UV16dLFxZUBAAAAuNdlimA0fPhwTZw4US+88IKqV68uSfrtt9/Ur18/HTp0SKNHj3ZxhQAAAADuZZliKl2ePHn03nvvqVWrVg7ts2fP1gsvvKCTJ0861R9T6QAAAABIWWy57suXL6tKlSop2u+//35duULKAQAAAJCxMkUweu655/TRRx+laP/kk0/Upk0bF1QEAAAAwEpcdo1R//797V/bbDZ99tlnWrp0qapVqyZJ2rBhgw4dOqR27dq5qkQAAAAAFuGya4weffTRVO1ns9n066+/OtU31xgBAAAAkLLofYzSC8EIAAAAgJTFFl8AAAAAAFciGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMuzGWOMq4vAveXSpUsaO3ashgwZIm9vb1eXA7gE5wHAeQBInAdZCcEI6S4uLk6BgYE6e/asAgICXF0O4BKcBwDnASBxHmQlTKUDAAAAYHkEIwAAAACWRzACAAAAYHkEI6Q7b29vjRgxggsMYWmcBwDnASBxHmQlLL4AAAAAwPIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjC3rkkUf04osvpnr/b7/9ViVKlJC7u7tTxwGZmc1m07fffpvq/VesWCGbzaYzZ85kWE1AZsH7HZmRs59fUiskJESTJk266faDBw/KZrNp69at6f7cHTp0UJMmTdK9X6QNwQi31b17dz3zzDM6fPiwXnvttQw7iadPn64cOXKke7/AjcTExOiJJ55I1z5HjhypihUrpmufEh9SkfH++4EzIiJCMTExCgwMvGs1ZOSHT0Dic4bk/B8FrcbD1QUgc4uPj9fx48dVr149FShQwNXlAOkiMTFR+fLlc3UZQKbl5eXFOQLAchgxsrhLly5pwIABKliwoLJly6YHH3xQK1askHT1r9TZs2eXJNWuXVs2m02PPPKIZsyYoYULF8pms8lms2nFihWKiIjQoEGDHPo+ceKEPD09tWrVKknS6dOn1a5dOwUFBcnPz09PPPGE9u7da3+ujh076uzZs/Z+R44cedsagdR45JFH1Lt3b7344ovKnTu36tWrl+KvZuvWrVPFihXl4+OjKlWq6Ntvv73hX6//+OMPValSRX5+foqIiFBUVJSkq3+JHDVqlLZt22Z/D0+fPl2tW7dWy5YtHfq4fPmycufOrS+++ELS1fd4nz59FBwcLB8fHz300EPatGmTpKt/RX/00UclSUFBQbLZbOrQoYMkKTk5WWPHjlWxYsXk6+urChUqaO7cuRnwE8S9rEOHDlq5cqXeffddh/fuf0cp16xZo4cffli+vr4qXLiw+vTpo/Pnz0uSXnnlFT344IMp+q5QoYJGjx4t6er7dfTo0SpUqJC8vb1VsWJFLVmyxL5vsWLFJEmVKlWy/39zzWeffaYyZcrIx8dHpUuX1ocffpgBPwlkBVeuXFHv3r0VGBio3Llza9iwYbp255m0fs6QpAsXLqhTp07Knj27ihQpok8++STFc+/Zs0cRERHy8fHRfffdp5UrV9q3JSUlqXPnzvbfx2FhYXr33Xcdjk9KSlL//v2VI0cO5cqVSy+//LJSe9ecTz75RAUKFFBycrJDe+PGjdWpUyf79x999JFCQ0Pl5eWlsLAwzZw5074tJCREktS0aVPZbDb795K0cOFCVa5cWT4+PipevLhGjRqlK1eupKq2e4qB5dSqVcv07dvXGGNMly5dTEREhFm1apXZt2+fefvtt423t7f566+/zKVLl0xUVJSRZObNm2diYmLM2bNnTYsWLUz9+vVNTEyMiYmJMZcuXTIffPCBKVKkiElOTrY/z/vvv+/Q1qhRI1OmTBmzatUqs3XrVlOvXj1TokQJk5iYaC5dumQmTZpkAgIC7P2eO3futjUCqVGrVi3j7+9vBg4caPbs2WP27NljJJkFCxYYY4w5e/asyZkzp2nbtq3ZuXOn+eGHH0ypUqWMJLNlyxZjjDHLly83ksyDDz5oVqxYYXbu3GkefvhhExERYYwx5sKFC+all14yZcuWtb+HL1y4YBYtWmR8fX3t72djjPn++++Nr6+viYuLM8YY06dPH1OgQAHzww8/mJ07d5r27duboKAgExsba65cuWLmzZtnJJmoqCgTExNjzpw5Y4wxZsyYMaZ06dJmyZIlZv/+/WbatGnG29vbrFix4u79cJHlnTlzxlSvXt107drV/t795ZdfjCRz+vRpY4wx+/btM9myZTPvvPOO+euvv8zatWtNpUqVTIcOHYwxxvz5559Gktm3b5+932tte/fuNcYYM3HiRBMQEGBmz55t9uzZY15++WXj6elp/12+ceNGI8n88ssvJiYmxsTGxhpjjPnyyy9N/vz5zbx588yBAwfMvHnzTM6cOc306dPv4k8JmcG13+V9+/Y1e/bsMV9++aXx8/Mzn3zyiTEm7Z8zihYtanLmzGkmT55s9u7da8aOHWvc3NzMnj17jDHGREdHG0mmUKFCZu7cuWbXrl2mS5cuJnv27ObkyZPGGGMSExPN8OHDzaZNm8yBAwfstX311Vf2+t98800TFBRk5s2bZ3bt2mU6d+5ssmfPbho3bnzb137q1Cnj5eVlfvnlF3tbbGysQ9v8+fONp6enmTx5somKijITJkww7u7u5tdffzXGGHP8+HEjyUybNs3ExMSY48ePG2OMWbVqlQkICDDTp083+/fvN0uXLjUhISFm5MiRd/gvlvUQjCzoWjD6+++/jbu7uzl69KjD9jp16pghQ4YYY4w5ffq0kWSWL19u396+ffsUJ/Hx48eNh4eHWbVqlb2tevXqZtCgQcYYY/766y8jyaxdu9a+/eTJk8bX19d8/fXXxhhjpk2bZgIDAx36TU2NwO3UqlXLVKpUyaHt+mD00UcfmVy5cpmEhAT79k8//fSGwej6/5QWL15sJNmPGzFihKlQoYLD81y+fNnkzp3bfPHFF/a2Vq1amZYtWxpjjImPjzeenp4mMjLSvj0xMdEUKFDAvPXWWw7Pfe1DqjHGXLx40fj5+Zl169Y5PF/nzp1Nq1atnPjpAI5/MDMm5Xuuc+fOplu3bg7HrF692ri5udnf/xUqVDCjR4+2bx8yZIh58MEH7d8XKFDAvP766w59PPDAA6Znz57GmP99+Lx2zl0TGhpqZs2a5dD22muvmerVq6fptSLrqlWrlilTpozDH2EHDRpkypQpk+bPGcZcDUZt27a1f5+cnGyCg4PNRx99ZIz533tz3Lhx9n0uX75sChUqZN58882b1turVy/z9NNP27/Pnz+//ff69X2kJhgZY0zjxo1Np06d7N9PmTLFFChQwCQlJRljjImIiDBdu3Z1OKZ58+amQYMG9u+v/7/vmjp16pg33njDoW3mzJkmf/78qarrXsJUOgvbsWOHkpKSVKpUKfn7+9sfK1eu1P79+53qK0+ePHr88ccVGRkpSYqOjtZvv/2mNm3aSJJ2794tDw8Ph6kWuXLlUlhYmHbv3n1XaoS13X///TfdFhUVpfLly8vHx8feVrVq1RvuW758efvX+fPnlyQdP378pn17eHioRYsW9nPj/PnzWrhwof3c2L9/vy5fvqwaNWrYj/H09FTVqlVveW7s27dPFy5c0GOPPeZwbnzxxRecG0h327Zt0/Tp0x3ea/Xq1VNycrKio6MlSW3atNGsWbMkScYYzZ492/4+j4uL0z///OPwPpekGjVq3PJ9fv78ee3fv1+dO3d2eO4xY8bwPreoatWqyWaz2b+vXr269u7dq127dqXpc8Y11/9ut9lsypcvX4rf7dWrV7d/7eHhoSpVqjj0PXnyZN1///3KkyeP/P399cknn+jQoUOSpLNnzyomJsahvmt9pFabNm00b948Xbp0SZIUGRmpZ599Vm5uVz/O79692+lzTLp6fo8ePdrhHOvatatiYmJ04cKFVNd3L2DxBQuLj4+Xu7u7/vjjD7m7uzts8/f3d7q/Nm3aqE+fPnr//fc1a9YslStXTuXKlctUNcK6smXLli79eHp62r++9p/zf+d8/1ebNm1Uq1YtHT9+XD///LN8fX1Vv379O6ojPj5ekrR48WIVLFjQYZu3t/cd9Q38V3x8vLp3764+ffqk2FakSBFJUqtWrTRo0CBt3rxZCQkJOnz4cIrr69LyvJL06aefpriG6b//JwB34vrf7dLV3++3+91+vTlz5mjAgAGaMGGCqlevruzZs+vtt9/Whg0b0q3Gp556SsYYLV68WA888IBWr16td9555477jY+P16hRo9SsWbMU267/g6EVEIwsrFKlSkpKStLx48f18MMPp/o4Ly8vJSUlpWhv3LixunXrpiVLlmjWrFlq166dfVuZMmV05coVbdiwQREREZKk2NhYRUVFKTw8/Kb9prVGwBlhYWH68ssvdenSJXuouLb4gTNudm5ERESocOHC+uqrr/Tjjz+qefPm9v+Er10ku3btWhUtWlTS1cUZNm3aZF8+2cvLS5Ic+g4PD5e3t7cOHTqkWrVqOV0rcL2bvXevqVy5snbt2qUSJUrcdJ9ChQqpVq1aioyMVEJCgh577DEFBwdLkgICAlSgQAGtXbvW4f26du1a++jsjd7nefPmVYECBXTgwAH76BOs7b9BY/369SpZsqTCw8PT9DnDGevXr1fNmjUlXV0E4o8//lDv3r0lXX0vR0REqGfPnvb9rx/VDAwMVP78+bVhw4YUfVSuXDlVz+/j46NmzZopMjJS+/btU1hYmMOxZcqU0dq1a9W+fXt729q1a+2vX7oaAP/7M6hcubKioqJueX5bBcHIwkqVKqU2bdqoXbt2mjBhgipVqqQTJ05o2bJlKl++vBo2bHjD40JCQvTTTz8pKipKuXLlUmBgoDw9PZUtWzY1adJEw4YN0+7du9WqVSv7MSVLllTjxo3VtWtXTZkyRdmzZ9fgwYNVsGBBNW7c2N5vfHy8li1bpgoVKsjPzy/NNQLOaN26tV599VV169ZNgwcP1qFDhzR+/HhJcpiycTshISGKjo7W1q1bVahQIWXPnt0etFq3bq2PP/5Yf/31l5YvX24/Jlu2bHr++ec1cOBA5cyZU0WKFNFbb72lCxcuqHPnzpKkokWLymazadGiRWrQoIF8fX2VPXt2DRgwQP369VNycrIeeughnT17VmvXrlVAQIDDf4zA7YSEhGjDhg06ePCg/P39U/ylfNCgQapWrZp69+6tLl26KFu2bNq1a5d+/vlnffDBB/b92rRpoxEjRigxMTHFX7IHDhyoESNGKDQ0VBUrVtS0adO0detW+zTT4OBg+fr6asmSJSpUqJB8fHwUGBioUaNGqU+fPgoMDFT9+vV16dIl/f777zp9+rT69++f8T8cZCqHDh1S//791b17d23evFnvv/++JkyYkObPGX5+fql+7smTJ6tkyZIqU6aM3nnnHZ0+fdq+IlzJkiX1xRdf6KefflKxYsU0c+ZMbdq0yb7aoiT17dtX48aNU8mSJVW6dGlNnDjR6fvTtWnTRk8++aR27typtm3bOmwbOHCgWrRooUqVKqlu3br6/vvvNX/+fP3yyy/2fUJCQrRs2TLVqFFD3t7eCgoK0vDhw/Xkk0+qSJEieuaZZ+Tm5qZt27bpzz//1JgxY5yqL8tz9UVOuPuuv8j22ioqISEhxtPT0+TPn980bdrUbN++3Rhz48UXjh8/bh577DHj7++fYtsPP/xgJJmaNWumeN5Tp06Z5557zgQGBhpfX19Tr169FCvL9ejRw+TKlctIMiNGjEhVjcDt/PfCcmNSXoC6du1aU758eePl5WXuv/9+M2vWLCPJvirRjRZA2LJli5FkoqOjjTFXF0R4+umnTY4cOewr/1yza9cuI8kULVrU4cJhY4xJSEgwL7zwgsmdO7fx9vY2NWrUMBs3bnTYZ/To0SZfvnzGZrOZ9u3bG2OuXiA8adIkExYWZjw9PU2ePHlMvXr1zMqVK+/o5wXriYqKMtWqVTO+vr729+5/3+8bN260/+7Pli2bKV++fIrFFE6fPm28vb2Nn5+fw0qMxhiTlJRkRo4caQoWLGg8PT1NhQoVzI8//uiwz6effmoKFy5s3NzcTK1ateztkZGRpmLFisbLy8sEBQWZmjVrmvnz56f7zwGZW61atUzPnj1Njx49TEBAgAkKCjKvvPKK/XdqWj9nFC1a1LzzzjsO+1WoUMG+/driC7NmzTJVq1Y1Xl5eJjw83L7amzFXf/936NDBBAYGmhw5cpjnn3/eDB482GFBnsuXL5u+ffuagIAAkyNHDtO/f3/Trl27VC++YMzV8yh//vxGktm/f3+K7R9++KEpXry48fT0NKVKlXJY+McYY7777jtTokQJ4+HhYYoWLWpvX7JkiYmIiDC+vr4mICDAVK1a1b7an5XYjEnlAuoAYCGRkZH2e174+vq6uhwAAJDBmEoHAJK++OILFS9eXAULFtS2bds0aNAgtWjRglAEAIBFEIwAQNKxY8c0fPhwHTt2TPnz51fz5s31+uuvu7osAIAFHDp0yGGRhP/atWuXfQVIZBym0gEAAAAudOXKFR08ePCm20NCQuThwXhGRiMYAQAAALA8N1cXAAAAAACuRjACAAAAYHkEIwAAAACWRzACAAAAYHkEIwAAAACWRzACAKTZsWPH1LdvX5UoUUI+Pj7KmzevatSooY8++kgXLlyw7xcSEiKbzSabzaZs2bKpcuXK+uabb1Jsu9GjQ4cOt6xh7dq18vDwUMWKFe1tTz31lOrXr3/D/VevXi2bzabt27en2FauXDn16NHjhsfNnDlT3t7eOnny5G1+KpLNZtO333572/0AAJkHwQgAkCYHDhxQpUqVtHTpUr3xxhvasmWLfvvtN7388statGiRfvnlF4f9R48erZiYGG3ZskUPPPCAWrZsqXXr1mnTpk2KiYlRTEyM5s2bJ0mKioqyt7377rs3reHMmTNq166d6tSp49DeuXNn/fzzzzpy5EiKY6ZNm6YqVaqofPnyKbZ17txZc+bMUUJCwg2Pa9SokXLnzp2qnw8AIGshGAEA0qRnz57y8PDQ77//rhYtWqhMmTIqXry4GjdurMWLF+upp55y2D979uzKly+fSpUqpcmTJ8vX11fff/+98uTJo3z58ilfvnzKmTOnJCk4ONjeFhgYeNMaevToodatW6t69eoO7U8++aTy5Mmj6dOnO7THx8frm2++UefOnW/YX9u2bZWQkGAPaNdER0drxYoV9uM++ugjhYaGysvLS2FhYZo5c6Z935CQEElS06ZNZbPZ7N9L0sKFC1W5cmX5+PioePHiGjVqlK5cuXLT1wcAuHsIRgAAp8XGxmrp0qXq1auXsmXLdsN9bDbbTY/38PCQp6enEhMT01zDtGnTdODAAY0YMeKG/bdr107Tp0/X9fcx/+abb5SUlKRWrVrdsM/cuXOrcePGmjp1qkP79OnTVahQIT3++ONasGCB+vbtq5deekl//vmnunfvro4dO2r58uWSpE2bNtnri4mJsX+/evVqtWvXTn379tWuXbs0ZcoUTZ8+Xa+//nqafwYAgPRDMAIAOG3fvn0yxigsLMyhPXfu3PL395e/v78GDRp0w2MTExM1duxYnT17VrVr107T8+/du1eDBw/Wl19+KQ8Pjxvu06lTJ+3fv18rV660t02bNk1PP/30LUehOnfurBUrVig6OlqSZIzRjBkz1L59e7m5uWn8+PHq0KGDevbsqVKlSql///5q1qyZxo8fL0nKkyePJClHjhzKly+f/ftRo0Zp8ODBat++vYoXL67HHntMr732mqZMmZKmnwEAIH0RjAAA6Wbjxo3aunWrypYtq0uXLjlsGzRokPz9/eXn56c333xT48aNU8OGDW/b57Wg5e/vrx49eigpKUmtW7fWqFGjVKpUqZseV7p0aUVERNhHf/bt26fVq1ffdBrdNY899pgKFSqkadOmSZKWLVumQ4cOqWPHjpKk3bt3q0aNGg7H1KhRQ7t3775lv9u2bdPo0aMdXk/Xrl0VExPjsFAFAMA1bvxnNgAAbqFEiRKy2WyKiopyaC9evLgkydfXN8UxAwcOVIcOHeTv76+8efPecqrd9bZu3Wr/OiAgQOfOndPvv/+uLVu2qHfv3pKk5ORkGWPk4eGhpUuX2keiOnfurBdeeEGTJ0/WtGnTFBoaqlq1at3y+dzc3NShQwfNmDFDI0eO1LRp0/Too4/aX1taxcfHa9SoUWrWrFmKbT4+PnfUNwDgzjFiBABwWq5cufTYY4/pgw8+0Pnz51N1TO7cuVWiRAnly5cv1aFIuhrCrj2Cg4MVEBCgHTt2aOvWrfZHjx49FBYWpq1bt+rBBx+0H9uiRQu5ublp1qxZ+uKLL9SpU6dUPXfHjh11+PBhzZ8/XwsWLHAYZSpTpozWrl3rsP/atWsVHh5u/97T01NJSUkO+1SuXFlRUVEOr+faw82N/44BwNUYMQIApMmHH36oGjVqqEqVKho5cqTKly8vNzc3bdq0SXv27NH999+fIc/r5uam++67z6EtODhYPj4+Kdr9/f3VsmVLDRkyRHFxcbe9J9I1xYoVU+3atdWtWzd5e3s7jPIMHDhQLVq0UKVKlVS3bl19//33mj9/vsPy5CEhIVq2bJlq1Kghb29vBQUFafjw4XryySdVpEgRPfPMM3Jzc9O2bdv0559/asyYMWn/gQAA0gV/ogIApEloaKi2bNmiunXrasiQIapQoYKqVKmi999/XwMGDNBrr73m6hIlXZ1Od/r0adWrV08FChRw+rjWrVs7THVr0qSJ3n33XY0fP15ly5bVlClTNG3aND3yyCP2fSZMmKCff/5ZhQsXVqVKlSRJ9erV06JFi7R06VI98MADqlatmt555x0VLVo03V4rACDtbOb6dUwBAAAAwIIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgef8HWA2Jk61Jt7wAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x700 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.metrics import confusion_matrix\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming df is your DataFrame\n",
+    "\n",
+    "# True labels and predictions\n",
+    "y_true = filtered_df[\"type\"]\n",
+    "y_pred = filtered_df[\"gpt_vote\"]\n",
+    "\n",
+    "# Compute the confusion matrix\n",
+    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "\n",
+    "# Create a pandas DataFrame from the confusion matrix\n",
+    "conf_matrix_df = pd.DataFrame(conf_matrix, index=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"], columns=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "\n",
+    "# Plotting the heatmap\n",
+    "plt.figure(figsize=(10, 7))\n",
+    "sns.heatmap(conf_matrix_df, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False)\n",
+    "plt.title(\"Arena Human vs GPT-4V Confusion Matrix\")\n",
+    "plt.xlabel(\"GPT-4V Vote\")\n",
+    "plt.ylabel(\"Arena Human Vote\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.5842911877394636\n",
+      "F1 Score (Macro): 0.514392348541452\n",
+      "F1 Score (Micro): 0.5842911877394636\n",
+      "F1 Score (Weighted): 0.5536668839130223\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import accuracy_score, f1_score\n",
+    "\n",
+    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
+    "y_true = filtered_df['type']\n",
+    "y_pred = filtered_df['gpt_vote']\n",
+    "\n",
+    "# Calculate accuracy\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "print(f'Accuracy: {accuracy}')\n",
+    "\n",
+    "# Calculate F1 score, here using 'macro' average to treat all classes equally\n",
+    "f1 = f1_score(y_true, y_pred, average='macro')\n",
+    "print(f'F1 Score (Macro): {f1}')\n",
+    "\n",
+    "# If you want to calculate F1 score with other averages, for example 'micro' or 'weighted', you can do:\n",
+    "f1_micro = f1_score(y_true, y_pred, average='micro')\n",
+    "print(f'F1 Score (Micro): {f1_micro}')\n",
+    "\n",
+    "f1_weighted = f1_score(y_true, y_pred, average='weighted')\n",
+    "print(f'F1 Score (Weighted): {f1_weighted}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cohen's Kappa Score: 0.3442144615665177\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import cohen_kappa_score\n",
+    "\n",
+    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
+    "y_true = filtered_df['type']\n",
+    "y_pred = filtered_df['gpt_vote']\n",
+    "\n",
+    "# Calculate Cohen's Kappa score\n",
+    "kappa = cohen_kappa_score(y_true, y_pred)\n",
+    "print(f'Cohen\\'s Kappa Score: {kappa}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy Score: 0.5842911877394636\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import accuracy_score\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "print(f'Accuracy Score: {accuracy}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Pearson Correlation Coefficient: 0.2880096104357029\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming filtered_df is your DataFrame and it contains 'type' and 'gpt_vote' columns\n",
+    "# Convert 'type' and 'gpt_vote' to categorical codes\n",
+    "filtered_df['type_int'] = pd.factorize(filtered_df['type'])[0]\n",
+    "filtered_df['gpt_vote_int'] = pd.factorize(filtered_df['gpt_vote'])[0]\n",
+    "\n",
+    "# Now you can calculate Pearson correlation between these new integer columns\n",
+    "pearson_correlation = filtered_df['type_int'].corr(filtered_df['gpt_vote_int'])\n",
+    "print(f'Pearson Correlation Coefficient: {pearson_correlation}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "otask",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

arena_elo/generation_model_info.json

@@ -0,0 +1,42 @@
+{
+    "LCM": {
+        "Link": "https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7",
+        "License": "MIT License",
+        "Organization": "Tsinghua University"
+    },
+    "Playground v2": {
+        "Link": "https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic",
+        "License": "Playground v2 Community License",
+        "Organization": "Playground"
+    },
+    "OpenJourney": {
+        "Link": "https://huggingface.co/prompthero/openjourney",
+        "License": "creativeml-openrail-m",
+        "Organization": "PromptHero"
+    },
+    "SDXLTurbo": {
+        "Link": "https://huggingface.co/stabilityai/sdxl-turbo",
+        "License": "sai-nc-community (other)",
+        "Organization": "Stability AI"
+    },
+    "SDXL": {
+        "Link": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+        "License": "openrail++",
+        "Organization": "Stability AI"
+    },
+    "PixArtAlpha": {
+        "Link": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS",
+        "License": "openrail++",
+        "Organization": "PixArt-alpha"
+    },
+    "SDXLLightning": {
+        "Link": "https://huggingface.co/ByteDance/SDXL-Lightning",
+        "License": "openrail++",
+        "Organization": "ByteDance"
+    },
+    "StableCascade": {
+        "Link": "https://huggingface.co/stabilityai/stable-cascade",
+        "License": "stable-cascade-nc-community (other)",
+        "Organization": "Stability AI"
+    }
+}

arena_elo/get_latest_data.sh

@@ -0,0 +1,17 @@
+
+# set LOGDIR to default if not set before
+if [ -z "$LOGDIR" ]; then
+    export LOGDIR="./vision-arena-logs"
+fi
+mkdir -p results
+
+
+# # for battle data
+python -m elo_rating.clean_battle_data --model_infos_file "./model_infos.json" --mode conv_release
+battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Battle data last updated on $battle_cutoff_date"
+
+mkdir -p ./results/latest
+mkdir -p ./results/$battle_cutoff_date && mv ./clean_battle_conv_$battle_cutoff_date.json ./results/$battle_cutoff_date/clean_battle_conv.json
+cp ./results/$battle_cutoff_date/clean_battle_conv.json ./results/latest/clean_battle_conv.json
+
+echo "Battle data last updated on $battle_cutoff_date" >> ./results/latest/latest_updated_date.txt

arena_elo/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "arena_elo"
+version = "0.2.35"
+description = "Elo rating system for WildVision Bench Arena"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+]
+dependencies = [
+    "numpy", "prompt_toolkit>=3.0.0", "uvicorn","polyglot", "pyicu", "pycld2", "morfessor", "scikit-learn",
+    "pytz", "tqdm", "pandas", "plotly", "fire", "Pillow"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/WildVision-Bench/Arena-Elo"
+"Bug Tracker" = "https://github.com/WildVision-Bench/Arena-Elo/issues"
+
+[tool.setuptools.packages.find]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
+
+[tool.wheel]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]

arena_elo/requirements.txt

@@ -0,0 +1,28 @@
+-e git+https://github.com/WildVision-Bench/Arena-Elo.git@9dc2fa8543a2e9eda3d5bc01c2212fdfcdd4bfb5#egg=arena_elo
+click==8.1.7
+fire==0.5.0
+h11==0.14.0
+joblib==1.3.2
+Morfessor==2.0.6
+numpy==1.26.4
+packaging==23.2
+pandas==2.2.0
+pillow==10.2.0
+plotly==5.18.0
+polyglot==16.7.4
+prompt-toolkit==3.0.43
+pycld2==0.41
+PyICU==2.12
+python-dateutil==2.8.2
+pytz==2024.1
+scikit-learn==1.4.0
+scipy==1.12.0
+six==1.16.0
+tenacity==8.2.3
+termcolor==2.4.0
+threadpoolctl==3.2.0
+tqdm==4.66.2
+typing_extensions==4.9.0
+tzdata==2024.1
+uvicorn==0.27.1
+wcwidth==0.2.13

arena_elo/results/20240220/elo_results_image_editing.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f41023a65a4dc1831a482dfa6098ccd528af9de297a1ea518881d49ce2885f0e
+size 57121

arena_elo/results/20240220/elo_results_t2i_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f383f3920ef3834f6e6ec213691a955992b4ce49ccaf7658c0b35ff72a2219d3
+size 54505

arena_elo/results/20240220/image_editing_leaderboard.csv

@@ -0,0 +1,8 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+Prompt2prompt,Prompt2prompt,1252.820838097007,1216.6489026518666,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
+PNP,PNP,1175.6261555831445,1171.3279007979363,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
+InstructPix2Pix,InstructPix2Pix,1155.8431458813104,1142.6827834982837,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+MagicBrush,MagicBrush,1051.428411953954,1089.4499296239383,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
+Pix2PixZero,Pix2PixZero,955.5903260059122,929.2296611307636,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
+CycleDiffusion,CycleDiffusion,771.4360186105207,753.4930725653142,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+SDEdit,SDEdit,637.2551038681513,697.1677497318974,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240220/t2i_generation_leaderboard.csv

@@ -0,0 +1,7 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+PlayGroundV2,PlayGroundV2,1151.1834096302248,1150.901721636401,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+PixArtAlpha,PixArtAlpha,1078.3583466674136,1069.815012597113,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,1027.258044463298,1035.47732509915,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,972.0904914158416,969.4933207298967,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,921.3424873878607,906.3184453708288,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,849.7672204353615,868.2154196730218,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240315/clean_battle_image_editing.json

@@ -0,0 +1,794 @@
+[
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707712630.872
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707712699.668
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707712896.0427
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707712929.7061
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713147.0445
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713198.9284
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713210.1306
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713747.5115
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707715613.7226
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707765708.2644
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707765861.2742
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707765975.0206
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707768866.9065
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707771673.2989
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784377.6617
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784466.8915
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784983.9581
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707785277.16
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707795299.0619
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707795798.752
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707796435.7996
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "CycleDiffusion",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707797278.7369
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707797279.6004
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805086.9739
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805220.3253
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805332.6322
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805476.0509
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707818374.3438
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707834631.9088
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707834954.0147
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835366.544
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835643.6178
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835789.25
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707836852.671
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707836952.6082
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "SDEdit",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707837020.7148
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707837226.2259
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707838166.1449
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707838405.0013
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707839133.3126
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707839484.6824
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707850104.2499
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707851384.7689
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707851936.9466
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707852836.3291
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707852878.673
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707853008.1359
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707856807.6229
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707863740.3507
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707866312.1118
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883083.3533
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883181.1397
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883187.9173
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883507.587
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883939.6125
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707892689.4407
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707908988.749
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707912639.2701
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707917685.9574
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707919429.336
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707932651.9192
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707932749.3107
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707933208.5797
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707945335.6341
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708031168.6838
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708038931.5388
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708057382.78
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708093689.8237
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708093910.4683
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708095090.8232
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708095305.4665
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708140553.1694
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708145512.3656
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708145724.4127
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708146846.5098
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708189738.4864
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708235874.9246
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708257619.7115
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708341265.7655
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708350183.3086
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708399707.1681
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708441502.4707
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708441716.8195
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708546759.2009
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708546805.4892
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708547082.7124
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708547166.9685
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708547293.7107
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708575046.0529
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708615466.9264
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708615516.3341
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1709205399.0098
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1709205767.8923
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1709443700.05
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1709702898.9291
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "Prompt2prompt",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710091925.1861
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710517781.1525
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1710517859.2942
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710535672.9791
+  }
+]

arena_elo/results/20240315/elo_results_image_editing.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e528d30840c8a5787b0d2f08f27758b02f7eb718ccab695010b30df2127efe5e
+size 57064

arena_elo/results/20240315/image_editing_leaderboard.csv

@@ -0,0 +1,8 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+Prompt2prompt,Prompt2prompt,1227.5508595026165,1182.7966878908553,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
+InstructPix2Pix,InstructPix2Pix,1160.2057367236093,1104.6020920372373,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+PNP,PNP,1142.693603173293,1163.4165011464634,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
+MagicBrush,MagicBrush,1053.1728944865915,1104.2259166081321,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
+Pix2PixZero,Pix2PixZero,918.6047552604578,899.6573774461386,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
+CycleDiffusion,CycleDiffusion,865.0529105743963,820.8487760803515,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+SDEdit,SDEdit,632.7192402790356,724.4526487908217,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240327/elo_results_t2i_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec01fe5af62dce3990634cffd1d926330ccbf170ef0c3b5d2f07fb06c4cf149
+size 65189

arena_elo/results/20240327/t2i_generation_leaderboard.csv

@@ -0,0 +1,11 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+Playground v2.5,Playground v2.5,1212.4660228554317,1233.021110469063,N/A,N/A,N/A
+StableCascade,StableCascade,1098.8180832734447,1081.4707812969855,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+PlayGroundV2,PlayGroundV2,1089.993871580802,1088.6262085724481,N/A,N/A,N/A
+Playground v2,Playground v2,1049.6156124554975,1051.618375116693,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+SDXLLightning,SDXLLightning,1036.8582186059539,1039.3079223370821,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1016.2085497703334,1002.5100184720693,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,960.5073412035289,965.3037978455568,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,907.997473382927,910.1644152252661,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,836.9689192463355,827.9470053715127,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/clean_battle_image_editing_20240315.json

@@ -0,0 +1,794 @@
+[
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707712630.872
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707712699.668
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707712896.0427
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707712929.7061
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713147.0445
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713198.9284
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713210.1306
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707713747.5115
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707715613.7226
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707765708.2644
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707765861.2742
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707765975.0206
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707768866.9065
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707771673.2989
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784377.6617
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784466.8915
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707784983.9581
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707785277.16
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707795299.0619
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "SDEdit",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707795798.752
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707796435.7996
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "CycleDiffusion",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707797278.7369
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707797279.6004
+  },
+  {
+    "model_a": "SDEdit",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805086.9739
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805220.3253
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805332.6322
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707805476.0509
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707818374.3438
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707834631.9088
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707834954.0147
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835366.544
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "SDEdit",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835643.6178
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707835789.25
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707836852.671
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707836952.6082
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "SDEdit",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707837020.7148
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707837226.2259
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707838166.1449
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707838405.0013
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707839133.3126
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707839484.6824
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707850104.2499
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707851384.7689
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707851936.9466
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707852836.3291
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707852878.673
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707853008.1359
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707856807.6229
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707863740.3507
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707866312.1118
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883083.3533
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883181.1397
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883187.9173
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883507.587
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707883939.6125
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707892689.4407
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707908988.749
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707912639.2701
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707917685.9574
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707919429.336
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "CycleDiffusion",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707932651.9192
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707932749.3107
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1707933208.5797
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1707945335.6341
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708031168.6838
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708038931.5388
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708057382.78
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708093689.8237
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708093910.4683
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708095090.8232
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708095305.4665
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "Prompt2prompt",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708140553.1694
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Prompt2prompt",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708145512.3656
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "Prompt2prompt",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708145724.4127
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708146846.5098
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708189738.4864
+  },
+  {
+    "model_a": "Prompt2prompt",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708235874.9246
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708257619.7115
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708341265.7655
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708350183.3086
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708399707.1681
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708441502.4707
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708441716.8195
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708546759.2009
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708546805.4892
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708547082.7124
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708547166.9685
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708547293.7107
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "PNP",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708575046.0529
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1708615466.9264
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "MagicBrush",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1708615516.3341
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1709205399.0098
+  },
+  {
+    "model_a": "InstructPix2Pix",
+    "model_b": "PNP",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1709205767.8923
+  },
+  {
+    "model_a": "PNP",
+    "model_b": "InstructPix2Pix",
+    "winner": "model_b",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1709443700.05
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "Pix2PixZero",
+    "winner": "model_a",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1709702898.9291
+  },
+  {
+    "model_a": "CycleDiffusion",
+    "model_b": "Prompt2prompt",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710091925.1861
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710517781.1525
+  },
+  {
+    "model_a": "MagicBrush",
+    "model_b": "InstructPix2Pix",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": false,
+    "tstamp": 1710517859.2942
+  },
+  {
+    "model_a": "Pix2PixZero",
+    "model_b": "CycleDiffusion",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_::1",
+    "anony": true,
+    "tstamp": 1710535672.9791
+  }
+]

arena_elo/results/latest/elo_results_image_editing.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e528d30840c8a5787b0d2f08f27758b02f7eb718ccab695010b30df2127efe5e
+size 57064

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec01fe5af62dce3990634cffd1d926330ccbf170ef0c3b5d2f07fb06c4cf149
+size 65189

arena_elo/results/latest/image_editing_leaderboard.csv

@@ -0,0 +1,8 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+Prompt2prompt,Prompt2prompt,1227.5508595026165,1182.7966878908553,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
+InstructPix2Pix,InstructPix2Pix,1160.2057367236093,1104.6020920372373,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+PNP,PNP,1142.693603173293,1163.4165011464634,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
+MagicBrush,MagicBrush,1053.1728944865915,1104.2259166081321,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
+Pix2PixZero,Pix2PixZero,918.6047552604578,899.6573774461386,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
+CycleDiffusion,CycleDiffusion,865.0529105743963,820.8487760803515,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+SDEdit,SDEdit,632.7192402790356,724.4526487908217,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/latest/t2i_generation_leaderboard.csv

@@ -0,0 +1,11 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+Playground v2.5,Playground v2.5,1212.4660228554317,1233.021110469063,N/A,N/A,N/A
+StableCascade,StableCascade,1098.8180832734447,1081.4707812969855,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+PlayGroundV2,PlayGroundV2,1089.993871580802,1088.6262085724481,N/A,N/A,N/A
+Playground v2,Playground v2,1049.6156124554975,1051.618375116693,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+SDXLLightning,SDXLLightning,1036.8582186059539,1039.3079223370821,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1016.2085497703334,1002.5100184720693,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,960.5073412035289,965.3037978455568,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,907.997473382927,910.1644152252661,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,836.9689192463355,827.9470053715127,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/simple_test.py

@@ -0,0 +1,16 @@
+import pickle
+with open("./results/latest/elo_results.pkl",'rb') as f:
+    data = pickle.load(f)
+    print()
+    df = data["anony"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+
+    print()
+
+    df = data["full"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+    print('done') 

arena_elo/update_elo_rating.sh

@@ -0,0 +1,49 @@
+# set LOGDIR to default if not set before
+if [ -z "$LOGDIR" ]; then
+    echo "LOGDIR is not set. Please set LOGDIR to the directory where the logs will be stored. Command: export LOGDIR=/path/to/logdir"
+    exit 1
+fi
+
+mkdir -p results
+
+# # for battle data
+python -m elo_rating.clean_battle_data --task_name "image_editing"
+edition_battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Image editing battle data last updated on $edition_battle_cutoff_date"
+
+python -m elo_rating.clean_battle_data --task_name "t2i_generation"
+generation_battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "T2I image generation battle data last updated on $generation_battle_cutoff_date"
+
+mkdir -p ./results/$edition_battle_cutoff_date
+mkdir -p ./results/$generation_battle_cutoff_date
+
+cp clean_battle_image_editing_$edition_battle_cutoff_date.json ./results/latest/
+cp clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/latest/
+mv clean_battle_image_editing_$edition_battle_cutoff_date.json ./results/$edition_battle_cutoff_date/clean_battle_image_editing.json
+mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
+
+
+python3 -m elo_rating.elo_analysis --clean-battle-file clean_battle_image_editing_$edition_battle_cutoff_date.json
+mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
+
+python3 -m elo_rating.elo_analysis --clean-battle-file clean_battle_t2i_generation_$generation_battle_cutoff_date.json
+mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
+
+# generat the leaderboard
+
+python -m elo_rating.generate_leaderboard \
+    --model_info_file "./edition_model_info.json" \
+    --elo_rating_pkl "./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl" \
+    --output_csv "./results/$edition_battle_cutoff_date/image_editing_leaderboard.csv"
+
+python -m elo_rating.generate_leaderboard \
+    --model_info_file "./generation_model_info.json" \
+    --elo_rating_pkl "./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl" \
+    --output_csv "./results/$generation_battle_cutoff_date/t2i_generation_leaderboard.csv"
+
+mkdir -p ./results/latest
+cp ./results/$edition_battle_cutoff_date/image_editing_leaderboard.csv ./results/latest/image_editing_leaderboard.csv
+cp ./results/$generation_battle_cutoff_date/t2i_generation_leaderboard.csv ./results/latest/t2i_generation_leaderboard.csv
+cp ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl ./results/latest/elo_results_image_editing.pkl
+cp ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl ./results/latest/elo_results_t2i_generation.pkl
+
+