Spaces:
Running
on
Zero
Running
on
Zero
Commit
•
f6608c4
1
Parent(s):
73848fe
update
Browse files- arena_elo/elo_rating/clean_battle_data.py +2 -1
- arena_elo/generation_model_info.json +6 -1
- arena_elo/results/20240315/elo_results_image_editing.pkl +2 -2
- arena_elo/results/20240327/clean_battle_t2i_generation.json +0 -0
- arena_elo/results/20240327/elo_results_t2i_generation.pkl +2 -2
- arena_elo/results/20240327/t2i_generation_leaderboard.csv +9 -10
- arena_elo/results/latest/clean_battle_t2i_generation_20240327.json +0 -0
- arena_elo/results/latest/elo_results_image_editing.pkl +2 -2
- arena_elo/results/latest/elo_results_t2i_generation.pkl +2 -2
- arena_elo/results/latest/t2i_generation_leaderboard.csv +9 -10
- arena_elo/update_elo_rating.sh +2 -2
arena_elo/elo_rating/clean_battle_data.py
CHANGED
@@ -82,6 +82,7 @@ def replace_model_name(old_name, tstamp):
|
|
82 |
"claude-instant-v1": "claude-instant-1",
|
83 |
"oasst-sft-1-pythia-12b": "oasst-pythia-12b",
|
84 |
"claude-2": "claude-2.0",
|
|
|
85 |
}
|
86 |
if old_name in ["gpt-4", "gpt-3.5-turbo"]:
|
87 |
if tstamp > 1687849200:
|
@@ -203,7 +204,6 @@ def clean_battle_data(
|
|
203 |
# continue
|
204 |
|
205 |
# Replace bard with palm
|
206 |
-
models = [replace_model_name(m, row["tstamp"]) for m in models]
|
207 |
if task_name == "image_editing":
|
208 |
if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
|
209 |
# print(f"Invalid model names: {models}")
|
@@ -222,6 +222,7 @@ def clean_battle_data(
|
|
222 |
|
223 |
else:
|
224 |
raise ValueError(f"Invalid task_name: {task_name}")
|
|
|
225 |
|
226 |
# Exclude certain models
|
227 |
if exclude_model_names and any(x in exclude_model_names for x in models):
|
|
|
82 |
"claude-instant-v1": "claude-instant-1",
|
83 |
"oasst-sft-1-pythia-12b": "oasst-pythia-12b",
|
84 |
"claude-2": "claude-2.0",
|
85 |
+
"PlayGroundV2": "Playground v2",
|
86 |
}
|
87 |
if old_name in ["gpt-4", "gpt-3.5-turbo"]:
|
88 |
if tstamp > 1687849200:
|
|
|
204 |
# continue
|
205 |
|
206 |
# Replace bard with palm
|
|
|
207 |
if task_name == "image_editing":
|
208 |
if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
|
209 |
# print(f"Invalid model names: {models}")
|
|
|
222 |
|
223 |
else:
|
224 |
raise ValueError(f"Invalid task_name: {task_name}")
|
225 |
+
models = [replace_model_name(m, row["tstamp"]) for m in models]
|
226 |
|
227 |
# Exclude certain models
|
228 |
if exclude_model_names and any(x in exclude_model_names for x in models):
|
arena_elo/generation_model_info.json
CHANGED
@@ -9,6 +9,11 @@
|
|
9 |
"License": "Playground v2 Community License",
|
10 |
"Organization": "Playground"
|
11 |
},
|
|
|
|
|
|
|
|
|
|
|
12 |
"OpenJourney": {
|
13 |
"Link": "https://huggingface.co/prompthero/openjourney",
|
14 |
"License": "creativeml-openrail-m",
|
@@ -39,4 +44,4 @@
|
|
39 |
"License": "stable-cascade-nc-community (other)",
|
40 |
"Organization": "Stability AI"
|
41 |
}
|
42 |
-
}
|
|
|
9 |
"License": "Playground v2 Community License",
|
10 |
"Organization": "Playground"
|
11 |
},
|
12 |
+
"Playground v2.5": {
|
13 |
+
"Link": "https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic",
|
14 |
+
"License": "Playground v2 Community License",
|
15 |
+
"Organization": "Playground"
|
16 |
+
},
|
17 |
"OpenJourney": {
|
18 |
"Link": "https://huggingface.co/prompthero/openjourney",
|
19 |
"License": "creativeml-openrail-m",
|
|
|
44 |
"License": "stable-cascade-nc-community (other)",
|
45 |
"Organization": "Stability AI"
|
46 |
}
|
47 |
+
}
|
arena_elo/results/20240315/elo_results_image_editing.pkl
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
|
3 |
+
size 57091
|
arena_elo/results/20240327/clean_battle_t2i_generation.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
arena_elo/results/20240327/elo_results_t2i_generation.pkl
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
|
3 |
+
size 62414
|
arena_elo/results/20240327/t2i_generation_leaderboard.csv
CHANGED
@@ -1,11 +1,10 @@
|
|
1 |
key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
|
2 |
-
Playground v2.5,Playground v2.5,
|
3 |
-
StableCascade,StableCascade,
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
|
|
|
1 |
key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
|
2 |
+
Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
|
3 |
+
StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
|
4 |
+
Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
|
5 |
+
SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
|
6 |
+
PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
|
7 |
+
SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
|
8 |
+
SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
|
9 |
+
OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
|
10 |
+
LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
|
|
arena_elo/results/latest/clean_battle_t2i_generation_20240327.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
arena_elo/results/latest/elo_results_image_editing.pkl
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
|
3 |
+
size 57091
|
arena_elo/results/latest/elo_results_t2i_generation.pkl
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
|
3 |
+
size 62414
|
arena_elo/results/latest/t2i_generation_leaderboard.csv
CHANGED
@@ -1,11 +1,10 @@
|
|
1 |
key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
|
2 |
-
Playground v2.5,Playground v2.5,
|
3 |
-
StableCascade,StableCascade,
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
|
|
|
1 |
key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
|
2 |
+
Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
|
3 |
+
StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
|
4 |
+
Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
|
5 |
+
SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
|
6 |
+
PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
|
7 |
+
SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
|
8 |
+
SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
|
9 |
+
OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
|
10 |
+
LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
|
|
arena_elo/update_elo_rating.sh
CHANGED
@@ -22,10 +22,10 @@ mv clean_battle_image_editing_$edition_battle_cutoff_date.json ./results/$editio
|
|
22 |
mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
|
23 |
|
24 |
|
25 |
-
python3 -m elo_rating.elo_analysis --clean-battle-file
|
26 |
mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
|
27 |
|
28 |
-
python3 -m elo_rating.elo_analysis --clean-battle-file
|
29 |
mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
|
30 |
|
31 |
# generat the leaderboard
|
|
|
22 |
mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
|
23 |
|
24 |
|
25 |
+
python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$edition_battle_cutoff_date/clean_battle_image_editing.json
|
26 |
mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
|
27 |
|
28 |
+
python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
|
29 |
mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
|
30 |
|
31 |
# generat the leaderboard
|