Spaces:
Runtime error
Runtime error
File size: 2,512 Bytes
9ba23bb 2d73c50 17d41ef 9ba23bb ccecac4 ba1ec42 fa7f487 ba1ec42 9ba23bb d238af3 c775ff6 d238af3 48006fb c775ff6 1ff07b8 c775ff6 9ba23bb ccecac4 9ba23bb ba1ec42 9ba23bb a54c958 9ba23bb 806fb5d 9ba23bb c775ff6 9ba23bb b405f8c ea1d7f8 a54c958 806fb5d a54c958 9ba23bb 83bf0a7 55a5625 9ba23bb ccecac4 4a98ecf 90a662c c775ff6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
import pandas as pd
benchmark_user = 'vsd-benchmark'
fashion_dataset = f'{benchmark_user}/vsd-fashion'
benchmark_tag = 'vsd'
hf_api = HfApi()
models = list(hf_api.list_models(filter=benchmark_tag))
SUPPORTED_TASKS = [
'in_catalog_retrieval_zero_shot',
'in_catalog_open_catalog',
'in_catalog_closed_catalog',
'consumer-catalog_wild_zero_shot',
]
print("Tagged models", models)
def create_model_link(model_id, link=None, type='repos'):
if link is None:
type_url_part = ''
if type != 'repos' and not None:
type_url_part = f"/{type}"
link = f"https://huggingface.co{type_url_part}/{model_id}"
return (
f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_id}</a>'
)
def get_model_results(model_meta):
metrics_meta = []
for index in model_meta['model-index']:
for result in index['results']:
if result['dataset']['type'].split('/')[0] == benchmark_user and result['dataset']['config'] in SUPPORTED_TASKS:
metrics_dict = {metric['name']: metric['value'] for metric in result['metrics']}
metrics_meta += [dict(dataset=result['dataset']['type'], task=result['dataset']['config'], **metrics_dict)]
return metrics_meta
results = []
for model in models:
readme_path = hf_hub_download(model.modelId, filename="README.md")
meta = metadata_load(readme_path)
model_results = get_model_results(meta)
for result in model_results:
results += [dict(model=create_model_link(model.modelId), **result)]
paper_models_df = pd.read_csv('./paper_models.csv', index_col=0)
paper_models_df['source'] = 'VSD Paper'
results_df = pd.DataFrame(results)
results_df['source'] = 'HuggingFace'
df = pd.concat([results_df, paper_models_df])
print(df)
block = gr.Blocks()
with block:
for task in SUPPORTED_TASKS:
group = df[df['task'] == task]
if len(group) > 0:
gr.Markdown(f"## Task - {task}")
group = group.sort_values('ROC_AUC', ascending=False)
group['dataset'] = group['dataset'].apply(lambda x: create_model_link(x, type="datasets"))
gr.DataFrame(
group.reset_index(drop=True),
datatype=['markdown', 'markdown'] + ['number'] * len(group.columns),
wrap=True,
)
block.launch()
|