File size: 2,512 Bytes
9ba23bb
 
 
 
 
 
2d73c50
 
17d41ef
9ba23bb
 
 
 
 
ccecac4
ba1ec42
 
 
 
 
fa7f487
ba1ec42
9ba23bb
d238af3
c775ff6
 
 
 
d238af3
48006fb
c775ff6
1ff07b8
c775ff6
 
 
 
 
 
 
9ba23bb
 
 
 
ccecac4
9ba23bb
ba1ec42
9ba23bb
 
 
a54c958
9ba23bb
806fb5d
9ba23bb
 
 
 
 
c775ff6
9ba23bb
b405f8c
ea1d7f8
 
a54c958
806fb5d
a54c958
 
9ba23bb
83bf0a7
 
55a5625
9ba23bb
 
ccecac4
 
 
 
 
 
 
 
 
 
 
 
 
4a98ecf
90a662c
c775ff6
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
import gradio as gr

from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
import pandas as pd

benchmark_user = 'vsd-benchmark'
fashion_dataset = f'{benchmark_user}/vsd-fashion'
benchmark_tag = 'vsd'

hf_api = HfApi()

models = list(hf_api.list_models(filter=benchmark_tag))

SUPPORTED_TASKS = [
    'in_catalog_retrieval_zero_shot',
    'in_catalog_open_catalog',
    'in_catalog_closed_catalog',
    'consumer-catalog_wild_zero_shot',
]

print("Tagged models", models)

def create_model_link(model_id, link=None, type='repos'):
    
    if link is None:
        type_url_part = ''

        if type != 'repos' and not None:
            type_url_part = f"/{type}"

        link = f"https://huggingface.co{type_url_part}/{model_id}"


    return (
        f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_id}</a>'
    )


def get_model_results(model_meta):
    metrics_meta = []
    for index in model_meta['model-index']:
        for result in index['results']:
            if result['dataset']['type'].split('/')[0] == benchmark_user and result['dataset']['config'] in SUPPORTED_TASKS:
                metrics_dict = {metric['name']: metric['value'] for metric in result['metrics']}
                metrics_meta += [dict(dataset=result['dataset']['type'], task=result['dataset']['config'], **metrics_dict)]

    return metrics_meta

results = []

for model in models:
    readme_path = hf_hub_download(model.modelId, filename="README.md")
    meta = metadata_load(readme_path)
    model_results = get_model_results(meta)

    for result in model_results:
        results += [dict(model=create_model_link(model.modelId), **result)]

paper_models_df = pd.read_csv('./paper_models.csv', index_col=0)
paper_models_df['source'] = 'VSD Paper'

results_df = pd.DataFrame(results)
results_df['source'] = 'HuggingFace'

df = pd.concat([results_df, paper_models_df])

print(df)

block = gr.Blocks()

with block:
    for task in SUPPORTED_TASKS: 
        group = df[df['task'] == task]

        if len(group) > 0:
            gr.Markdown(f"## Task - {task}")

            group = group.sort_values('ROC_AUC', ascending=False)
            group['dataset'] = group['dataset'].apply(lambda x: create_model_link(x, type="datasets"))
            gr.DataFrame(
                group.reset_index(drop=True), 
                datatype=['markdown', 'markdown'] + ['number'] * len(group.columns),
                wrap=True,
            )

block.launch()