Use dataset config as tasks

app.py

@@ -12,11 +12,14 @@ hf_api = HfApi()
 
 models = list(hf_api.list_models(filter=benchmark_tag))
 
-print("Tagged models", models)
+supported_tasks = [
+    'in_catalog_retrieval_zero_shot',
+    'in_catalog_open_catalog',
+    'in_catalog_closed_catalog',
+    'consumer-catalog_wild_zero_shot',
+]
 
-MAIN_METRIC_PER_TASK = {
-    fashion_dataset: "ROC_AUC"
-}
+print("Tagged models", models)
 
 def create_model_link(model_id, link=None, type='repos'):
     
@@ -38,9 +41,9 @@ def get_model_results(model_meta):
     metrics_meta = []
     for index in model_meta['model-index']:
         for result in index['results']:
-            if result['dataset']['type'].split('/')[0] == benchmark_user:
+            if result['dataset']['type'].split('/')[0] == benchmark_user and result['dataset']['config'] in supported_tasks:
                 metrics_dict = {metric['name']: metric['value'] for metric in result['metrics']}
-                metrics_meta += [dict(dataset=result['dataset']['type'], **metrics_dict)]
+                metrics_meta += [dict(dataset=result['dataset']['type'], task=result['dataset']['config'], **metrics_dict)]
 
     return metrics_meta
 
@@ -67,10 +70,10 @@ print(df)
 block = gr.Blocks()
 
 with block:
-    for group_name, group in df.groupby('dataset'):
+    for group_name, group in df.groupby('task'):
         gr.Markdown(f"## Task - {group_name}")
 
-    group = group.sort_values(MAIN_METRIC_PER_TASK[group_name], ascending=False)
+    group = group.sort_values('ROC_AUC', ascending=False)
     group['dataset'] = group['dataset'].apply(lambda x: create_model_link(x, type="datasets"))
     gr.DataFrame(
         group.reset_index(drop=True), 

paper_models.csv

@@ -1,5 +1,5 @@
-,dataset,model,ROC_AUC,MRR@5
-0,vsd-benchmark/vsd-fashion,Beit,75.5,93.5
-1,vsd-benchmark/vsd-fashion,DINO,70.4,93.3
-2,vsd-benchmark/vsd-fashion,ResNext,62.7,84.5
-3,vsd-benchmark/vsd-fashion,CLIP,67.8,84.8
+,dataset,task,model,ROC_AUC,MRR@5
+0,vsd-benchmark/vsd-fashion,in_catalog_retrieval_zero_shot,Beit,75.5,93.5
+1,vsd-benchmark/vsd-fashion,in_catalog_retrieval_zero_shot,DINO,70.4,93.3
+2,vsd-benchmark/vsd-fashion,in_catalog_retrieval_zero_shot,ResNext,62.7,84.5
+3,vsd-benchmark/vsd-fashion,in_catalog_retrieval_zero_shot,CLIP,67.8,84.8