update

cli/analysis-cli.py

@@ -111,8 +111,6 @@ if data_map is None:
         for dataset_name, results_dict in data["results"].items():
             for metric_name, value in results_dict.items():
 
-                # print(model_name, dataset_name, metric_name, value)
-
                 if ',' in metric_name and '_stderr' not in metric_name \
                         and 'f1' not in metric_name \
                         and model_name_to_model_map[model_name]["likes"] > 128:
@@ -160,9 +158,8 @@ if data_map is None:
                     if 'fever' in dataset_name:
                         to_add = False
 
-                    if 'xsum' in dataset_name:
-                        # to_add = False
-                        pass
+                    if ('xsum' in dataset_name or 'cnn' in dataset_name) and 'v2' in dataset_name:
+                        to_add = False
 
                     if 'rouge' in metric_name:
                         value /= 100.0
@@ -186,8 +183,10 @@ if data_map is None:
 
 model_name_lst = [m for m in data_map.keys()]
 
+nb_max_metrics = max(len(data_map[model_name]) for model_name in model_name_lst)
+
 for model_name in model_name_lst:
-    if len(data_map[model_name]) < 14:
+    if len(data_map[model_name]) < nb_max_metrics - 5:
         del data_map[model_name]
 
 plot_type_lst = ['all', 'summ', 'qa', 'instr', 'detect', 'rc']
@@ -293,27 +292,30 @@ for plot_type in plot_type_lst:
 
     print('figsize', (fig_width, fig_height))
 
-    print(f'Generating clustermap for {plot_type}')
-
-    # fig = sns.clustermap(df, method='average', metric='cosine', cmap='coolwarm', figsize=(16, 12), annot=True)
-    fig = sns.clustermap(df,
-                         method='ward',
-                         metric='euclidean',
-                         cmap='coolwarm',
-                         figsize=(fig_width, fig_height),  # figsize=(24, 16),
-                         annot=True,
-                         mask=o_df.isnull(),
-                         dendrogram_ratio=dendrogram_ratio,
-                         fmt='.2f',
-                         col_cluster=col_cluster,
-                         row_cluster=row_cluster)
-
-    # Adjust the size of the cells (less wide)
-    plt.setp(fig.ax_heatmap.get_yticklabels(), rotation=0)
-    plt.setp(fig.ax_heatmap.get_xticklabels(), rotation=90)
-
-    # Save the clustermap to file
-    fig.savefig(f'plots/clustermap_{plot_type}.pdf')
-    fig.savefig(f'plots/clustermap_{plot_type}.png')
-
     o_df.to_json(f'plots/clustermap_{plot_type}.json', orient='split')
+
+    print(f'Generating the clustermaps for {plot_type}')
+
+    for cmap in [None, 'coolwarm', 'viridis']:
+        fig = sns.clustermap(df,
+                             method='ward',
+                             metric='euclidean',
+                             cmap=cmap,
+                             figsize=(fig_width, fig_height),  # figsize=(24, 16),
+                             annot=True,
+                             mask=o_df.isnull(),
+                             dendrogram_ratio=dendrogram_ratio,
+                             fmt='.2f',
+                             col_cluster=col_cluster,
+                             row_cluster=row_cluster)
+
+        # Adjust the size of the cells (less wide)
+        plt.setp(fig.ax_heatmap.get_yticklabels(), rotation=0)
+        plt.setp(fig.ax_heatmap.get_xticklabels(), rotation=90)
+
+        cmap_suffix = '' if cmap is None else f'_{cmap}'
+
+        # Save the clustermap to file
+        fig.savefig(f'plots/clustermap_{plot_type}{cmap_suffix}.pdf')
+        fig.savefig(f'plots/clustermap_{plot_type}{cmap_suffix}.png')
+        fig.savefig(f'plots/clustermap_{plot_type}{cmap_suffix}_t.png', transparent=True, facecolor="none")

cli/eval-cli.py

@@ -36,7 +36,8 @@ def main():
     # my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
     # my_task = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
     # my_task = Task("truefalse_cieacf", "acc", "TrueFalse", 5)
-    my_task = Task("faithdial_hallu", "acc", "FaithDIAL", 2)
+    # my_task = Task("faithdial_hallu", "acc", "FaithDIAL", 2)
+    my_task = Task("fever11", "acc", "FEVER", 8)
 
     eval_logger = utils.eval_logger
     import logging
@@ -59,7 +60,7 @@ def main():
 
         # breakpoint()
         results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=task.num_fewshot,
-                                            batch_size=1, device="mps", use_cache=None, limit=10, write_out=True)
+                                            batch_size=1, device="mps", use_cache=None, limit=1000, write_out=True)
         print('AAA', results["results"])
 
         breakpoint()

cli/fever-upload-cli.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+
+import random
+from tqdm import tqdm
+
+from datasets import Dataset, DatasetDict, load_dataset
+
+
+def convert(list_of_dicts):
+    res = {}
+    for d in list_of_dicts:
+        for k, v in d.items():
+            res.setdefault(k, []).append(v)
+    return res
+
+
+v10 = load_dataset("fever", "v1.0")
+name_lst = ['train', 'labelled_dev']
+
+old_to_new_label_map = {
+    'SUPPORTS': 'supported',
+    'REFUTES': 'refuted'
+}
+
+data_map = {}
+
+for name in name_lst:
+    instance_lst = []
+
+    for entry in tqdm(v10[name]):
+        id_ = entry['id']
+        label = entry['label']
+        claim = entry['claim']
+
+        evidence_id = entry['evidence_id']
+        evidence_wiki_url = entry['evidence_wiki_url']
+
+        if evidence_id != -1:
+            assert label in {'SUPPORTS', 'REFUTES'}
+
+            instance = {'id': id_, 'label': old_to_new_label_map[label], 'claim': claim}
+            instance_lst.append(instance)
+
+    key = 'dev' if name in {'labelled_dev'} else name
+
+    instance_lst = sorted([dict(t) for t in {tuple(d.items()) for d in instance_lst}], key=lambda d: d['claim'])
+
+    label_to_instance_lst = {}
+    for e in instance_lst:
+        if e['label'] not in label_to_instance_lst:
+            label_to_instance_lst[e['label']] = []
+        label_to_instance_lst[e['label']].append(e)
+
+    min_len = min(len(v) for k, v in label_to_instance_lst.items())
+
+    new_instance_lst = []
+    for k in sorted(label_to_instance_lst.keys()):
+        new_instance_lst += label_to_instance_lst[k][:min_len]
+
+    random.Random(42).shuffle(new_instance_lst)
+    data_map[key] = new_instance_lst
+
+ds_path = 'pminervini/hl-fever'
+
+task_to_ds_map = {k: Dataset.from_dict(convert(v)) for k, v in data_map.items()}
+ds_dict = DatasetDict(task_to_ds_map)
+
+ds_dict.push_to_hub(ds_path, "v1.0")
+
+# breakpoint()

src/backend/envs.py

@@ -46,6 +46,7 @@ class Tasks(Enum):
     task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
 
     # task15 = Task("fever10", "acc", "FEVER", 16)
+    task15_1 = Task("fever11", "acc", "FEVER", 8)
 
     task16 = Task("squadv2", "exact", "SQuADv2", 4)
 

src/backend/tasks/fever/fever11.yaml

@@ -0,0 +1,17 @@
+group: fever
+task: fever11
+dataset_path: pminervini/hl-fever
+dataset_name: v1.0
+output_type: multiple_choice
+training_split: train
+validation_split: dev
+test_split: null
+doc_to_text: "Claim: {{claim}}\nLabel:"
+doc_to_choice: ["supported", "refuted"]
+doc_to_target: label
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0