update

cli/analysis-cli.py

@@ -72,6 +72,7 @@ def sanitise_dataset(name: str) -> str:
     res = res.replace("summarization", "Summarization")
     res = res.replace("dialogue", "Dialog")
     res = res.replace("halueval", "HaluEval")
+    res = res.replace("_v2", "")
     res = res.replace("_", " ")
     return res
 
@@ -135,11 +136,6 @@ if data_map is None:
                     if 'memo-trap_v2' in dataset_name:
                         to_add = False
 
-                    if 'selfcheck' in dataset_name:
-                        # if 'max' in metric_name:
-                        #     to_add = False
-                        pass
-
                     if 'faithdial' in dataset_name:
                         to_add = False
 
@@ -166,7 +162,7 @@ if data_map is None:
                     if 'fever' in dataset_name:
                         to_add = False
 
-                    if ('xsum' in dataset_name or 'cnn' in dataset_name) and 'v2' in dataset_name:
+                    if ('xsum' in dataset_name or 'cnn' in dataset_name) and 'v2' not in dataset_name:
                         to_add = False
 
                     if isinstance(value, str):

cli/completed-cli.py

@@ -97,7 +97,7 @@ def process_finished_requests() -> bool:
     random.shuffle(eval_requests)
 
     from src.leaderboard.read_evals import get_raw_eval_results
-    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND, True)
+    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
 
     result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
     result_name_to_result = {r.eval_name: r for r in eval_results}
@@ -117,7 +117,7 @@ def process_finished_requests() -> bool:
                 eval_request: EvalRequest = result_name_to_request[result_name]
 
                 # print(eval_result)
-                print(result_name, 'is incomplete -- missing task:', task_name)
+                print(result_name, 'is incomplete -- missing task:', task_name, eval_result, eval_request.likes)
 
 
 if __name__ == "__main__":

cli/eval-cli.py

@@ -8,15 +8,21 @@ from src.backend.manage_requests import EvalRequest
 from src.backend.run_eval_suite import run_evaluation
 
 from src.backend.tasks.xsum.task import XSum
+from src.backend.tasks.xsum.task_v2 import XSumv2
+
 from src.backend.tasks.cnndm.task import CNNDM
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
+from src.backend.tasks.cnndm.task_v2 import CNNDMv2
+
+from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
 
-from lm_eval.tasks import initialize_tasks, include_task_folder
+from lm_eval.tasks import TaskManager
 from lm_eval import tasks, evaluator, utils
 
 from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
 from src.envs import QUEUE_REPO
 
+from lm_eval.models.huggingface import HFLM
+
 
 def main():
     # snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
@@ -29,15 +35,24 @@ def main():
     status = [PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS]
 
     # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-    eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
-    eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
+    eval_requests: list[EvalRequest] = get_eval_requests(job_status=status,
+                                                         hf_repo=QUEUE_REPO,
+                                                         local_dir=EVAL_REQUESTS_PATH_BACKEND,
+                                                         do_download=False)
+    # eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
+    eval_request = [r for r in eval_requests if 'meta-llama/Llama-2-7b-hf' in r.model][0]
 
     # my_task = Task("memo-trap", "acc", "memo-trap", 0)
     # my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
     # my_task = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
     # my_task = Task("truefalse_cieacf", "acc", "TrueFalse", 5)
     # my_task = Task("faithdial_hallu", "acc", "FaithDIAL", 2)
-    my_task = Task("fever11", "acc", "FEVER", 8)
+
+    # my_task = Task("nq_swap", "exact_match", "NQ-Swap", 2)
+    # my_task = Task("memo-trap_v2", "acc", "XXX", 2)
+    my_task = Task("xsum_v2", "rougeL", "XXX", 0)
+    # my_task = Task("squadv2", "exact", "XXX", 0)
+    # my_task = Task("scrolls_qasper", "f1", "XXX", 0)
 
     eval_logger = utils.eval_logger
     import logging
@@ -47,12 +62,13 @@ def main():
     # task_names = ['triviaqa']
     # TASKS_HARNESS = [task.value for task in Tasks]
 
-    include_task_folder("src/backend/tasks/")
-    initialize_tasks('INFO')
+    # include_task_folder("src/backend/tasks/")
+    task_manager = TaskManager(include_path="./src/backend/tasks/")
+    # task_manager.initialize_tasks(include_path="src/backend/tasks/")
 
     # breakpoint()
 
-    print(tasks.ALL_TASKS)
+    print(task_manager.all_tasks)
 
     for task in TASKS_HARNESS:
         print(f"Selected Tasks: [{task}]")
@@ -60,7 +76,7 @@ def main():
 
         # breakpoint()
         results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=task.num_fewshot,
-                                            batch_size=1, device="mps", use_cache=None, limit=1000, write_out=True)
+                                            batch_size=1, device="mps", use_cache=None, limit=2, write_out=True, task_manager=task_manager)
         print('AAA', results["results"])
 
         breakpoint()

cli/nqswap-upload-cli.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+from datasets import load_dataset
+
+path = 'pminervini/NQ-Swap'
+
+ds = load_dataset("json",
+                  data_files={
+                      'original': 'nqswap/original.jsonl',
+                      'substituted': 'nqswap/substituted.jsonl'
+                  })
+ds.push_to_hub(path)