update

cli/fix-requests-cli.py

@@ -41,7 +41,7 @@ for path in json_files:
                 data["model_type"] = "fine-tuned"
                 to_overwrite = True
 
-            is_instruction_tuned = 'nstruct' in model_id
+            is_instruction_tuned = ('nstruct' in model_id) or ('chat' in model_id)
             if is_instruction_tuned:
                 data["model_type"] = "instruction-tuned"
                 to_overwrite = True

cli/halueval-cli.py

@@ -7,6 +7,8 @@ from src.backend.manage_requests import get_eval_requests
 from src.backend.manage_requests import EvalRequest
 from src.backend.run_eval_suite import run_evaluation
 
+from src.backend.tasks.xsum.task import XSum
+
 from lm_eval.tasks import initialize_tasks, include_task_folder
 from lm_eval import tasks, evaluator, utils
 
@@ -15,7 +17,7 @@ from src.envs import QUEUE_REPO
 
 
 def main():
-    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+    # snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
 
     PENDING_STATUS = "PENDING"
     RUNNING_STATUS = "RUNNING"
@@ -28,7 +30,10 @@ def main():
     eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
-    TASKS_HARNESS = [t.value for t in Tasks if 'xsum' in t.value.benchmark]
+    # my_task = Task("memo-trap", "acc", "memo-trap", 0)
+    my_task = Task("xsum", "rougeLsum", "XSum", 2)
+
+    TASKS_HARNESS = [my_task]
     # task_names = ['triviaqa']
     # TASKS_HARNESS = [task.value for task in Tasks]
 

src/backend/envs.py

@@ -23,12 +23,18 @@ class Tasks(Enum):
     task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
     task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
     # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
+
     # task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
     task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
     task4 = Task("truthfulqa_mc2", "acc", "TruthfulQA MC2", 0)
+
     task5 = Task("halueval_qa", "acc", "HaluEval QA", 0)
+    # task6 = Task("halueval_dialogue", "acc", "HaluEval Dialogue", 0)
+    # task7 = Task("halueval_summarization", "acc", "HaluEval Summarization", 0)
+
     # task6 = Task("xsum", "rougeL_acc", "XSum", 8)
-    task6 = Task("memo-trap", "acc", "memo-trap", 0)
+
+    task8 = Task("memo-trap", "acc", "memo-trap", 0)
 
 # NUM_FEWSHOT = 64  # Change with your few shot