update

cli/halueval-cli.py

@@ -8,6 +8,8 @@ from src.backend.manage_requests import EvalRequest
 from src.backend.run_eval_suite import run_evaluation
 
 from src.backend.tasks.xsum.task import XSum
+from src.backend.tasks.cnndm.task import CNNDM
+from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
 
 from lm_eval.tasks import initialize_tasks, include_task_folder
 from lm_eval import tasks, evaluator, utils
@@ -31,7 +33,7 @@ def main():
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
     # my_task = Task("memo-trap", "acc", "memo-trap", 0)
-    my_task = Task("xsum", "rougeLsum", "XSum", 2)
+    my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
 
     TASKS_HARNESS = [my_task]
     # task_names = ['triviaqa']

src/backend/envs.py

@@ -40,6 +40,8 @@ class Tasks(Enum):
     task11 = Task("nq8", "em", "NQ Open 8", 8)
     task12 = Task("tqa8", "em", "TriviaQA 8", 8)
 
+    task13 = Task("ifeval", "inst_level_strict_acc", "IFEval", 0)
+
 # NUM_FEWSHOT = 64  # Change with your few shot