Added english tasks

custom_tasks.py

@@ -9,11 +9,13 @@ Author:
 from src.custom_tasks.heq_task import *
 from src.custom_tasks.sentiment_task import *
 from src.custom_tasks.winograd_task import *
+from src.custom_tasks.commonsense_task import *
+from src.custom_tasks.arc_challenge_task import *
 
 ## MODULE LOGIC
 # You should not need to touch this
 # Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in [heq_task, sentiment_task, winograd_task]]
+TASKS_TABLE = [task.as_dict() for task in [heq_task, sentiment_task, winograd_task, commonsense_qa_task, arc_challenge_task]]
 
 if __name__ == "__main__":
     print(t["name"] for t in TASKS_TABLE)

requirements.txt

@@ -14,7 +14,7 @@ tqdm==4.65.0
 transformers
 tokenizers>=0.15.0
 # git+https://github.com/huggingface/lighteval.git#egg=lighteval
-git+https://github.com/shaltielshmid/lighteval.git@temp-for-heb-leaderboard#egg=lighteval
+git+https://github.com/shaltielshmid/lighteval.git@fix-greedy-generate-bugs#egg=lighteval
 accelerate==0.24.1
 sentencepiece
 Levenshtein

src/about.py

@@ -21,5 +21,5 @@ TASKS_HARNESS = [task.value.benchmark for task in Tasks]
 # ---------------------------------------------------
 
 # TASKS_LIGHTEVAL = "lighteval|anli:r1|0|0,lighteval|logiqa|0|0" 
-tasks = ['heq-qa-tlnls', 'sentiment-acc', 'winograd-acc']
+tasks = ['heq-qa-tlnls', 'sentiment-acc', 'winograd-acc', 'arc:challenge']
 TASKS_LIGHTEVAL = ','.join(f'custom|{t}|0|0' for t in tasks)

src/backend/run_eval_suite_lighteval.py

@@ -22,6 +22,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
 
     args = DefaultNamespace(**{
             "endpoint_model_name": eval_request.model,
+            "model_dtype": eval_request.precision,
             "accelerator": accelerator,
             "vendor": vendor,
             "region": region,
@@ -41,8 +42,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
             "tasks": task_names,
             "dataset_loading_processes": 24,
             "num_fewshot_seeds": 0,
-            "reuse_existing": False,
-            "model_info": ModelInfo(eval_request.model, eval_request.revision, eval_request.precision, eval_request.params),
+            "reuse_existing": False
     })
 
     try:

src/custom_tasks/arc_challenge_task.py

@@ -0,0 +1,24 @@
+import re
+import string
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.metrics import Metrics, MetricCategory
+from lighteval.metrics.utils import CorpusLevelMetric, MetricUseCase
+from aenum import extend_enum
+import numpy as np
+from lighteval.tasks.requests import Doc
+from Levenshtein import distance
+import collections
+from lighteval.utils import as_list
+from lighteval.tasks.tasks_prompt_formatting import LETTER_INDICES
+
+arc_challenge_task = LightevalTaskConfig(
+    name="arc:challenge",
+    prompt_function="arc",
+    hf_repo="ai2_arc",
+    hf_subset="ARC-Challenge",
+    evaluation_splits=["test"],
+    generation_size=1,
+    metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+    trust_dataset=True,
+    stop_sequence=["\n"],
+)

src/custom_tasks/commonsense_task.py

@@ -0,0 +1,31 @@
+import re
+import string
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.metrics import Metrics, MetricCategory
+from lighteval.metrics.utils import CorpusLevelMetric, MetricUseCase
+from aenum import extend_enum
+import numpy as np
+from lighteval.tasks.requests import Doc
+from Levenshtein import distance
+import collections
+from lighteval.utils import as_list
+from lighteval.tasks.tasks_prompt_formatting import LETTER_INDICES
+
+def commonsense_qa_prompt(line, task_name: str = None):
+    return Doc(
+        task_name=task_name,
+        query=line["question"],
+        choices=[f" {c}" for c in line["choices"]["text"]],
+        gold_index=LETTER_INDICES.index(line["answerKey"].strip()),
+        instruction="",
+    )
+
+commonsense_qa_task = LightevalTaskConfig(
+    name="commonsense_qa",
+    prompt_function="commonsense_qa_prompt",
+    hf_repo="commonsense_qa",
+    hf_subset="default",
+    metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+    trust_dataset=True,
+    stop_sequence=["\n"],
+),

src/custom_tasks/heq_task.py

@@ -112,5 +112,4 @@ heq_task = LightevalTaskConfig(
     metric=['heq_tlnls_metric'],
     stop_sequence=['\n'],
     generation_size=64
-)
-heq_task.stop_sequence = as_list(heq_task.stop_sequence)
+)

src/custom_tasks/sentiment_task.py

@@ -56,5 +56,4 @@ sentiment_task = LightevalTaskConfig(
     metric=['sentiment_acc_metric'],
     stop_sequence=['\n'],
     generation_size=32
-)
-sentiment_task.stop_sequence = as_list(sentiment_task.stop_sequence)
+)

src/custom_tasks/winograd_task.py

@@ -53,5 +53,4 @@ winograd_task = LightevalTaskConfig(
     metric=['winograd_acc_metric'],
     stop_sequence=['\n'],
     generation_size=32
-)
-winograd_task.stop_sequence = as_list(winograd_task.stop_sequence)
+)