update

backend-cli.py

@@ -88,8 +88,8 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 
 def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
-    # batch_size = 1
-    batch_size = "auto"
+    batch_size = 1
+    # batch_size = "auto"
     results = run_evaluation(eval_request=eval_request, task_names=[task.benchmark], num_fewshot=task.num_fewshot,
                              batch_size=batch_size, device=DEVICE, use_cache=None, limit=LIMIT)
 

src/backend/manage_requests.py

@@ -26,7 +26,7 @@ class EvalRequest:
     license: Optional[str] = ""
     
     def get_model_args(self) -> str:
-        model_args = f"pretrained={self.model},revision={self.revision},parallelize=True,max_length=4096"
+        model_args = f"pretrained={self.model},revision={self.revision},parallelize=True" # ,max_length=4096"
 
         if self.precision in ["float16", "float32", "bfloat16"]:
             model_args += f",dtype={self.precision}"

src/backend/tasks/nq_swap/nq_swap.yaml

@@ -0,0 +1,31 @@
+task: nq_swap
+dataset_path: pminervini/NQ-Swap
+output_type: generate_until
+validation_split: substituted
+description: "Answer the following question based on the provided context:\n\n"
+doc_to_text: "Context: {{context}}\nQuestion: {{question}}?\nAnswer:"
+doc_to_target: "{{answer}}" # TODO: should be multi-target
+fewshot_delimiter: "\n\n"
+generation_kwargs:
+  until:
+    - "\n"
+    - "."
+    - ","
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+    - "\\b(?:The |the |An |A |The |a |an )"
+metadata:
+  version: 0.0