pminervini commited on
Commit
572d61c
1 Parent(s): e829671
src/backend/tasks/memo-trap/memo-trap.yaml ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ task: memo-trap
2
+ dataset_path: pminervini/inverse-scaling
3
+ dataset_name: memo-trap
4
+ output_type: multiple_choice
5
+ training_split: null
6
+ validation_split: data
7
+ test_split: null
8
+ num_fewshot: 0
9
+ doc_to_text: "{{prompt}}"
10
+ doc_to_target: answer_index
11
+ doc_to_choice: "{{classes}}"
12
+ should_decontaminate: False
13
+ doc_to_decontamination_query: prompt
14
+ metric_list:
15
+ - metric: acc
16
+ aggregation: mean
17
+ higher_is_better: true
18
+ metadata:
19
+ - version: 0.0