Spaces:
Runtime error
Runtime error
pminervini
commited on
Commit
•
572d61c
1
Parent(s):
e829671
update
Browse files
src/backend/tasks/memo-trap/memo-trap.yaml
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
task: memo-trap
|
2 |
+
dataset_path: pminervini/inverse-scaling
|
3 |
+
dataset_name: memo-trap
|
4 |
+
output_type: multiple_choice
|
5 |
+
training_split: null
|
6 |
+
validation_split: data
|
7 |
+
test_split: null
|
8 |
+
num_fewshot: 0
|
9 |
+
doc_to_text: "{{prompt}}"
|
10 |
+
doc_to_target: answer_index
|
11 |
+
doc_to_choice: "{{classes}}"
|
12 |
+
should_decontaminate: False
|
13 |
+
doc_to_decontamination_query: prompt
|
14 |
+
metric_list:
|
15 |
+
- metric: acc
|
16 |
+
aggregation: mean
|
17 |
+
higher_is_better: true
|
18 |
+
metadata:
|
19 |
+
- version: 0.0
|