Muennighoff commited on
Commit
4e82575
1 Parent(s): e245f9e
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. config.json +30 -0
  3. evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json +9 -0
  4. evaluation_l1/Muennighoff_xstory_cloze/ar/Choose_Story_Ending/results.json +9 -0
  5. evaluation_l1/Muennighoff_xstory_cloze/ar/Generate_Ending/results.json +9 -0
  6. evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json +9 -0
  7. evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json +9 -0
  8. evaluation_l1/Muennighoff_xstory_cloze/es/Answer_Given_options/results.json +9 -0
  9. evaluation_l1/Muennighoff_xstory_cloze/es/Choose_Story_Ending/results.json +9 -0
  10. evaluation_l1/Muennighoff_xstory_cloze/es/Generate_Ending/results.json +9 -0
  11. evaluation_l1/Muennighoff_xstory_cloze/es/Novel_Correct_Ending/results.json +9 -0
  12. evaluation_l1/Muennighoff_xstory_cloze/es/Story_Continuation_and_Options/results.json +9 -0
  13. evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json +9 -0
  14. evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json +9 -0
  15. evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json +9 -0
  16. evaluation_l1/Muennighoff_xstory_cloze/eu/Story_Continuation_and_Options/results.json +9 -0
  17. evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json +9 -0
  18. evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json +9 -0
  19. evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json +9 -0
  20. evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json +9 -0
  21. evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json +9 -0
  22. evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json +9 -0
  23. evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json +9 -0
  24. evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json +9 -0
  25. evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json +9 -0
  26. evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json +9 -0
  27. evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json +9 -0
  28. evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json +9 -0
  29. evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json +9 -0
  30. evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json +9 -0
  31. evaluation_l1/Muennighoff_xwinograd/fr/underscore_refer_to/results.json +9 -0
  32. evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json +9 -0
  33. evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json +9 -0
  34. evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json +9 -0
  35. evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json +9 -0
  36. evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json +9 -0
  37. evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json +9 -0
  38. evaluation_l1/Muennighoff_xwinograd/zh/True_or_False/results.json +9 -0
  39. evaluation_l1/Muennighoff_xwinograd/zh/does_underscore_refer_to/results.json +9 -0
  40. evaluation_l1/Muennighoff_xwinograd/zh/stand_for/results.json +9 -0
  41. evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json +9 -0
  42. evaluation_l1/anli/dev_r1/GPT-3_style/results.json +9 -0
  43. evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json +9 -0
  44. evaluation_l1/anli/dev_r1/can_we_infer/results.json +9 -0
  45. evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json +9 -0
  46. evaluation_l1/anli/dev_r1/justified_in_saying/results.json +9 -0
  47. evaluation_l1/anli/dev_r2/GPT-3_style/results.json +9 -0
  48. evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json +9 -0
  49. evaluation_l1/anli/dev_r2/can_we_infer/results.json +9 -0
  50. evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json +9 -0
.gitattributes CHANGED
@@ -30,3 +30,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
33
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "apply_residual_connection_post_layernorm": false,
3
+ "attention_dropout": 0.0,
4
+ "architectures": [
5
+ "BloomModel"
6
+ ],
7
+ "attention_softmax_in_fp32": true,
8
+ "bias_dropout_fusion": true,
9
+ "bos_token_id": 1,
10
+ "eos_token_id": 2,
11
+ "pad_token_id": 3,
12
+ "unk_token_id": 0,
13
+ "hidden_dropout": 0.0,
14
+ "initializer_range": 0.02,
15
+ "layer_norm_epsilon": 1e-05,
16
+ "masked_softmax_fusion": true,
17
+ "model_type": "bloom",
18
+ "n_embed": 1024,
19
+ "n_inner": null,
20
+ "n_layer": 24,
21
+ "num_attention_heads": 16,
22
+ "offset_alibi": 100,
23
+ "pretraining_tp": 1,
24
+ "seq_length": 2048,
25
+ "skip_bias_add": true,
26
+ "skip_bias_add_qkv": false,
27
+ "transformers_version": "4.20.0",
28
+ "use_cache": true,
29
+ "vocab_size": 250880
30
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.5248180013236268
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.528788881535407
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5076108537392455
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5367306419589676
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.5274652547981469
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/es/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "es",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.49768365320979485
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/es/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "es",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5585704831237591
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/es/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "es",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5579086697551291
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/es/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "es",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5493050959629384
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/es/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "es",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.5532759761747187
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.4553275976174719
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5056254136333554
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.46790205162144277
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.4943745863666446
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5320979483785573
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5373924553275976
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5334215751158173
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.5201853077432164
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5651886168100596
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5453342157511581
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5592322964923891
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.5420251489080079
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5006451612903225
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5023655913978494
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.5180722891566265
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5180722891566265
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.43373493975903615
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.4939759036144578
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.4819277108433735
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.5019011406844106
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5095057034220533
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.49049429657794674
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.5209125475285171
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.4866920152091255
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.5416666666666666
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5476190476190477
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5436507936507936
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.49603174603174605
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5337301587301587
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/GPT-3_style/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "GPT-3 style",
5
+ "evaluation": {
6
+ "accuracy": 0.296
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "MNLI crowdsource",
5
+ "evaluation": {
6
+ "accuracy": 0.334
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/can_we_infer/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "can we infer",
5
+ "evaluation": {
6
+ "accuracy": 0.29
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "guaranteed/possible/impossible",
5
+ "evaluation": {
6
+ "accuracy": 0.333
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/justified_in_saying/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "justified in saying",
5
+ "evaluation": {
6
+ "accuracy": 0.281
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/GPT-3_style/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "GPT-3 style",
5
+ "evaluation": {
6
+ "accuracy": 0.32
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "MNLI crowdsource",
5
+ "evaluation": {
6
+ "accuracy": 0.334
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/can_we_infer/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "can we infer",
5
+ "evaluation": {
6
+ "accuracy": 0.315
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "guaranteed/possible/impossible",
5
+ "evaluation": {
6
+ "accuracy": 0.332
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/350mt0/bloomz-560m/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }