| { | |
| "results": { | |
| "assin2_rte": { | |
| "f1_macro,all": 0.4338803088803089, | |
| "f1_macro_stderr,all": 0.0064412473962861885, | |
| "acc,all": 0.5490196078431373, | |
| "acc_stderr,all": 0.007089614500405025, | |
| "alias": "assin2_rte" | |
| }, | |
| "assin2_sts": { | |
| "pearson,all": 0.06305387762906761, | |
| "pearson_stderr,all": 0.011844941303603512, | |
| "mse,all": 2.945608660130719, | |
| "mse_stderr,all": "N/A", | |
| "alias": "assin2_sts" | |
| }, | |
| "bluex": { | |
| "acc,all": 0.22809457579972184, | |
| "acc_stderr,all": 0.009036866632545269, | |
| "acc,exam_id__UNICAMP_2019": 0.24, | |
| "acc_stderr,exam_id__UNICAMP_2019": 0.03491203817579102, | |
| "acc,exam_id__USP_2021": 0.21153846153846154, | |
| "acc_stderr,exam_id__USP_2021": 0.03277032703346746, | |
| "acc,exam_id__USP_2019": 0.2, | |
| "acc_stderr,exam_id__USP_2019": 0.03647552543859845, | |
| "acc,exam_id__USP_2018": 0.2777777777777778, | |
| "acc_stderr,exam_id__USP_2018": 0.03517522991808499, | |
| "acc,exam_id__USP_2024": 0.21951219512195122, | |
| "acc_stderr,exam_id__USP_2024": 0.037373693975962165, | |
| "acc,exam_id__UNICAMP_2021_1": 0.2826086956521739, | |
| "acc_stderr,exam_id__UNICAMP_2021_1": 0.03830117454843993, | |
| "acc,exam_id__UNICAMP_2021_2": 0.2549019607843137, | |
| "acc_stderr,exam_id__UNICAMP_2021_2": 0.035207556573718095, | |
| "acc,exam_id__UNICAMP_2020": 0.2545454545454545, | |
| "acc_stderr,exam_id__UNICAMP_2020": 0.034035694534803984, | |
| "acc,exam_id__USP_2022": 0.20408163265306123, | |
| "acc_stderr,exam_id__USP_2022": 0.033232989277758127, | |
| "acc,exam_id__UNICAMP_2023": 0.20930232558139536, | |
| "acc_stderr,exam_id__UNICAMP_2023": 0.03573962694254937, | |
| "acc,exam_id__USP_2023": 0.18181818181818182, | |
| "acc_stderr,exam_id__USP_2023": 0.03355734384475543, | |
| "acc,exam_id__UNICAMP_2022": 0.28205128205128205, | |
| "acc_stderr,exam_id__UNICAMP_2022": 0.04167152779427693, | |
| "acc,exam_id__UNICAMP_2024": 0.28888888888888886, | |
| "acc_stderr,exam_id__UNICAMP_2024": 0.039043511589523974, | |
| "acc,exam_id__UNICAMP_2018": 0.18518518518518517, | |
| "acc_stderr,exam_id__UNICAMP_2018": 0.030512369270458792, | |
| "acc,exam_id__USP_2020": 0.14285714285714285, | |
| "acc_stderr,exam_id__USP_2020": 0.0270317754368945, | |
| "alias": "bluex" | |
| }, | |
| "enem_challenge": { | |
| "alias": "enem", | |
| "acc,all": 0.20433869839048285, | |
| "acc_stderr,all": 0.0061656410435202685, | |
| "acc,exam_id__2016": 0.19834710743801653, | |
| "acc_stderr,exam_id__2016": 0.020959354242452308, | |
| "acc,exam_id__2015": 0.2184873949579832, | |
| "acc_stderr,exam_id__2015": 0.021859047894972602, | |
| "acc,exam_id__2016_2": 0.2032520325203252, | |
| "acc_stderr,exam_id__2016_2": 0.020922533176615875, | |
| "acc,exam_id__2022": 0.21052631578947367, | |
| "acc_stderr,exam_id__2022": 0.020364275440710126, | |
| "acc,exam_id__2023": 0.22962962962962963, | |
| "acc_stderr,exam_id__2023": 0.02090627658961272, | |
| "acc,exam_id__2017": 0.20689655172413793, | |
| "acc_stderr,exam_id__2017": 0.021648507169879216, | |
| "acc,exam_id__2009": 0.2, | |
| "acc_stderr,exam_id__2009": 0.02159332165710835, | |
| "acc,exam_id__2011": 0.19658119658119658, | |
| "acc_stderr,exam_id__2011": 0.02128761226602946, | |
| "acc,exam_id__2010": 0.19658119658119658, | |
| "acc_stderr,exam_id__2010": 0.0212068848192361, | |
| "acc,exam_id__2014": 0.25688073394495414, | |
| "acc_stderr,exam_id__2014": 0.02416729521302893, | |
| "acc,exam_id__2012": 0.19827586206896552, | |
| "acc_stderr,exam_id__2012": 0.021434003593162023, | |
| "acc,exam_id__2013": 0.12962962962962962, | |
| "acc_stderr,exam_id__2013": 0.01863446504359953 | |
| }, | |
| "faquad_nli": { | |
| "f1_macro,all": 0.4396551724137931, | |
| "f1_macro_stderr,all": 0.0035796984729087084, | |
| "acc,all": 0.7846153846153846, | |
| "acc_stderr,all": 0.011396120309131327, | |
| "alias": "faquad_nli" | |
| }, | |
| "hatebr_offensive": { | |
| "alias": "hatebr_offensive_binary", | |
| "f1_macro,all": 0.2769855132917938, | |
| "f1_macro_stderr,all": 0.006238058759395454, | |
| "acc,all": 0.425, | |
| "acc_stderr,all": 0.00935342459207753 | |
| }, | |
| "oab_exams": { | |
| "acc,all": 0.24829157175398633, | |
| "acc_stderr,all": 0.005311657614966229, | |
| "acc,exam_id__2015-17": 0.2564102564102564, | |
| "acc_stderr,exam_id__2015-17": 0.028537581067717782, | |
| "acc,exam_id__2011-03": 0.24242424242424243, | |
| "acc_stderr,exam_id__2011-03": 0.02489486644457539, | |
| "acc,exam_id__2014-14": 0.2875, | |
| "acc_stderr,exam_id__2014-14": 0.029212677784729673, | |
| "acc,exam_id__2010-01": 0.21176470588235294, | |
| "acc_stderr,exam_id__2010-01": 0.025617056537777887, | |
| "acc,exam_id__2016-19": 0.24358974358974358, | |
| "acc_stderr,exam_id__2016-19": 0.028079011331469186, | |
| "acc,exam_id__2017-24": 0.25, | |
| "acc_stderr,exam_id__2017-24": 0.0279587130919447, | |
| "acc,exam_id__2016-20": 0.225, | |
| "acc_stderr,exam_id__2016-20": 0.02696032277512517, | |
| "acc,exam_id__2012-07": 0.2625, | |
| "acc_stderr,exam_id__2012-07": 0.028342009973799126, | |
| "acc,exam_id__2012-08": 0.2875, | |
| "acc_stderr,exam_id__2012-08": 0.029159129736545907, | |
| "acc,exam_id__2016-21": 0.25, | |
| "acc_stderr,exam_id__2016-21": 0.027932015464899256, | |
| "acc,exam_id__2017-22": 0.2625, | |
| "acc_stderr,exam_id__2017-22": 0.02841496508694686, | |
| "acc,exam_id__2014-13": 0.2375, | |
| "acc_stderr,exam_id__2014-13": 0.02746965327978516, | |
| "acc,exam_id__2017-23": 0.2, | |
| "acc_stderr,exam_id__2017-23": 0.02582607855525874, | |
| "acc,exam_id__2015-18": 0.2125, | |
| "acc_stderr,exam_id__2015-18": 0.026296568931969733, | |
| "acc,exam_id__2012-06": 0.2625, | |
| "acc_stderr,exam_id__2012-06": 0.028412779857701947, | |
| "acc,exam_id__2014-15": 0.3333333333333333, | |
| "acc_stderr,exam_id__2014-15": 0.03076950178202261, | |
| "acc,exam_id__2012-06a": 0.2125, | |
| "acc_stderr,exam_id__2012-06a": 0.026352683979907383, | |
| "acc,exam_id__2015-16": 0.2375, | |
| "acc_stderr,exam_id__2015-16": 0.027486163755159596, | |
| "acc,exam_id__2016-20a": 0.2375, | |
| "acc_stderr,exam_id__2016-20a": 0.027439589316868403, | |
| "acc,exam_id__2013-12": 0.2, | |
| "acc_stderr,exam_id__2013-12": 0.025795418452346924, | |
| "acc,exam_id__2010-02": 0.26, | |
| "acc_stderr,exam_id__2010-02": 0.02536548615872908, | |
| "acc,exam_id__2011-05": 0.2375, | |
| "acc_stderr,exam_id__2011-05": 0.027446350611756456, | |
| "acc,exam_id__2011-04": 0.1875, | |
| "acc_stderr,exam_id__2011-04": 0.025291591821421322, | |
| "acc,exam_id__2012-09": 0.2597402597402597, | |
| "acc_stderr,exam_id__2012-09": 0.028906532222162228, | |
| "acc,exam_id__2013-10": 0.2875, | |
| "acc_stderr,exam_id__2013-10": 0.029178862116368304, | |
| "acc,exam_id__2018-25": 0.2875, | |
| "acc_stderr,exam_id__2018-25": 0.029301396141291788, | |
| "acc,exam_id__2013-11": 0.275, | |
| "acc_stderr,exam_id__2013-11": 0.028884882319914113, | |
| "alias": "oab_exams" | |
| }, | |
| "portuguese_hate_speech": { | |
| "alias": "portuguese_hate_speech_binary", | |
| "f1_macro,all": 0.29184578119465004, | |
| "f1_macro_stderr,all": 0.020412302602489146, | |
| "acc,all": 0.6733254994124559, | |
| "acc_stderr,all": 0.011336369997977094 | |
| }, | |
| "tweetsentbr": { | |
| "f1_macro,all": 0.43107917544524593, | |
| "f1_macro_stderr,all": 0.007482157906358252, | |
| "acc,all": 0.4880597014925373, | |
| "acc_stderr,all": 0.007890279587055024, | |
| "alias": "tweetsentbr" | |
| } | |
| }, | |
| "configs": { | |
| "assin2_rte": { | |
| "task": "assin2_rte", | |
| "group": [ | |
| "pt_benchmark", | |
| "assin2" | |
| ], | |
| "dataset_path": "assin2", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Premissa: {{premise}}\nHipótese: {{hypothesis}}\nPergunta: A hipótese pode ser inferida pela premissa? Sim ou Não?\nResposta:", | |
| "doc_to_target": "{{['Não', 'Sim'][entailment_judgment]}}", | |
| "description": "Abaixo estão pares de premissa e hipótese. Para cada par, indique se a hipótese pode ser inferida a partir da premissa, responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 1, | |
| 3251, | |
| 2, | |
| 3252, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 3253, | |
| 7, | |
| 3254, | |
| 3255, | |
| 3256, | |
| 8, | |
| 9, | |
| 10, | |
| 3257, | |
| 11, | |
| 3258, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 3259, | |
| 3260, | |
| 3261, | |
| 3262, | |
| 3263, | |
| 16, | |
| 17, | |
| 3264, | |
| 18, | |
| 3265, | |
| 3266, | |
| 3267, | |
| 19, | |
| 20, | |
| 3268, | |
| 3269, | |
| 21, | |
| 3270, | |
| 3271, | |
| 22, | |
| 3272, | |
| 3273, | |
| 23, | |
| 3274, | |
| 24, | |
| 25, | |
| 3275 | |
| ], | |
| "id_column": "sentence_pair_id" | |
| } | |
| }, | |
| "num_fewshot": 15, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "assin2_sts": { | |
| "task": "assin2_sts", | |
| "group": [ | |
| "pt_benchmark", | |
| "assin2" | |
| ], | |
| "dataset_path": "assin2", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Frase 1: {{premise}}\nFrase 2: {{hypothesis}}\nPergunta: Quão similares são as duas frases? Dê uma pontuação entre 1,0 a 5,0.\nResposta:", | |
| "doc_to_target": "<function assin2_float_to_pt_str at 0x1456ca4ce5c0>", | |
| "description": "Abaixo estão pares de frases que você deve avaliar o grau de similaridade. Dê uma pontuação entre 1,0 e 5,0, sendo 1,0 pouco similar e 5,0 muito similar.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 1, | |
| 3251, | |
| 2, | |
| 3252, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 3253, | |
| 7, | |
| 3254, | |
| 3255, | |
| 3256, | |
| 8, | |
| 9, | |
| 10, | |
| 3257, | |
| 11, | |
| 3258, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 3259, | |
| 3260, | |
| 3261, | |
| 3262, | |
| 3263, | |
| 16, | |
| 17, | |
| 3264, | |
| 18, | |
| 3265, | |
| 3266, | |
| 3267, | |
| 19, | |
| 20, | |
| 3268, | |
| 3269, | |
| 21, | |
| 3270, | |
| 3271, | |
| 22, | |
| 3272, | |
| 3273, | |
| 23, | |
| 3274, | |
| 24, | |
| 25, | |
| 3275 | |
| ], | |
| "id_column": "sentence_pair_id" | |
| } | |
| }, | |
| "num_fewshot": 10, | |
| "metric_list": [ | |
| { | |
| "metric": "pearson", | |
| "aggregation": "pearsonr", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "mse", | |
| "aggregation": "mean_squared_error", | |
| "higher_is_better": false | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "number_filter", | |
| "type": "float", | |
| "range_min": 1.0, | |
| "range_max": 5.0, | |
| "on_outside_range": "clip", | |
| "fallback": 5.0 | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "bluex": { | |
| "task": "bluex", | |
| "group": [ | |
| "pt_benchmark", | |
| "vestibular" | |
| ], | |
| "dataset_path": "eduagarcia-temp/BLUEX_without_images", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function enem_doc_to_text at 0x1456ca4cdb20>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha de provas de vestibular de universidades brasileiras, selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\", \"D\" ou \"E\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "USP_2018_3", | |
| "UNICAMP_2018_2", | |
| "USP_2018_35", | |
| "UNICAMP_2018_16", | |
| "USP_2018_89" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D", | |
| "E" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCDE])\\b", | |
| "\\b([ABCDE])\\.", | |
| "\\b([ABCDE]) ?[.):-]", | |
| "\\b([ABCDE])$", | |
| "\\b([ABCDE])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function enem_doc_to_text at 0x1456ca4cde40>", | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "enem_challenge": { | |
| "task": "enem_challenge", | |
| "task_alias": "enem", | |
| "group": [ | |
| "pt_benchmark", | |
| "vestibular" | |
| ], | |
| "dataset_path": "eduagarcia/enem_challenge", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function enem_doc_to_text at 0x1456ca4ce020>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha do Exame Nacional do Ensino Médio (ENEM), selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\", \"D\" ou \"E\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "2022_21", | |
| "2022_88", | |
| "2022_143" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D", | |
| "E" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCDE])\\b", | |
| "\\b([ABCDE])\\.", | |
| "\\b([ABCDE]) ?[.):-]", | |
| "\\b([ABCDE])$", | |
| "\\b([ABCDE])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function enem_doc_to_text at 0x1456ca4ce200>", | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "faquad_nli": { | |
| "task": "faquad_nli", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "ruanchaves/faquad-nli", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Pergunta: {{question}}\nResposta: {{answer}}\nA resposta dada satisfaz à pergunta? Sim ou Não?", | |
| "doc_to_target": "{{['Não', 'Sim'][label]}}", | |
| "description": "Abaixo estão pares de pergunta e resposta. Para cada par, você deve julgar se a resposta responde à pergunta de maneira satisfatória e aparenta estar correta. Escreva apenas \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n", | |
| "sampler_config": { | |
| "fewshot_indices": [ | |
| 1893, | |
| 949, | |
| 663, | |
| 105, | |
| 1169, | |
| 2910, | |
| 2227, | |
| 2813, | |
| 974, | |
| 558, | |
| 1503, | |
| 1958, | |
| 2918, | |
| 601, | |
| 1560, | |
| 984, | |
| 2388, | |
| 995, | |
| 2233, | |
| 1982, | |
| 165, | |
| 2788, | |
| 1312, | |
| 2285, | |
| 522, | |
| 1113, | |
| 1670, | |
| 323, | |
| 236, | |
| 1263, | |
| 1562, | |
| 2519, | |
| 1049, | |
| 432, | |
| 1167, | |
| 1394, | |
| 2022, | |
| 2551, | |
| 2194, | |
| 2187, | |
| 2282, | |
| 2816, | |
| 108, | |
| 301, | |
| 1185, | |
| 1315, | |
| 1420, | |
| 2436, | |
| 2322, | |
| 766 | |
| ] | |
| } | |
| }, | |
| "num_fewshot": 15, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "hatebr_offensive": { | |
| "task": "hatebr_offensive", | |
| "task_alias": "hatebr_offensive_binary", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/portuguese_benchmark", | |
| "dataset_name": "HateBR_offensive_binary", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O texto é ofensivo?\nResposta:", | |
| "doc_to_target": "{{'Sim' if label == 1 else 'Não'}}", | |
| "description": "Abaixo contém o texto de comentários de usuários do Instagram em português, sua tarefa é classificar se o texto é ofensivo ou não. Responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 48, | |
| 44, | |
| 36, | |
| 20, | |
| 3511, | |
| 88, | |
| 3555, | |
| 16, | |
| 56, | |
| 3535, | |
| 60, | |
| 40, | |
| 3527, | |
| 4, | |
| 76, | |
| 3579, | |
| 3523, | |
| 3551, | |
| 68, | |
| 3503, | |
| 84, | |
| 3539, | |
| 64, | |
| 3599, | |
| 80, | |
| 3563, | |
| 3559, | |
| 3543, | |
| 3547, | |
| 3587, | |
| 3595, | |
| 3575, | |
| 3567, | |
| 3591, | |
| 24, | |
| 96, | |
| 92, | |
| 3507, | |
| 52, | |
| 72, | |
| 8, | |
| 3571, | |
| 3515, | |
| 3519, | |
| 3531, | |
| 28, | |
| 32, | |
| 0, | |
| 12, | |
| 3583 | |
| ], | |
| "id_column": "idx" | |
| } | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "oab_exams": { | |
| "task": "oab_exams", | |
| "group": [ | |
| "legal_benchmark", | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/oab_exams", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function doc_to_text at 0x1456ca4ced40>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha do Exame de Ordem da Ordem dos Advogados do Brasil (OAB), selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\" ou \"D\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "2010-01_1", | |
| "2010-01_11", | |
| "2010-01_13", | |
| "2010-01_23", | |
| "2010-01_26", | |
| "2010-01_28", | |
| "2010-01_38", | |
| "2010-01_48", | |
| "2010-01_58", | |
| "2010-01_68", | |
| "2010-01_76", | |
| "2010-01_83", | |
| "2010-01_85", | |
| "2010-01_91", | |
| "2010-01_99" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCD])\\b", | |
| "\\b([ABCD])\\.", | |
| "\\b([ABCD]) ?[.):-]", | |
| "\\b([ABCD])$", | |
| "\\b([ABCD])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function doc_to_text at 0x1456ca4cefc0>", | |
| "metadata": { | |
| "version": 1.5 | |
| } | |
| }, | |
| "portuguese_hate_speech": { | |
| "task": "portuguese_hate_speech", | |
| "task_alias": "portuguese_hate_speech_binary", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/portuguese_benchmark", | |
| "dataset_name": "Portuguese_Hate_Speech_binary", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O texto contém discurso de ódio?\nResposta:", | |
| "doc_to_target": "{{'Sim' if label == 1 else 'Não'}}", | |
| "description": "Abaixo contém o texto de tweets de usuários do Twitter em português, sua tarefa é classificar se o texto contém discurso de ódio ou não. Responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 52, | |
| 50, | |
| 39, | |
| 28, | |
| 3, | |
| 105, | |
| 22, | |
| 25, | |
| 60, | |
| 11, | |
| 66, | |
| 41, | |
| 9, | |
| 4, | |
| 91, | |
| 42, | |
| 7, | |
| 20, | |
| 76, | |
| 1, | |
| 104, | |
| 13, | |
| 67, | |
| 54, | |
| 97, | |
| 27, | |
| 24, | |
| 14, | |
| 16, | |
| 48, | |
| 53, | |
| 40, | |
| 34, | |
| 49, | |
| 32, | |
| 119, | |
| 114, | |
| 2, | |
| 58, | |
| 83, | |
| 18, | |
| 36, | |
| 5, | |
| 6, | |
| 10, | |
| 35, | |
| 38, | |
| 0, | |
| 21, | |
| 46 | |
| ], | |
| "id_column": "idx" | |
| } | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "tweetsentbr": { | |
| "task": "tweetsentbr", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/tweetsentbr_fewshot", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O sentimento do texto é Positivo, Neutro ou Negativo?\nResposta:", | |
| "doc_to_target": "{{'Positivo' if label == 'Positive' else ('Negativo' if label == 'Negative' else 'Neutro')}}", | |
| "description": "Abaixo contém o texto de tweets de usuários do Twitter em português, sua tarefa é classificar se o sentimento do texto é Positivo, Neutro ou Negativo. Responda apenas com uma das opções.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Positivo", | |
| "Neutro", | |
| "Negativo" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "assin2_rte": 1.1, | |
| "assin2_sts": 1.1, | |
| "bluex": 1.1, | |
| "enem_challenge": 1.1, | |
| "faquad_nli": 1.1, | |
| "hatebr_offensive": 1.0, | |
| "oab_exams": 1.5, | |
| "portuguese_hate_speech": 1.0, | |
| "tweetsentbr": 1.0 | |
| }, | |
| "n-shot": { | |
| "assin2_rte": 15, | |
| "assin2_sts": 10, | |
| "bluex": 3, | |
| "enem_challenge": 3, | |
| "faquad_nli": 15, | |
| "hatebr_offensive": 25, | |
| "oab_exams": 3, | |
| "portuguese_hate_speech": 25, | |
| "tweetsentbr": 25 | |
| }, | |
| "model_meta": { | |
| "truncated": 0, | |
| "non_truncated": 14150, | |
| "padded": 0, | |
| "non_padded": 14150, | |
| "fewshots_truncated": 0, | |
| "has_chat_template": true, | |
| "chat_type": "user_assistant", | |
| "n_gpus": 1, | |
| "accelerate_num_process": null, | |
| "model_sha": "None", | |
| "model_dtype": "torch.float32", | |
| "model_memory_footprint": 9778521920, | |
| "model_num_parameters": 2444628480, | |
| "model_is_loaded_in_4bit": null, | |
| "model_is_loaded_in_8bit": null, | |
| "model_is_quantized": null, | |
| "model_device": "cuda:0", | |
| "batch_size": 16, | |
| "max_length": 4096, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32 | |
| }, | |
| "task_model_meta": { | |
| "assin2_rte": { | |
| "sample_size": 2448, | |
| "truncated": 0, | |
| "non_truncated": 2448, | |
| "padded": 0, | |
| "non_padded": 2448, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1061.423202614379, | |
| "min_seq_length": 1046, | |
| "max_seq_length": 1100, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 15.0, | |
| "mean_effective_fewshot_size": 15.0 | |
| }, | |
| "assin2_sts": { | |
| "sample_size": 2448, | |
| "truncated": 0, | |
| "non_truncated": 2448, | |
| "padded": 0, | |
| "non_padded": 2448, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 747.4232026143791, | |
| "min_seq_length": 732, | |
| "max_seq_length": 786, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 10.0, | |
| "mean_effective_fewshot_size": 10.0 | |
| }, | |
| "bluex": { | |
| "sample_size": 719, | |
| "truncated": 0, | |
| "non_truncated": 719, | |
| "padded": 0, | |
| "non_padded": 719, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1198.817802503477, | |
| "min_seq_length": 932, | |
| "max_seq_length": 1829, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 3.0 | |
| }, | |
| "enem_challenge": { | |
| "sample_size": 1429, | |
| "truncated": 0, | |
| "non_truncated": 1429, | |
| "padded": 0, | |
| "non_padded": 1429, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1035.4177746675998, | |
| "min_seq_length": 857, | |
| "max_seq_length": 2512, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 3.0 | |
| }, | |
| "faquad_nli": { | |
| "sample_size": 650, | |
| "truncated": 0, | |
| "non_truncated": 650, | |
| "padded": 0, | |
| "non_padded": 650, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1083.1338461538462, | |
| "min_seq_length": 1051, | |
| "max_seq_length": 1149, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 15.0, | |
| "mean_effective_fewshot_size": 15.0 | |
| }, | |
| "hatebr_offensive": { | |
| "sample_size": 1400, | |
| "truncated": 0, | |
| "non_truncated": 1400, | |
| "padded": 0, | |
| "non_padded": 1400, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1090.4407142857142, | |
| "min_seq_length": 1075, | |
| "max_seq_length": 1284, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| }, | |
| "oab_exams": { | |
| "sample_size": 2195, | |
| "truncated": 0, | |
| "non_truncated": 2195, | |
| "padded": 0, | |
| "non_padded": 2195, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 863.024145785877, | |
| "min_seq_length": 690, | |
| "max_seq_length": 1139, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 3.0 | |
| }, | |
| "portuguese_hate_speech": { | |
| "sample_size": 851, | |
| "truncated": 0, | |
| "non_truncated": 851, | |
| "padded": 0, | |
| "non_padded": 851, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1442.021151586369, | |
| "min_seq_length": 1415, | |
| "max_seq_length": 1478, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| }, | |
| "tweetsentbr": { | |
| "sample_size": 2010, | |
| "truncated": 0, | |
| "non_truncated": 2010, | |
| "padded": 0, | |
| "non_padded": 2010, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1370.4194029850746, | |
| "min_seq_length": 1353, | |
| "max_seq_length": 1427, | |
| "max_ctx_length": 4064, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| } | |
| }, | |
| "config": { | |
| "model": "huggingface", | |
| "model_args": "pretrained=/lustre/mlnvme/data/asen_hpc-mula/Tucano-2b4-DPO", | |
| "batch_size": "auto", | |
| "batch_sizes": [], | |
| "device": "cuda:0", | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null | |
| }, | |
| "git_hash": null | |
| } |