| { | |
| "model": { | |
| "trust_remote_code": "True", | |
| "model": "meta-llama/Meta-Llama-3-8B-Instruct", | |
| "api": "hf", | |
| "architecture": "LlamaForCausalLM", | |
| "max_length": 8192, | |
| "type": "instruction-tuned", | |
| "dtype": "bfloat16", | |
| "num_parameters": "8b" | |
| }, | |
| "results": [ | |
| { | |
| "name": "belebele_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.6633333333333333, | |
| "acc_norm": 0.6633333333333333 | |
| }, | |
| { | |
| "name": "exams_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.2697201017811705, | |
| "acc_norm": 0.3104325699745547 | |
| }, | |
| { | |
| "name": "check_worthiness", | |
| "task": "multiple_choice", | |
| "acc": 0.4218464351005484, | |
| "acc_norm": 0.5644424131627057 | |
| }, | |
| { | |
| "name": "ironytr", | |
| "task": "text_classification", | |
| "acc": 0.545, | |
| "acc_norm": 0.6466666666666666 | |
| }, | |
| { | |
| "name": "mkqa_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.0424681858538029, | |
| "f1": 0.11050423163975964 | |
| }, | |
| { | |
| "name": "mnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3201, | |
| "acc_norm": 0.3653 | |
| }, | |
| { | |
| "name": "news_cat", | |
| "task": "text_classification", | |
| "acc": 0.628, | |
| "acc_norm": 0.588 | |
| }, | |
| { | |
| "name": "offenseval_tr", | |
| "task": "text_classification", | |
| "acc": 0.3081065759637188, | |
| "acc_norm": 0.7304421768707483 | |
| }, | |
| { | |
| "name": "relevance_judgment", | |
| "task": "multiple_choice", | |
| "acc": 0.603290676416819, | |
| "acc_norm": 0.5790676416819013 | |
| }, | |
| { | |
| "name": "snli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3283, | |
| "acc_norm": 0.353 | |
| }, | |
| { | |
| "name": "sts_tr", | |
| "task": "text_classification", | |
| "acc": 0.14213197969543148, | |
| "acc_norm": 0.21537345902828137 | |
| }, | |
| { | |
| "name": "tquad", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.1289237668161435, | |
| "f1": 0.4134057883004977 | |
| }, | |
| { | |
| "name": "turkish_plu_goal_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.38829151732377537, | |
| "acc_norm": 0.43130227001194743 | |
| }, | |
| { | |
| "name": "turkish_plu_next_event_prediction", | |
| "task": "multiple_choice", | |
| "acc": 0.4549618320610687, | |
| "acc_norm": 0.517557251908397 | |
| }, | |
| { | |
| "name": "turkish_plu_step_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.3137254901960784, | |
| "acc_norm": 0.44281045751633985 | |
| }, | |
| { | |
| "name": "turkish_plu_step_ordering", | |
| "task": "multiple_choice", | |
| "acc": 0.6160626836434868, | |
| "acc_norm": 0.6160626836434868 | |
| }, | |
| { | |
| "name": "xcopa_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.586, | |
| "acc_norm": 0.586 | |
| }, | |
| { | |
| "name": "xnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.4389558232931727, | |
| "acc_norm": 0.4389558232931727 | |
| }, | |
| { | |
| "name": "xquad_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.09747899159663864, | |
| "f1": 0.24450355256139333 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.005007463045885695 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.40612528796779146, | |
| "rouge2": 0.25769550481564407, | |
| "rougeL": 0.3281187592669974 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.23621778991663983, | |
| "rouge2": 0.08052321922363763, | |
| "rougeL": 0.1710165526266978 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 0.823814082821166, | |
| "bleu": 0.13572050882587958 | |
| }, | |
| { | |
| "name": "tr-wikihow-summ", | |
| "task": "summarization", | |
| "rouge1": 0.2470049240628421, | |
| "rouge2": 0.07886512585932554, | |
| "rougeL": 0.17270243799905735 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.29619456321037296, | |
| "rouge2": 0.13520487191226377, | |
| "rougeL": 0.220446635816053 | |
| } | |
| ] | |
| } |