from typing import List COLUMNS_PRETTY = { "bleu": "BLEU", "chrf": "ChrF", "rouge1": "ROUGE-1", "rouge2": "ROUGE-2", "rougeL": "ROUGE-L", "bertscore": "BERTScore", "bertscore_normalized": "BERTScore (Normalized)", "model_name": "Model Name", "model_availability": "Availability", "urls": "Resources", "context_size": "Context Size", "submitted_by": "Submitted By", "EM infile": "EM infile", "EM inproject": "EM inproject", "EM common": "EM common", "EM commited": "EM committed", "EM non_informative": "EM non-informative", "EM random": "EM random", "EM all": "EM all", "dataset": "Dataset", } METRICS_PER_TASK = { "commit_message_generation": [ "BLEU", "ChrF", "ROUGE-1", "ROUGE-2", "ROUGE-L", "BERTScore", "BERTScore (Normalized)", ], "project_code_completion": [ "EM infile", "EM inproject", "EM common", "EM committed", "EM non-informative", "EM random", "EM all", ] } SORT_COLUMN_PER_TASK = { "commit_message_generation": "ROUGE-1", "project_code_completion": "Dataset" } def get_columns_per_task(task_id: str) -> List[str]: metrics_per_task = METRICS_PER_TASK[task_id] if task_id == 'project_code_completion': return ["Model Name", "Availability", "Context Size", "Dataset"] + metrics_per_task + ["Submitted By", "Resources"] return ["Model Name", "Context Size"] + metrics_per_task + ["Availability", "Submitted By", "Resources"] def get_types_per_task(task_id: str) -> List[str]: metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0)) if task_id == 'project_code_completion': return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "html"] return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]