Display urls on the leaderboard, tweak interface a little and fix CMG descriptions

app.py

@@ -82,7 +82,7 @@ with gr.Blocks() as demo:
             with gr.Column():
                 url_textbox = gr.Textbox(
                     label="Relevant URLs",
-                    placeholder="URLs to relevant resources with additional details about your submission (optional).",
+                    placeholder='URLs to relevant resources with additional details about your submission (optional). Use the following format: "[text1](link1), [text2](link2)".',
                 )
                 model_availability_textbox = gr.Textbox(
                     label="Availability",
@@ -107,9 +107,12 @@ with gr.Blocks() as demo:
                 )
 
         gr.Markdown(SUBMISSION_TEXT_FILES, elem_classes="markdown-text")
-        task_specific_instructions = gr.Markdown(get_submission_text_files_for_task(None))
-        task_selection.select(get_submission_text_files_for_task, [task_selection], task_specific_instructions)
-        file_output = gr.File(file_count="multiple")
+        with gr.Row():
+            with gr.Column(variant="panel"):
+                task_specific_instructions = gr.Markdown(get_submission_text_files_for_task(None))
+                task_selection.select(get_submission_text_files_for_task, [task_selection], task_specific_instructions)
+            with gr.Column():
+                file_output = gr.File(file_count="multiple")
 
         gr.Markdown(SUBMISSION_TEXT_SUBMIT, elem_classes="markdown-text")
         submit_button = gr.Button("Submit")

src/formatting.py

@@ -10,5 +10,5 @@ def styled_message(message):
     return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
 
 
-def model_hyperlink(link, model_name):
+def model_hyperlink(model_name, link):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'

src/get_results_for_task.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import re
 
 import pandas as pd  # type: ignore[import]
 from datasets import get_dataset_config_names, load_dataset  # type: ignore[import]
@@ -12,6 +13,7 @@ from .leaderboard_formatting import (
     get_columns_per_task,
 )
 from .tasks_content import TASKS_PRETTY_REVERSE
+from .utils import MD_LINK_PATTERN
 
 try:
     AVAILABLE_TASKS = get_dataset_config_names(os.environ["DATASET_ID"])
@@ -32,6 +34,7 @@ def _get_results_stub() -> pd.DataFrame:
                 "BERTScore": "X",
                 "BERTScore (Normalized)": "X",
                 "Submitted By": "🏟 Long Code Arena Team",
+                "Resources": "",
             },
             {
                 "Model Name": "CodeLlama-7b (instruct)",
@@ -43,14 +46,24 @@ def _get_results_stub() -> pd.DataFrame:
                 "BERTScore": "X",
                 "BERTScore (Normalized)": "X",
                 "Submitted By": "🏟 Long Code Arena Team",
+                "Resources": "",
             },
         ]
     )
     return stub_df
 
 
+def _process_urls(raw_urls: str) -> str:
+    if not raw_urls:
+        return raw_urls
+    html_urls = [model_hyperlink(*re.search(MD_LINK_PATTERN, url.strip()).groups()) for url in raw_urls.split(",")]
+    return ", ".join(html_urls)
+
+
 def _get_results_dataset(task_id: str) -> pd.DataFrame:
-    results_df = load_dataset(os.environ["DATASET_ID"], task_id, split="test").to_pandas()
+    results_df = load_dataset(
+        os.environ["DATASET_ID"], task_id, split="test", download_mode="force_redownload"
+    ).to_pandas()
     results_df = results_df.rename(columns=COLUMNS_PRETTY, errors="ignore")
     results_df["Context Size"] = results_df["Context Size"].map(lambda x: f"{int(x) // 1000}k" if int(x) >= 1000 else x)
 
@@ -66,6 +79,7 @@ def _get_results_dataset(task_id: str) -> pd.DataFrame:
         model_hyperlink(link=link, model_name=model_name) if link else model_name
         for link, model_name in zip(results_df["model_url"], results_df["Model Name"])
     ]
+    results_df["Resources"] = [_process_urls(urls) for urls in results_df["Resources"]]
     results_df = results_df[get_columns_per_task(task_id)]
     return results_df
 

src/leaderboard_formatting.py

@@ -10,7 +10,7 @@ COLUMNS_PRETTY = {
     "bertscore_normalized": "BERTScore (Normalized)",
     "model_name": "Model Name",
     "model_availability": "Availability",
-    "urls": "URLs",
+    "urls": "Resources",
     "context_size": "Context Size",
     "submitted_by": "Submitted By",
 }
@@ -35,9 +35,9 @@ SORT_COLUMN_PER_TASK = {"commit_message_generation": "ROUGE-1"}
 def get_columns_per_task(task_id: str) -> List[str]:
     metrics_per_task = METRICS_PER_TASK[task_id]
 
-    return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By"]
+    return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
 
 
 def get_types_per_task(task_id: str) -> List[str]:
     metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
-    return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown"]
+    return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]

src/submission_uploader.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import re
 import time
 from tempfile import TemporaryDirectory
 from typing import List, Optional
@@ -13,6 +14,7 @@ from tqdm import tqdm
 from .evaluation import METRICS
 from .formatting import styled_error, styled_message, styled_warning
 from .tasks_content import TASKS_PRETTY_REVERSE
+from .utils import MD_LINK_PATTERN
 
 
 class AlreadyExists(Exception):
@@ -199,6 +201,12 @@ class SubmissionUploader:
         except:
             raise ValueError("Please, specify a model's context size as an integer (e.g., 16000).")
 
+        if urls is not None and "," in urls:
+            urls_list = urls.split(",")
+            assert all(
+                re.match(rf"^{MD_LINK_PATTERN}$", url.strip()) for url in urls_list
+            ), 'Please, use the following format for URLs: "[text1](link1), [text2](link2)"'
+
         assert submitted_by, "Please, specify non-empty information about a submission's author(s)."
         assert filenames, "Please, attach at least one file with predictions."
         assert contact_information, "Please, fill in the field with contact information."

src/tasks_content.py

@@ -13,7 +13,7 @@ TASKS_PRETTY_REVERSE = {value: key for key, value in TASKS_PRETTY.items()}
 TASKS_DESCRIPTIONS = {
     "commit_message_generation": """# Commit Message Generation\n
         
-        Our Commit Message Generation benchmark 🤗 [JetBrains-Research/lca-cmg](https://huggingface.co/datasets/JetBrains-Research/lca-cmg) includes 163 manually curated commits from Python projects.  
+        Our Commit Message Generation benchmark 🤗 [JetBrains-Research/lca-commit-message-generation](https://huggingface.co/datasets/JetBrains-Research/lca-commit-message-generation) includes 163 manually curated commits from Python projects.  
         
         We use the following metrics for evaluation:
         * [BLEU](https://huggingface.co/spaces/evaluate-metric/sacrebleu)
@@ -21,7 +21,9 @@ TASKS_DESCRIPTIONS = {
         * [ChrF](https://huggingface.co/spaces/evaluate-metric/chrf)
         * [BERTScore](https://huggingface.co/spaces/evaluate-metric/bertscore)
         
-        For further details on the dataset and the baselines from 🏟️ Long Code Arena Team, refer to `commit_message_generation` folder in [our baselines repository](https://github.com/JetBrains-Research/lca-baselines) or to our preprint (TODO).       
+        For further details on the dataset and the baselines from 🏟️ Long Code Arena Team, refer to `commit_message_generation` folder in [our baselines repository](https://github.com/JetBrains-Research/lca-baselines) or to our preprint (TODO).
+        
+        **Note.** The leaderboard is sorted by ROUGE-1 metric by default.       
         """,
     "bug_localization": "cool description for Bug Localization on Issue task",
     "module_to_text": "cool description for Module-to-Text task",

src/utils.py

@@ -0,0 +1 @@
+MD_LINK_PATTERN = r"\[(.*)\]\((.*?)\)"