Updated leaderboard

.gitignore

@@ -6,8 +6,4 @@ __pycache__/
 *ipynb
 .vscode/
 
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
 logs/

app.py

@@ -3,6 +3,7 @@ import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+from pathlib import Path
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -26,7 +27,7 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
@@ -34,30 +35,34 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+
+results_files = [fn for fn in Path(EVAL_RESULTS_PATH).glob("*.json")]
+if not len(results_files):
+    print(f"No results found in results path: {EVAL_RESULTS_PATH}")
+
+
+raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
 # Searching and filtering
@@ -242,92 +247,92 @@ with demo:
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        #     with gr.Column():
+        #         with gr.Row():
+        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+        #
+        #         with gr.Column():
+        #             with gr.Accordion(
+        #                 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     finished_eval_table = gr.components.Dataframe(
+        #                         value=finished_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #             with gr.Accordion(
+        #                 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     running_eval_table = gr.components.Dataframe(
+        #                         value=running_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #
+        #             with gr.Accordion(
+        #                 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     pending_eval_table = gr.components.Dataframe(
+        #                         value=pending_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #     with gr.Row():
+        #         gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+        #
+        #     with gr.Row():
+        #         with gr.Column():
+        #             model_name_textbox = gr.Textbox(label="Model name")
+        #             revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+        #             model_type = gr.Dropdown(
+        #                 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+        #                 label="Model type",
+        #                 multiselect=False,
+        #                 value=None,
+        #                 interactive=True,
+        #             )
+        #
+        #         with gr.Column():
+        #             precision = gr.Dropdown(
+        #                 choices=[i.value.name for i in Precision if i != Precision.Unknown],
+        #                 label="Precision",
+        #                 multiselect=False,
+        #                 value="float16",
+        #                 interactive=True,
+        #             )
+        #             weight_type = gr.Dropdown(
+        #                 choices=[i.value.name for i in WeightType],
+        #                 label="Weights type",
+        #                 multiselect=False,
+        #                 value="Original",
+        #                 interactive=True,
+        #             )
+        #             base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+        #
+        #     submit_button = gr.Button("Submit Eval")
+        #     submission_result = gr.Markdown()
+        #     submit_button.click(
+        #         add_new_eval,
+        #         [
+        #             model_name_textbox,
+        #             base_model_name_textbox,
+        #             revision_name_textbox,
+        #             precision,
+        #             weight_type,
+        #             model_type,
+        #         ],
+        #         submission_result,
+        #     )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

eval-results/results_HuggingFaceH4-zephyr-7b-beta.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "HuggingFaceH4/zephyr-7b-beta", "model_sha": "main"}, "results": {"hs": {"delta": -0.039}, "d": {"delta": 0.312}, "hy": {"delta": -0.018}, "pd": {"delta": -0.025}, "pa": {"delta": -0.044}, "pf": {"delta": 0.133}, "sc": {"delta": 0.144}, "ma": {"delta": 0.16}, "si": {"delta": 0.278}, "l": {"delta": -0.016555555}}}

eval-results/results_NousResearch-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "NousResearch/Meta-Llama-3-8B-Instruct", "model_sha": "main"}, "results": {"hs": {"delta": 0.041}, "d": {"delta": 0.322}, "hy": {"delta": 0.01}, "pd": {"delta": 0.012}, "pa": {"delta": -0.061}, "pf": {"delta": 0.211}, "sc": {"delta": 0.231}, "ma": {"delta": 0.091}, "si": {"delta": 0.382}, "l": {"delta": -0.018}}}

eval-results/results_NousResearch-Meta-Llama-3-8B.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "NousResearch/Meta-Llama-3-8B", "model_sha": "main"}, "results": {"hs": {"delta": -0.006}, "d": {"delta": 0.176}, "hy": {"delta": -0.012}, "pd": {"delta": -0.013}, "pa": {"delta": 0.0}, "pf": {"delta": 0.059}, "sc": {"delta": 0.101}, "ma": {"delta": 0.033}, "si": {"delta": 0.231}, "l": {"delta": -0.007333333}}}

eval-results/results_berkeley-nest-Starling-LM-7B-alpha.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "berkeley-nest/Starling-LM-7B-alpha", "model_sha": "main"}, "results": {"hs": {"delta": -0.073}, "d": {"delta": 0.681}, "hy": {"delta": 0.0}, "pd": {"delta": -0.029}, "pa": {"delta": -0.111}, "pf": {"delta": 0.262}, "sc": {"delta": 0.26}, "ma": {"delta": 0.011}, "si": {"delta": 0.5}, "l": {"delta": 0.008333334}}}

eval-results/results_google-gemma-7b-it.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "google/gemma-7b-it", "model_sha": "main"}, "results": {"hs": {"delta": 0.1}, "d": {"delta": 0.475}, "hy": {"delta": 0.105}, "pd": {"delta": 0.021}, "pa": {"delta": 0.09}, "pf": {"delta": 0.079}, "sc": {"delta": 0.21}, "ma": {"delta": 0.048}, "si": {"delta": 0.335}, "l": {"delta": -0.0123333335}}}

eval-results/results_google-gemma-7b.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "google/gemma-7b", "model_sha": "main"}, "results": {"hs": {"delta": -0.033}, "d": {"delta": 0.42}, "hy": {"delta": 0.06}, "pd": {"delta": 0.015}, "pa": {"delta": 0.123}, "pf": {"delta": 0.208}, "sc": {"delta": 0.164}, "ma": {"delta": 0.18}, "si": {"delta": 0.491}, "l": {"delta": -0.04222222}}}

eval-results/results_meta-llama-Llama-2-7b-chat-hf.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "meta-llama/Llama-2-7b-chat-hf", "model_sha": "main"}, "results": {"hs": {"delta": -0.037}, "d": {"delta": 0.059}, "hy": {"delta": -0.035}, "pd": {"delta": -0.014}, "pa": {"delta": 0.022}, "pf": {"delta": 0.122}, "sc": {"delta": 0.181}, "ma": {"delta": 0.19}, "si": {"delta": 0.132}, "l": {"delta": -0.047333334}}}

eval-results/results_meta-llama-Llama-2-7b-hf.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "meta-llama/Llama-2-7b-hf", "model_sha": "main"}, "results": {"hs": {"delta": -0.054}, "d": {"delta": 0.128}, "hy": {"delta": 0.02}, "pd": {"delta": 0.025}, "pa": {"delta": 0.088}, "pf": {"delta": 0.021}, "sc": {"delta": 0.078}, "ma": {"delta": 0.048}, "si": {"delta": 0.2}, "l": {"delta": 0.021000002}}}

eval-results/results_microsoft-Phi-3-mini-128k-instruct.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "microsoft/Phi-3-mini-128k-instruct", "model_sha": "main"}, "results": {"hs": {"delta": 0.043}, "d": {"delta": 0.659}, "hy": {"delta": 0.442}, "pd": {"delta": 0.157}, "pa": {"delta": 0.087}, "pf": {"delta": 0.204}, "sc": {"delta": 0.428}, "ma": {"delta": 0.042}, "si": {"delta": 0.476}, "l": {"delta": 0.018777778}}}

eval-results/results_mistralai-Mistral-7B-Instruct-v0-1.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mistral-7B-Instruct-v0.1", "model_sha": "main"}, "results": {"hs": {"delta": -0.032}, "d": {"delta": 0.442}, "hy": {"delta": -0.025}, "pd": {"delta": 0.02}, "pa": {"delta": -0.065}, "pf": {"delta": 0.198}, "sc": {"delta": 0.325}, "ma": {"delta": 0.039}, "si": {"delta": 0.563}, "l": {"delta": -0.038999997}}}

eval-results/results_mistralai-Mistral-7B-Instruct-v0-2.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mistral-7B-Instruct-v0.2", "model_sha": "main"}, "results": {"hs": {"delta": 0.149}, "d": {"delta": 0.474}, "hy": {"delta": 0.193}, "pd": {"delta": -0.014}, "pa": {"delta": 0.103}, "pf": {"delta": 0.323}, "sc": {"delta": 0.474}, "ma": {"delta": 0.132}, "si": {"delta": 0.405}, "l": {"delta": -0.004666667}}}

eval-results/results_mistralai-Mistral-7B-Instruct-v0-3.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mistral-7B-Instruct-v0.3", "model_sha": "main"}, "results": {"hs": {"delta": 0.113}, "d": {"delta": 0.312}, "hy": {"delta": 0.044}, "pd": {"delta": -0.006}, "pa": {"delta": 0.042}, "pf": {"delta": 0.251}, "sc": {"delta": 0.287}, "ma": {"delta": 0.142}, "si": {"delta": 0.489}, "l": {"delta": -0.025666667}}}

eval-results/results_mistralai-Mistral-7B-v0-3.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mistral-7B-v0.3", "model_sha": "main"}, "results": {"hs": {"delta": -0.073}, "d": {"delta": 0.44}, "hy": {"delta": -0.214}, "pd": {"delta": -0.013}, "pa": {"delta": 0.067}, "pf": {"delta": 0.16}, "sc": {"delta": 0.252}, "ma": {"delta": -0.019}, "si": {"delta": 0.449}, "l": {"delta": -0.011444445}}}

eval-results/results_mistralai-Mixtral-8x7B-Instruct-v0-1.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mixtral-8x7B-Instruct-v0.1", "model_sha": "main"}, "results": {"hs": {"delta": 0.238}, "d": {"delta": 0.539}, "hy": {"delta": 0.242}, "pd": {"delta": 0.114}, "pa": {"delta": 0.242}, "pf": {"delta": 0.49}, "sc": {"delta": 0.395}, "ma": {"delta": 0.076}, "si": {"delta": 0.388}, "l": {"delta": 0.0125555545}}}

eval-results/results_mistralai-Mixtral-8x7B-v0-1.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "mistralai/Mixtral-8x7B-v0.1", "model_sha": "main"}, "results": {"hs": {"delta": 0.014}, "d": {"delta": 0.379}, "hy": {"delta": 0.022}, "pd": {"delta": 0.045}, "pa": {"delta": -0.023}, "pf": {"delta": 0.342}, "sc": {"delta": 0.42}, "ma": {"delta": 0.08}, "si": {"delta": 0.384}, "l": {"delta": -0.06111111}}}

eval-results/results_tiiuae-falcon-7b-instruct.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "tiiuae/falcon-7b-instruct", "model_sha": "main"}, "results": {"hs": {"delta": 0.032}, "d": {"delta": 0.131}, "hy": {"delta": -0.028}, "pd": {"delta": 0.003}, "pa": {"delta": 0.042}, "pf": {"delta": -0.017}, "sc": {"delta": 0.041}, "ma": {"delta": 0.019}, "si": {"delta": 0.145}, "l": {"delta": 0.002888889}}}

eval-results/results_tiiuae-falcon-7b.json

@@ -0,0 +1 @@
+{"config": {"model_dtype": "float16", "model_name": "tiiuae/falcon-7b", "model_sha": "main"}, "results": {"hs": {"delta": -0.135}, "d": {"delta": 0.145}, "hy": {"delta": -0.356}, "pd": {"delta": -0.07}, "pa": {"delta": 0.043}, "pf": {"delta": -0.027}, "sc": {"delta": 0.167}, "ma": {"delta": 0.08}, "si": {"delta": 0.236}, "l": {"delta": -0.063777775}}}

src/display/utils.py

@@ -26,7 +26,7 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Overall ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information

src/envs.py

@@ -4,22 +4,19 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "IrinaAbdullaeva" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/MindShift"
+# RESULTS_REPO = f"{OWNER}/results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -16,20 +16,20 @@ from src.submission.check_validity import is_model_on_hub
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
     org: str 
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
     @classmethod
@@ -76,7 +76,7 @@ class EvalResult:
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
-            mean_acc = np.mean(accs) * 100.0
+            mean_acc = np.sum(accs)
             results[task.benchmark] = mean_acc
 
         return self(
@@ -86,30 +86,14 @@ class EvalResult:
             model=model,
             results=results,
             precision=precision,  
-            revision= config.get("model_sha", ""),
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
         )
 
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        overall = sum([v for v in self.results.values() if v is not None])
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -119,7 +103,7 @@ class EvalResult:
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.average.name: overall,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
@@ -132,29 +116,8 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+
+def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
@@ -163,20 +126,20 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
 
-        # Sort the files by date
+        # Sort the files by date: old -> new
         try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+            files = sorted([os.path.join(root, file) for file in files], key=os.path.getmtime, reverse=False)
+            # files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
         for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
+            model_result_filepaths.append(file)
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
@@ -188,7 +151,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()  # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue

src/populate.py

@@ -8,9 +8,9 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)