update

app.py

@@ -122,11 +122,6 @@ def evaluate(
         samples = "__dummy__.jsonl"
 
     extra = subset + "_" if subset != "full" else ""
-    if os.path.isdir(samples):
-        result_path = os.path.join(samples, f"{extra}eval_results.json")
-    else:
-        assert samples.endswith(".jsonl")
-        result_path = samples.replace(".jsonl", f"_{extra}eval_results.json")
 
     problems = get_bigcodebench(subset=subset)
     dataset_hash = get_bigcodebench_hash(subset=subset)
@@ -139,98 +134,93 @@ def evaluate(
     gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems])
     failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems]
     
-    if os.path.isfile(result_path):
-        with open(result_path, "r") as f:
-            results = json.load(f)
-        results = compatible_eval_result(results)
-    else:
-        if check_gt_only:
-            if gt_pass_rate > 0.99:
-                cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green")
-            else:
-                cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red")
-            if len(failed_tasks) > 0:
-                cprint(f"Failed tasks: {failed_tasks}", "red")
-            return {"gt_pass_rate":float(gt_pass_rate), "failed_tasks": failed_tasks}
-        
-        results = {
-            "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
-            "eval": {},
-        }
+    if check_gt_only:
+        if gt_pass_rate > 0.99:
+            cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green")
+        else:
+            cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red")
+        if len(failed_tasks) > 0:
+            cprint(f"Failed tasks: {failed_tasks}", "red")
+        return {"gt_pass_rate":float(gt_pass_rate), "failed_tasks": failed_tasks}
+    
+    results = {
+        "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
+        "eval": {},
+    }
 
-        with ProcessPoolExecutor(max_workers=n_workers) as executor:
-            futures = []
-            completion_id = Counter()
-            n_samples = 0
-            eval_results = defaultdict(list)  # task_id ->
-            remainings = set()
+    with ProcessPoolExecutor(max_workers=n_workers) as executor:
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        eval_results = defaultdict(list)  # task_id ->
+        remainings = set()
 
-            print("Reading samples...")
-            for sample in tqdm(load_solutions(samples)):
-                task_id = sample["task_id"]
-                
-                if task_id not in problems:
-                    warn(
-                        f"Task {task_id} is found in the samples but not found in the dataset"
-                    )
-                    continue
-                solution = (
-                    sample["solution"]
-                    if "solution" in sample
-                    else problems[task_id]["complete_prompt"] + sample["completion"]
-                )
-                if "sanitized-calibrated" in samples:
-                    solution = problems[task_id]["code_prompt"] + "\n    pass\n" + solution
-                remainings.add(sample["_identifier"])
-                args = (
-                    completion_id[task_id],
-                    problems[task_id],
-                    solution,
-                    max_as_limit,
-                    max_data_limit,
-                    max_stack_limit,
-                    sample["_identifier"],
-                    min_time_limit,
-                    expected_time[task_id] if expected_time[task_id] else 20
+        print("Reading samples...")
+        for sample in tqdm(load_solutions(samples)):
+            task_id = sample["task_id"]
+            
+            if task_id not in problems:
+                warn(
+                    f"Task {task_id} is found in the samples but not found in the dataset"
                 )
-                futures.append(executor.submit(check_correctness, *args))
-                completion_id[task_id] += 1
-                n_samples += 1
-
-            assert n_samples == len(remainings), "Missing problems in unfinished"
-            assert len(completion_id) == len(problems), "Missing problems in samples"
-
-            def stucking_checker():
-                not_done = futures
-                while len(not_done) > 0:
-                    done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED)
+                continue
+            solution = (
+                sample["solution"]
+                if "solution" in sample
+                else problems[task_id]["complete_prompt"] + sample["completion"]
+            )
+            if "sanitized-calibrated" in samples:
+                solution = problems[task_id]["code_prompt"] + "\n    pass\n" + solution
+            remainings.add(sample["_identifier"])
+            args = (
+                completion_id[task_id],
+                problems[task_id],
+                solution,
+                max_as_limit,
+                max_data_limit,
+                max_stack_limit,
+                sample["_identifier"],
+                min_time_limit,
+                expected_time[task_id] if expected_time[task_id] else 20
+            )
+            futures.append(executor.submit(check_correctness, *args))
+            completion_id[task_id] += 1
+            n_samples += 1
 
-                    if len(done) == 0:
-                        warn("No samples have finished testing in the last 240s")
-                        warn(f"{len(remainings)} samples to be tested: {remainings}")
+        assert n_samples == len(remainings), "Missing problems in unfinished"
+        assert len(completion_id) == len(problems), "Missing problems in samples"
 
-            threading.Thread(target=stucking_checker).start()
+        def stucking_checker():
+            not_done = futures
+            while len(not_done) > 0:
+                done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED)
 
-            for future in tqdm(as_completed(futures), total=n_samples):
-                result = future.result()
-                remainings.remove(result["_identifier"])
-                eval_results[result["task_id"]].append(result)
+                if len(done) == 0:
+                    warn("No samples have finished testing in the last 240s")
+                    warn(f"{len(remainings)} samples to be tested: {remainings}")
 
+        threading.Thread(target=stucking_checker).start()
 
-        # sort the results for each problem by completion_id
-        for task_id, task_results in eval_results.items():
-            task_results.sort(key=lambda x: x["completion_id"])
-            results["eval"][task_id] = []
-            for res in task_results:
-                stat, details = res["base"]
-                results["eval"][task_id].append(
-                    {
-                        "task_id": task_id,
-                        "solution": res["solution"],
-                        "status": stat,
-                        "details": details,
-                    }
-                )
+        for future in tqdm(as_completed(futures), total=n_samples):
+            result = future.result()
+            remainings.remove(result["_identifier"])
+            eval_results[result["task_id"]].append(result)
+
+
+    # sort the results for each problem by completion_id
+    for task_id, task_results in eval_results.items():
+        task_results.sort(key=lambda x: x["completion_id"])
+        results["eval"][task_id] = []
+        for res in task_results:
+            stat, details = res["base"]
+            results["eval"][task_id].append(
+                {
+                    "task_id": task_id,
+                    "solution": res["solution"],
+                    "status": stat,
+                    "details": details,
+                }
+            )
 
     # Calculate pass@k.
     total = np.array([len(r) for k, r in results["eval"].items() if k in problems])
@@ -245,79 +235,18 @@ def evaluate(
     base_correct = np.array(base_correct)
 
     pass_at_k = {
-        f"pass@{k}": float(estimate_pass_at_k(total, base_correct, k).mean())
+        f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
         for k in pass_k
         if total.min() >= k
     }
-    pass_at_k["gt_pass_rate"] = float(gt_pass_rate)
+    pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0]
+    pass_at_k["split"] = split
+    pass_at_k["subset"] = subset
+    pass_at_k["calibrated"] = "sanitized-calibrated" in samples
+    pass_at_k["gt_pass_rate"] = gt_pass_rate
     pass_at_k["failed_tasks"] = failed_tasks
-    return pass_at_k
-
-    # mode = "-calibrated" if "sanitized-calibrated" in samples else ""
-    # extra = subset.capitalize()
-    # split = split.capitalize()
-    # cprint(f"BigCodeBench-{split}{mode} ({extra})", "green")
-        
-    # if no_gt:
-    #     cprint(f"Groundtruth is not checked", "yellow")
-    # else:
-    #     if gt_pass_rate > 0.99:
-    #         cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green")
-    #     else:
-    #         cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red")
-        
-    #     if len(failed_tasks) > 0:
-    #         cprint(f"Failed tasks: {failed_tasks}", "red")
-    
-    # for k, v in pass_at_k.items():
-    #     cprint(f"{k}:\t{v:.3f}", "green")
-
-    # # save results
-    # if os.path.isfile(result_path):
-    #     decision = ""
-    #     while decision.lower() not in ["y", "n"]:
-    #         print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...")
-    #         decision = input()
-
-    #     if decision.lower() == "y":
-    #         # mv the file to a backup
-    #         new_path = result_path + ".bak"
-    #         while os.path.isfile(new_path):
-    #             new_path += ".bak"
-    #         os.rename(result_path, new_path)
-    #         print(f"Backup {result_path} to {new_path}")
+    return results, pass_at_k
 
-    # if not os.path.isfile(result_path):
-    #     with open(result_path, "w") as f:
-    #         json.dump(results, f, indent=2)
-
-    # if save_pass_rate:
-    #     pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json")
-    #     pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0]
-    #     pass_at_k["calibrated"] = "sanitized-calibrated" in samples
-    #     pass_at_k["subset"] = subset
-
-    #     def save_pass_at_k():
-    #         with open(pass_at_k_path, "w") as f:
-    #             json.dump(pass_at_k, f, indent=2)
-
-    #     if os.path.isfile(pass_at_k_path):
-    #         saved_pass_at_k = json.load(open(pass_at_k_path, "r"))
-    #         # compare saved_pass_at_k with pass_at_k
-    #         for k in saved_pass_at_k.keys():
-    #             if pass_at_k[k] != saved_pass_at_k[k]:
-    #                 cprint(f"Warning: {k} is different from the saved one", "yellow")
-                    
-    #         # ask user whether to save the pass@k
-    #         decision = ""
-    #         while decision.lower() not in ["y", "n"]:
-    #             print(f"Save pass@k to {pass_at_k_path}? [Y/N]")
-    #             decision = input()
-    #         if decision.lower() == "y":
-    #             save_pass_at_k()
-                
-    #     else:
-    #         save_pass_at_k()
 
 def run_gradio():
     interface = gr.Interface(
@@ -335,7 +264,10 @@ def run_gradio():
             gr.Checkbox(label="Check GT Only"),
             gr.Checkbox(label="No GT"),
         ],
-        outputs="text",
+        outputs=[
+            gr.JSON(label="Results"),
+            gr.JSON(label="Eval Results"),
+        ],
         # concurrency_limit=None
     )
     interface.queue(default_concurrency_limit=None)

src/display/about.py

@@ -1,148 +0,0 @@
-TITLE = """<div style="text-align: center;"><h1> 🌸<span style='color: #C867B5;'>BigCodeBench</span> Leaderboard</h1></div>\
-            <br>\
-            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">⭐ Big Code Models Leaderboard</a>, we compare performance of LLMs on <a href="https://huggingface.co/datasets/bigcode/bigcodebench">BigCodeBench</a> benchmark.</p>
-            <p>To get started, please check out <a href="https://github.com/bigcode-project/bigcodebench">our GitHub repository</a>.
-            <br>\
-            For more details, please check our <a href="https://huggingface.co/blog/terryyz/bigcodebench-hard">blog on the Hard Set</a>, <a href="https://huggingface.co/blog/leaderboard-bigcodebench">blog on the Full Set</a> and <a href="https://arxiv.org/abs/2406.15877">paper</a>.</p>
-            """
-
-ABOUT_TEXT = """# Context
-We believe that there are three main expectations of a good execution-based programming benchmark:
-1. The benchmark should be easy to use and efficient in evaluating the fundamental capabilities of LLMs. Repo-level and agent-centric benchmarks (e.g., SWE-bench) are not suitable for this purpose.
-2. The benchmark should be practical, covering various programming scenarios. Algo-specific benchmarks (e.g., HumanEval and MBPP) are unsuitable. Domain-specific benchmarks (e.g., DS-1000) are also unsuitable for this purpose.
-3. The benchmark should be challenging, where the tasks require LLMs' strong compositional reasoning capabilities and instruction-following capabilities. The benchmarks with simple tasks (e.g., ODEX) are unsuitable.
-
-BigCodeBench is the first benchmark that meets all three expectations. It is an <u>*__easy-to-use__*</u> benchmark that evaluates LLMs with <u>*__practical__*</u> and <u>*__challenging__*</u> programming tasks, accompanied by an end-to-end evaluation framework [`bigcodebench`](https://github.com/bigcode-project/bigcodebench). We aim to assess how well LLMs can solve programming tasks in an open-ended setting, with the following two focuses:
-
-- Diverse Function Calls: This design requires LLMs to utilize diverse function calls.
-- Complex Instructions: This design requires LLMs to follow complex instructions.
-
-
-### Benchamrks & Prompts
-The dataset has 2 variants: 
-1. `BigCodeBench-Complete`: _Code Completion based on the structured long-context docstrings_.
-1. `BigCodeBench-Instruct`: _Code Generation based on the NL-oriented instructions_.
-
-Figure below shows the example of `Complete` vs `Instruct` prompt. For `Instruct`, we only focus on instruction-tuned LLMs.
-
-<img src="https://github.com/bigcode-bench/bigcode-bench.github.io/blob/main/asset/bigcodebench_prompt.svg?raw=true" alt="OctoCoder vs Base HumanEval prompt" width="800px">
-
-The specific prompt template can be found [here](https://github.com/bigcode-project/bigcodebench/blob/main/bigcodebench/model.py).
-
-There are some edge cases:
-- Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.
-- We have not included the `Instruct` results of Granite-Code-Instruct 8B & 3B as they constantly have empty outputs.
-
-### Evaluation Parameters
-- All models were evaluated with the [bigcodebench](https://github.com/bigcode-project/bigcodebench). You can install the [PyPI package](https://pypi.org/project/bigcodebench/).
-To get started, please first set up the environment:
-
-```bash
-# Install to use bigcodebench.evaluate
-pip install bigcodebench --upgrade
-# If you want to use the evaluate locally, you need to install the requirements
-pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt
-
-# Install to use bigcodebench.generate
-# You are strongly recommended to install the generate dependencies in a separate environment
-pip install bigcodebench[generate] --upgrade
-```
-
-### Scoring and Rankings
-- Models are ranked according to Pass@1 using greedy decoding. Setup details can be found <a href="https://github.com/bigcode-project/bigcodebench/blob/main/bigcodebench/generate.py">here</a>.
-- The code to compute Elo rating is [here](https://github.com/bigcode-project/bigcodebench/blob/main/analysis/get_results.py), which is based on [Chatbot Arena Notebook](https://colab.research.google.com/drive/1RAWb22-PFNI-X1gPVzc927SGUdfr6nsR#scrollTo=JdiJbB6pZB1B&line=2&uniqifier=1). We only compute the Elo rating for the `BigCodeBench-Complete` variant.
-
-### Contact
-If you have any questions, feel free to reach out to us at [terry.zhuo@monash.edu](mailto:terry.zhuo@monash.edu) or [contact@bigcode-project.org](mailto:contact@bigcode-project.org)
-
-### Citation Information
-
-```bibtex
-@article{zhuo2024bigcodebench,
-    title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, 
-    author={Terry Yue Zhuo and Minh Chien Vu and Jenny Chim and Han Hu and Wenhao Yu and Ratnadira Widyasari and Imam Nur Bani Yusuf and Haolan Zhan and Junda He and Indraneil Paul and Simon Brunner and Chen Gong and Thong Hoang and Armel Randy Zebaze and Xiaoheng Hong and Wen-Ding Li and Jean Kaddour and Ming Xu and Zhihan Zhang and Prateek Yadav and Naman Jain and Alex Gu and Zhoujun Cheng and Jiawei Liu and Qian Liu and Zijian Wang and David Lo and Binyuan Hui and Niklas Muennighoff and Daniel Fried and Xiaoning Du and Harm de Vries and Leandro Von Werra},
-    journal={arXiv preprint arXiv:2406.15877},
-    year={2024}
-}
-```
-"""
-
-SUBMISSION_TEXT = """
-<h1 align="center">
-How to submit models/results to the leaderboard?
-</h1>
-We welcome the community to submit evaluation results of new models. We also provide an experimental feature for submitting models that our team will evaluate on the 🤗 cluster.
-
-## Submitting Models (experimental feature)
-Inspired from the Open LLM Leaderboard, we welcome code models submission from the community that will be automatically evaluated. Please note that this is still an experimental feature.
-Below are some guidlines to follow before submitting your model:
-
-#### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet.
-#### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-#### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-#### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card.
-"""
-
-SUBMISSION_TEXT_2 = """
-## Sumbitting Results
-You also have the option for running evaluation yourself and submitting results. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
-
-### 1 - Running Evaluation
-
-We wrote a detailed guide for running the evaluation on your model. You can find the it in [bigcode-evaluation-harness/leaderboard](https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main/leaderboard). This will generate a json file summarizing the results, in addition to the raw generations and metric files.
-
-### 2- Submitting Results 🚀
-
-To submit your results create a **Pull Request** in the community tab to add them under the [folder](https://huggingface.co/spaces/bigcode/bigcodebench-code-evals/tree/main/community_results) `community_results` in this repository:
-- Create a folder called `ORG_MODELNAME_USERNAME` for example `bigcode_my_model_terry`
-- Put your json file with grouped scores from the guide, in addition generations folder and metrics folder in it.
-
-The title of the PR should be `[Community Submission] Model: org/model, Username: your_username`, replace org and model with those corresponding to the model you evaluated.
-"""
-
-SUBMISSION_TEXT_3 = """
-<h1 align="center">
-How to submit models/results to the leaderboard?
-</h1>
-We welcome the community to submit evaluation results of new models. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
-
-### 1 - Running Evaluation
-
-We wrote a detailed guide for running the evaluation on your model. You can find the it in [bigcode-evaluation-harness/leaderboard](https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main/leaderboard). This will generate a json file summarizing the results, in addition to the raw generations and metric files.
-
-### 2- Submitting Results 🚀
-
-To submit your results create a **Pull Request** in the community tab to add them under the [folder](https://huggingface.co/spaces/bigcode/multilingual-code-evals/tree/main/community_results) `community_results` in this repository:
-- Create a folder called `ORG_MODELNAME_USERNAME` for example `bigcode_starcoder_loubnabnl`
-- Put your json file with grouped scores from the guide, in addition generations folder and metrics folder in it.
-
-The title of the PR should be `[Community Submission] Model: org/model, Username: your_username`, replace org and model with those corresponding to the model you evaluated.
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-
-CITATION_BUTTON_TEXT = r"""
-@article{zhuo2024bigcodebench,
-  title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions},
-  author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others},
-  journal={arXiv preprint arXiv:2406.15877},
-  year={2024}
-}
-"""
-
-SUBMISSION_TEXT_3="""
-## We welcome the community to request for new models to be added to the leaderboard.
-## Please [file an issue](https://github.com/bigcode-project/bigcodebench/issues/new/choose) to add the model to the leaderboard or [start a discussion](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard/discussions/new) in the community🤗
-"""

src/display/css_html_js.py

@@ -1,115 +0,0 @@
-custom_css = """
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-/* Full width space */
-.gradio-container {
-    max-width: 95% !important;
-}
-
-/* Text style and margins */
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
-
-#search-bar {
-    padding: 0px;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-/* Filters style */
-#filter_type {
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span {
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap {
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner {
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input {
-    width: 1px;
-}
-#filter-columns-type {
-    border: 0;
-    padding: 0.5;
-}
-#filter-columns-size {
-    border: 0;
-    padding: 0.5;
-}
-#box-filter > .form {
-    border: 0;
-}
-
-/* Header styles */
-#header-title {
-    text-align: left;
-    display: inline-block;
-}
-
-#header-row {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-}
-
-#header-row .gradio-html {
-    flex-grow: 1;
-}
-
-#oauth-button {
-    height: auto;
-    min-width: max-content;
-    white-space: nowrap;
-    padding: 10px 20px;
-    border-radius: 4px;
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,37 +0,0 @@
-from huggingface_hub import HfApi
-
-API = HfApi()
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(df, model_col, link_col):
-    df[model_col] = df.apply(
-        lambda row: model_hyperlink(row[link_col], row[model_col]), axis=1
-    )
-    df["Openness"] = df.apply(
-        lambda row: "Open" if "huggingface.co" in row[link_col] else "Closed", axis=1
-    )
-    return df
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,142 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-import json
-import logging
-from datetime import datetime
-import pandas as pd
-
-
-# Configure logging
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-
-# Convert ISO 8601 dates to datetime objects for comparison
-def parse_iso8601_datetime(date_str):
-    if date_str.endswith('Z'):
-        date_str = date_str[:-1] + '+00:00'
-    return datetime.fromisoformat(date_str)
-
-def parse_datetime(datetime_str):
-    formats = [
-        "%Y-%m-%dT%H-%M-%S.%f",  # Format with dashes
-        "%Y-%m-%dT%H:%M:%S.%f",  # Standard format with colons
-        "%Y-%m-%dT%H %M %S.%f",  # Spaces as separator
-    ]
-
-    for fmt in formats:
-        try:
-            return datetime.strptime(datetime_str, fmt)
-        except ValueError:
-            continue
-    # in rare cases set unix start time for files with incorrect time (legacy files)
-    logging.error(f"No valid date format found for: {datetime_str}")
-    return datetime(1970, 1, 1)
-
-
-def load_json_data(file_path):
-    """Safely load JSON data from a file."""
-    try:
-        with open(file_path, "r") as file:
-            return json.load(file)
-    except json.JSONDecodeError:
-        print(f"Error reading JSON from {file_path}")
-        return None  # Or raise an exception
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-column_map = {
-    "T": "T",
-    "model": "Model",
-    "type": "Model Type",
-    "size_range": "Size Range",
-    "complete": "Complete",
-    "instruct": "Instruct",
-    "average": "Average",
-    "elo_mle": "Elo Rating",
-    "link": "Link",
-    "act_param": "#Act Params (B)",
-    "size": "#Params (B)",
-    "moe": "MoE",
-    # "lazy": "Lazy",
-    "openness": "Openness",
-    # "direct_complete": "Direct Completion",
-}
-
-type_map = {
-    "🔶": "🔶 Chat Models (RLHF, DPO, IFT, ...)",
-    "🟢": "🟢 Base Models"
-}
-
-moe_map = {
-    True: "MoE",
-    False: "Dense"
-}
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass(frozen=True)
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-    dummy: bool = False
-
-
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["T", ColumnContent, ColumnContent(column_map["T"], "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent(column_map["model"], "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["type", ColumnContent, ColumnContent(column_map["type"], "str", False, True)])
-auto_eval_column_dict.append(["size_range", ColumnContent, ColumnContent(column_map["size_range"], "str", False, True)])
-# Scores
-auto_eval_column_dict.append(["complete", ColumnContent, ColumnContent(column_map["complete"], "number", True)])
-auto_eval_column_dict.append(["instruct", ColumnContent, ColumnContent(column_map["instruct"], "number", True)])
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(column_map["average"], "number", True)])
-auto_eval_column_dict.append(["elo_mle", ColumnContent, ColumnContent(column_map["elo_mle"], "number", True)])
-
-# Model information
-auto_eval_column_dict.append(["act_param", ColumnContent, ColumnContent(column_map["act_param"], "number", True)])
-auto_eval_column_dict.append(["link", ColumnContent, ColumnContent(column_map["link"], "str", False, True)])
-auto_eval_column_dict.append(["size", ColumnContent, ColumnContent(column_map["size"], "number", False)])
-# auto_eval_column_dict.append(["lazy", ColumnContent, ColumnContent(column_map["lazy"], "bool", False, True)])
-auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent(column_map["moe"], "str", False, True)])
-auto_eval_column_dict.append(["openness", ColumnContent, ColumnContent(column_map["openness"], "str", False, True)])
-# auto_eval_column_dict.append(["direct_complete", ColumnContent, ColumnContent(column_map["direct_complete"], "bool", False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model_link = ColumnContent("link", "markdown", True)
-    model_name = ColumnContent("model", "str", True)
-
-@dataclass
-class ModelDetails:
-    name: str
-    symbol: str = ""  # emoji, only for the model type
-
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn)]
-TYPES = [c.type for c in fields(AutoEvalColumn)]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-
-NUMERIC_INTERVALS = {
-    "?": pd.Interval(-1, 0, closed="right"),
-    "~1.5": pd.Interval(0, 2, closed="right"),
-    "~3": pd.Interval(2, 4, closed="right"),
-    "~7": pd.Interval(4, 9, closed="right"),
-    "~13": pd.Interval(9, 20, closed="right"),
-    "~35": pd.Interval(20, 45, closed="right"),
-    "~60": pd.Interval(45, 70, closed="right"),
-    "70+": pd.Interval(70, 10000, closed="right"),
-}

src/envs.py

@@ -1,42 +0,0 @@
-import os
-from huggingface_hub import HfApi
-
-# clone / pull the lmeh eval data
-HF_TOKEN = os.environ.get("HF_TOKEN", None)
-
-DATA_VERSION = "v0.1.0_hf"
-
-REPO_ID = "bigcode/bigcodebench-evaluator"
-QUEUE_REPO = "bigcode/bigcodebench-requests"
-DATA_REPO = "bigcode/bigcodebench"
-RESULT_REPO = "bigcode/bigcodebench-results"
-HARD_RESULT_REPO = "bigcode/bigcodebench-hard-results"
-
-ELO_REPO = "bigcode/bigcodebench-elo"
-HARD_ELO_REPO = "bigcode/bigcodebench-hard-elo"
-SOLVE_REPO = "bigcode/bigcodebench-solve-rate"
-HARD_SOLVE_REPO = "bigcode/bigcodebench-hard-solve-rate"
-
-VOTES_REPO = "bigcode/bigcodebench-votes"
-
-HF_HOME = os.getenv("HF_HOME", ".")
-
-# Check HF_HOME write access
-print(f"Initial HF_HOME set to: {HF_HOME}")
-
-if not os.access(HF_HOME, os.W_OK):
-    print(f"No write access to HF_HOME: {HF_HOME}. Resetting to current directory.")
-    HF_HOME = "."
-    os.environ["HF_HOME"] = HF_HOME
-else:
-    print("Write access confirmed for HF_HOME")
-
-VOTES_PATH = os.path.join(HF_HOME, "model-votes")
-EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue")
-
-# Rate limit variables
-RATE_LIMIT_PERIOD = 7
-RATE_LIMIT_QUOTA = 5
-HAS_HIGHER_RATE_LIMIT = []
-
-API = HfApi(token=HF_TOKEN)

src/execute.py

@@ -1,194 +0,0 @@
-import gradio as gr
-import subprocess
-import sys
-import os
-import threading
-import time
-import uuid
-import glob
-import shutil
-from pathlib import Path
-
-default_command = "bigcodebench.evaluate"
-is_running = False
-
-def generate_command(
-    jsonl_file, split, subset, parallel,
-    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
-    check_gt_only, no_gt
-):
-    command = [default_command]
-    
-    if jsonl_file is not None:
-        # Copy the uploaded file to the current directory
-        local_filename = os.path.basename(jsonl_file.name)
-        shutil.copy(jsonl_file.name, local_filename)
-        command.extend(["--samples", local_filename])
-    
-    command.extend(["--split", split, "--subset", subset])
-    
-    if parallel is not None and parallel != 0:
-        command.extend(["--parallel", str(int(parallel))])
-    
-    command.extend([
-        "--min-time-limit", str(min_time_limit),
-        "--max-as-limit", str(int(max_as_limit)),
-        "--max-data-limit", str(int(max_data_limit)),
-        "--max-stack-limit", str(int(max_stack_limit))
-    ])
-    
-    if check_gt_only:
-        command.append("--check-gt-only")
-    
-    if no_gt:
-        command.append("--no-gt")
-    
-    return " ".join(command)
-
-
-def cleanup_previous_files(jsonl_file):
-    if jsonl_file is not None:
-        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
-    else:
-        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
-    for file in glob.glob("*"):
-        try:
-            if file not in file_list:
-                os.remove(file)
-        except Exception as e:
-            print(f"Error during cleanup of {file}: {e}")
-
-def find_result_file():
-    json_files = glob.glob("*.json")
-    if json_files:
-        return max(json_files, key=os.path.getmtime)
-    return None
-
-def run_bigcodebench(command):
-    global is_running
-    if is_running:
-        yield "A command is already running. Please wait for it to finish.\n"
-        return
-    is_running = True
-
-    try:
-        yield f"Executing command: {command}\n"
-        
-        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
-        
-        def kill_process():
-            if process.poll() is None:  # If the process is still running
-                process.terminate()
-                is_running = False
-                yield "Process terminated after 12 minutes timeout.\n"
-
-        # Start a timer to kill the process after 12 minutes
-        timer = threading.Timer(720, kill_process)
-        timer.start()
-        
-        for line in process.stdout:
-            yield line
-        
-        # process.wait()
-        
-        timer.cancel()
-        
-        if process.returncode != 0:
-            yield f"Error: Command exited with status {process.returncode}\n"
-        
-        yield "Evaluation completed.\n"
-        
-        result_file = find_result_file()
-        if result_file:
-            yield f"Result file found: {result_file}\n"
-        else:
-            yield "No result file found.\n"
-    finally:
-        is_running = False
-
-def stream_logs(command, jsonl_file=None):
-    global is_running
-        
-    if is_running:
-        yield "A command is already running. Please wait for it to finish.\n"
-        return
-    
-    cleanup_previous_files(jsonl_file)
-    yield "Cleaned up previous files.\n"
-
-    log_content = []
-    for log_line in run_bigcodebench(command):
-        log_content.append(log_line)
-        yield "".join(log_content)
-        
-# with gr.Blocks() as demo:
-#     gr.Markdown("# BigCodeBench Evaluator")
-    
-#     with gr.Row():
-#         jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
-#         split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
-#         subset = gr.Dropdown(choices=["hard", "full"], label="Subset", value="hard")
-    
-#     with gr.Row():
-#         parallel = gr.Number(label="Parallel (optional)", precision=0)
-#         min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
-#         max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
-    
-#     with gr.Row():
-#         max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
-#         max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
-#         check_gt_only = gr.Checkbox(label="Check GT Only")
-#         no_gt = gr.Checkbox(label="No GT")
-    
-#     command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
-#     with gr.Row():
-#         submit_btn = gr.Button("Run Evaluation")
-#         download_btn = gr.DownloadButton(label="Download Result")
-#     log_output = gr.Textbox(label="Execution Logs", lines=20)
-    
-#     input_components = [
-#         jsonl_file, split, subset, parallel,
-#         min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
-#         check_gt_only, no_gt
-#     ]
-    
-#     for component in input_components:
-#         component.change(generate_command, inputs=input_components, outputs=command_output)
-        
-    
-#     def start_evaluation(command, jsonl_file, subset, split):
-#         extra = subset + "_" if subset != "full" else ""
-#         if jsonl_file is not None:
-#             result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
-#         else:
-#             result_path = None
-
-#         for log in stream_logs(command, jsonl_file):
-#             if jsonl_file is not None:
-#                 yield log, gr.update(value=result_path, label=result_path), gr.update()
-#             else:
-#                 yield log, gr.update(), gr.update()
-#         result_file = find_result_file()
-#         if result_file:
-#             return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
-#                     # gr.Button(visible=False)#,
-#                     # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
-#         else:
-#             return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
-#                     # gr.Button("Run Evaluation", visible=True),
-#                     # gr.DownloadButton(visible=False))
-#     submit_btn.click(start_evaluation,
-#                  inputs=[command_output, jsonl_file, subset, split],
-#                  outputs=[log_output, download_btn])
-
-# REPO_ID = "bigcode/bigcodebench-evaluator"
-# HF_TOKEN = os.environ.get("HF_TOKEN", None)
-# API = HfApi(token=HF_TOKEN)
-
-# def restart_space():
-#     API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
-
-# demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
-# scheduler = BackgroundScheduler()
-# scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
-# scheduler.start()

src/populate.py

@@ -1,50 +0,0 @@
-import pathlib
-import pandas as pd
-from datasets import Dataset
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.display.utils import load_json_data, column_map, type_map, moe_map, NUMERIC_INTERVALS
-
-
-
-def get_evaluation_queue_df(save_path, cols):
-    """Generate dataframes for pending, running, and finished evaluation entries."""
-    save_path = pathlib.Path(save_path)
-    all_evals = []
-
-    for path in save_path.rglob("*.json"):
-        data = load_json_data(path)
-    # Organizing data by status
-    status_map = {
-        "PENDING": ["PENDING", "RERUN"],
-        "RUNNING": ["RUNNING"],
-        "FINISHED": ["FINISHED", "PENDING_NEW_EVAL"],
-    }
-    status_dfs = {status: [] for status in status_map}
-    for eval_data in all_evals:
-        for status, extra_statuses in status_map.items():
-            if eval_data["status"] in extra_statuses:
-                status_dfs[status].append(eval_data)
-
-    return tuple(pd.DataFrame(status_dfs[status], columns=cols) for status in ["FINISHED", "RUNNING", "PENDING"])
-
-
-def get_leaderboard_df(leaderboard_dataset: Dataset, cols: list):
-    """Retrieve and process leaderboard data."""
-    all_data_json = leaderboard_dataset.to_dict()
-    num_items = leaderboard_dataset.num_rows
-    all_data_json_list = [{k: all_data_json[k][ix] for k in all_data_json.keys()} for ix in range(num_items)]
-
-    df = pd.DataFrame.from_records(all_data_json_list)
-    # replace df.moe true to false, false to true
-    # map column names
-    df = df.rename(columns=column_map)
-    df[AutoEvalColumn.moe.name] = df[AutoEvalColumn.moe.name].map(moe_map)
-    df[AutoEvalColumn.T.name] = df[AutoEvalColumn.type.name]
-    df[AutoEvalColumn.type.name] = df[AutoEvalColumn.type.name].map(type_map)
-    df[AutoEvalColumn.average.name] = df.apply(lambda x: round((x[AutoEvalColumn.complete.name] + x[AutoEvalColumn.instruct.name]) / 2, 1) if not pd.isna(x[AutoEvalColumn.complete.name]) and not pd.isna(x[AutoEvalColumn.instruct.name]) else None, axis=1)
-    df[AutoEvalColumn.size_range.name] = df[AutoEvalColumn.size.name].apply(lambda x: next((k for k, v in NUMERIC_INTERVALS.items() if x in v), "?"))
-    df = make_clickable_model(df, AutoEvalColumn.model.name, AutoEvalColumn.link.name)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-    return df

src/tools/plots.py

@@ -1,72 +0,0 @@
-import plotly.graph_objects as go
-import plotly.express as px
-import numpy as np
-
-
-def plot_elo_mle(df):
-    fig = px.scatter(df, x="model", y="rating", error_y="error_y",
-                     error_y_minus="error_y_minus",
-                    #  title="Bootstrap of Elo MLE Estimates (BigCodeBench-Complete)"
-                     )
-    fig.update_layout(xaxis_title="Model", 
-                      yaxis_title="Rating",
-                      autosize=True,
-                    #   width=1300,
-                    #   height=900,
-                      )
-    return fig
-
-
-def plot_solve_rate(df, task, rows=30, cols=38):
-    keys = df["task_id"]
-    values = df["solve_rate"]
-    
-    values = np.array(values, dtype=float)  # Ensure values are floats
-
-    # Extract numerical IDs and sort by them
-    ids = [int(key.split('/')[-1]) for key in keys]
-    sorted_indices = np.argsort(ids)
-    keys = np.array(keys)[sorted_indices]
-    values = values[sorted_indices]
-    
-    n = len(values)
-    pad_width = rows * cols - n
-    
-    # Create a masked array
-    masked_values = np.ma.array(np.full(rows * cols, np.nan), mask=True)
-    masked_values[:n] = values
-    masked_values.mask[:n] = False
-    masked_values = masked_values.reshape((rows, cols))
-
-    keys_padded = np.pad(keys, (0, pad_width), 'constant', constant_values='')
-    keys_reshaped = keys_padded.reshape((rows, cols))
-
-    hover_text = np.empty_like(masked_values, dtype=object)
-    for i in range(rows):
-        for j in range(cols):
-            if not masked_values.mask[i, j]:
-                hover_text[i, j] = f"{keys_reshaped[i, j]}<br>Solve Rate: {masked_values[i, j]:.2f}"
-            else:
-                hover_text[i, j] = "NaN"
-
-    upper_solve_rate = round(np.count_nonzero(values) / n * 100, 2)
-    
-    fig = go.Figure(data=go.Heatmap(
-        z=masked_values,
-        text=hover_text,
-        hoverinfo='text',
-        colorscale='teal',
-        zmin=0,
-        zmax=100
-    ))
-
-    fig.update_layout(
-        title=f'BigCodeBench-{task}<br><i>Lowest Upper Limit: {upper_solve_rate}%</i>',
-        xaxis_nticks=cols,
-        yaxis_nticks=rows,
-        xaxis=dict(showticklabels=False),
-        yaxis=dict(showticklabels=False),
-        autosize=True,
-    )
-    
-    return fig

src/voting/vote_system.py

@@ -1,150 +0,0 @@
-import json
-import logging
-import pathlib
-import pandas as pd
-import gradio as gr
-import schedule
-import time
-from datetime import datetime, timezone
-
-from src.envs import API
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-class VoteManager:
-    def __init__(self, votes_path, eval_requests_path, repo_id):
-        self.votes_path = votes_path
-        self.eval_requests_path = eval_requests_path
-        self.repo_id = repo_id
-        self.vote_dataset = self.read_vote_dataset()
-        self.vote_check_set = self.make_check_set(self.vote_dataset)
-        self.votes_to_upload = []
-
-    def init_vote_dataset(self):
-        self.vote_dataset = self.read_vote_dataset()
-        self.vote_check_set = self.make_check_set(self.vote_dataset)
-
-    def read_vote_dataset(self):
-        result = []
-        votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
-        if votes_file.exists():
-            with open(votes_file, "r") as f:
-                for line in f:
-                    data = json.loads(line.strip())
-                    result.append(data)
-        result = pd.DataFrame(result)
-        return result
-
-    def make_check_set(self, vote_dataset: pd.DataFrame):
-        result = list()
-        for row in vote_dataset.itertuples(index=False, name='vote'):
-            result.append((row.model, row.revision, row.username))
-        return set(result)
-    
-    def get_model_revision(self, selected_model: str) -> str:
-        """Fetch the revision for the given model from the request files."""
-        for user_folder in pathlib.Path(self.eval_requests_path).iterdir():
-            if user_folder.is_dir():
-                for file in user_folder.glob("*.json"):
-                    with open(file, "r") as f:
-                        data = json.load(f)
-                        if data.get("model") == selected_model:
-                            return data.get("revision", "main")
-        return "main"
-
-    def create_request_vote_df(self, pending_models_df: gr.Dataframe):
-        if pending_models_df.empty or not "model_name" in pending_models_df.columns:
-            return pending_models_df
-        self.vote_dataset = self.read_vote_dataset()
-        vote_counts = self.vote_dataset.groupby(['model', 'revision']).size().reset_index(name='vote_count')
-
-        pending_models_df_votes = pd.merge(
-            pending_models_df, 
-            vote_counts, 
-            left_on=["model_name", 'revision'], 
-            right_on=['model', 'revision'], 
-            how='left'
-        )
-        # Filling empty votes
-        pending_models_df_votes['vote_count'] = pending_models_df_votes['vote_count'].fillna(0)
-        pending_models_df_votes = pending_models_df_votes.sort_values(by=["vote_count", "model_name"], ascending=[False, True])
-        # Removing useless columns
-        pending_models_df_votes = pending_models_df_votes.drop(["model_name", "model"], axis=1)
-        return pending_models_df_votes
-
-    # Function to be called when a user votes for a model
-    def add_vote(
-            self,
-            selected_model: str,
-            pending_models_df: gr.Dataframe,
-            profile: gr.OAuthProfile | None
-        ):
-        logger.debug(f"Type of list before usage: {type(list)}")
-        # model_name, revision, user_id, timestamp
-        if selected_model in ["str", ""]:
-            gr.Warning("No model selected")
-            return
-        
-        if profile is None:
-            gr.Warning("Hub Login required")
-            return
-
-        vote_username = profile.username
-        model_revision = self.get_model_revision(selected_model)
-        
-        # tuple (immutable) for checking than already voted for model
-        check_tuple = (selected_model, model_revision, vote_username)
-        if check_tuple in self.vote_check_set:
-            gr.Warning("Already voted for this model")
-            return
-        
-        current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-        vote_obj = {
-            "model": selected_model,
-            "revision": model_revision,
-            "username": vote_username,
-            "timestamp": current_time
-        }
-
-        # Append the vote to the JSONL file
-        try:
-            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
-            with open(votes_file, "a") as f:
-                f.write(json.dumps(vote_obj) + "\n")
-            logger.info(f"Vote added locally: {vote_obj}")
-
-            self.votes_to_upload.append(vote_obj)
-        except Exception as e:
-            logger.error(f"Failed to write vote to file: {e}")
-            gr.Warning("Failed to record vote. Please try again")
-            return
-        
-        self.vote_check_set.add(check_tuple)
-        gr.Info(f"Voted for {selected_model}")
-
-        return self.create_request_vote_df(pending_models_df)
-
-    def upload_votes(self):
-        if self.votes_to_upload:
-            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
-            try:
-                with open(votes_file, "rb") as f:
-                    API.upload_file(
-                        path_or_fileobj=f,
-                        path_in_repo="votes_data.jsonl",
-                        repo_id=self.repo_id,
-                        repo_type="dataset",
-                        commit_message="Updating votes_data.jsonl with new votes",
-                    )
-                logger.info("Votes uploaded to votes repository")
-                self.votes_to_upload.clear()
-            except Exception as e:
-                logger.error(f"Failed to upload votes to repository: {e}")
-
-def run_scheduler(vote_manager):
-    while True:
-        schedule.run_pending()
-        time.sleep(1)