Update app.py

app.py

@@ -1,647 +1,346 @@
+import gradio as gr
+import json
+import multiprocessing
 import os
-import logging
+import pickle
+import threading
 import time
-import datetime
-import gradio as gr
-from threading import Thread
-import datasets
-from huggingface_hub import snapshot_download, WebhooksServer, WebhookPayload, RepoCard
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-from apscheduler.schedulers.background import BackgroundScheduler
-
-# Start ephemeral Spaces on PRs (see config in README.md)
-from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
-
-from src.display.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    # INTRODUCTION_TEXT,
-    TITLE,
-    ABOUT_TEXT,
-    SUBMISSION_TEXT_3,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    fields,
-    EvalQueueColumn
-)
-from src.envs import (
-    API,
-    EVAL_REQUESTS_PATH,
-    RESULT_REPO,
-    DATA_VERSION,
-    DATA_REPO,
-    HARD_RESULT_REPO,
-    ELO_REPO,
-    HARD_ELO_REPO,
-    SOLVE_REPO,
-    HARD_SOLVE_REPO,
-    HF_TOKEN,
-    QUEUE_REPO,
-    REPO_ID,
-    VOTES_REPO,
-    VOTES_PATH,
-    HF_HOME,
-)
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.execute import generate_command, is_running, default_command, stream_logs, find_result_file
-from src.tools.plots import plot_elo_mle, plot_solve_rate
-# from src.voting.vote_system import VoteManager, run_scheduler
-
-# Configure logging
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-
-# Start ephemeral Spaces on PRs (see config in README.md)
-from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
-
-# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
-# This controls whether a full initialization should be performed.
-DO_FULL_INIT = True # os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
-NEW_DATA_ON_LEADERBOARD = True
-LEADERBOARD_DF = None
-HARD_LEADERBOARD_DF = None
-ELO_TASK_DF = None
-ELO_BENCH_DF = None
-HARD_ELO_TASK_DF = None
-HARD_ELO_BENCH_DF = None
-COMPLETE_SOLVE_DF = None
-INSTRUCT_SOLVE_DF = None
-HARD_COMPLETE_SOLVE_DF = None
-HARD_INSTRUCT_SOLVE_DF = None
-
-DATA = datasets.load_dataset(DATA_REPO, "default", cache_dir=HF_HOME, split=DATA_VERSION,
-                             verification_mode="no_checks")
-
-
-def filter_data(data, keyword):
-    if not keyword:
-        return data
-    filtered_data = [item for item in data if keyword.lower() in item['complete_prompt'].lower()]
-    return filtered_data
-
-
-def update_display(search_keyword, index, show_test):
-    filtered_data = filter_data(DATA, search_keyword)
-    
-    if not filtered_data:
-        return ["No data available. Check the search criteria."] + [""] * 4 + [0, gr.update(maximum=0, value=0)]
-    
-    max_index = len(filtered_data) - 1
-    index = min(max(0, index), max_index)
+from collections import Counter, defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED
+from datetime import datetime
+from typing import Any, Dict, List, Tuple
+from warnings import warn
+
+import numpy as np
+from termcolor import cprint
+from tqdm import tqdm
+
+from bigcodebench.data import get_bigcodebench, get_bigcodebench_hash, load_solutions
+from bigcodebench.data.utils import CACHE_DIR
+from bigcodebench.eval import PASS, compatible_eval_result, estimate_pass_at_k, untrusted_check
+from bigcodebench.gen.util import trusted_check
+
+Result = Tuple[str, List[bool]]
+
+
+def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit):
+    cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl")
+    if os.path.exists(cache_file):
+        if check_gt_only:
+            os.remove(cache_file)
+        else:
+            print(f"Load from ground-truth from {cache_file}")
+            with open(cache_file, "rb") as f:
+                return pickle.load(f)
+
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    print("\nAsserting the groundtruth...")
+    tbegin = time.time()
     
-    task_id = filtered_data[index]['task_id']
-    snippet1 = filtered_data[index]['complete_prompt']
-    snippet2 = filtered_data[index]['instruct_prompt']
-    # snippet3 = filtered_data[index]['canonical_solution'] if show_solution else ""
-    snippet4 = filtered_data[index]['test'] if show_test else ""
-    
-    return [
-        task_id,
-        snippet1,
-        snippet2,
-        # snippet3,
-        snippet4,
-        len(filtered_data),
-        gr.update(maximum=max_index, value=index)
-    ]
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
-
-
-def time_diff_wrapper(func):
-    def wrapper(*args, **kwargs):
-        start_time = time.time()
-        result = func(*args, **kwargs)
-        end_time = time.time()
-        diff = end_time - start_time
-        logging.info(f"Time taken for {func.__name__}: {diff} seconds")
-        return result
-
-    return wrapper
-
-
-@time_diff_wrapper
-def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5):
-    """Download dataset with exponential backoff retries."""
-    attempt = 0
-    while attempt < max_attempts:
-        try:
-            logging.info(f"Downloading {repo_id} to {local_dir}")
-            snapshot_download(
-                repo_id=repo_id,
-                local_dir=local_dir,
-                repo_type=repo_type,
-                tqdm_class=None,
-                etag_timeout=30,
-                max_workers=8,
-            )
-            logging.info("Download successful")
-            return
-        except Exception as e:
-            wait_time = backoff_factor**attempt
-            logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s")
-            time.sleep(wait_time)
-            attempt += 1
-    raise Exception(f"Failed to download {repo_id} after {max_attempts} attempts")
-
-def get_latest_data_leaderboard(
-    leaderboard_initial_df = None,
-    hard_leaderboard_initial_df = None,
-    elo_task_df = None,
-    elo_bench_df = None,
-    hard_elo_task_df = None,
-    hard_elo_bench_df = None,
-    complete_solve_df = None,
-    instruct_solve_df = None,
-    hard_complete_solve_df = None,
-    hard_instruct_solve_df = None
-    ):
-    global NEW_DATA_ON_LEADERBOARD
-    global LEADERBOARD_DF
-    global HARD_LEADERBOARD_DF
-    global ELO_TASK_DF
-    global ELO_BENCH_DF
-    global HARD_ELO_TASK_DF
-    global HARD_ELO_BENCH_DF
-    global COMPLETE_SOLVE_DF
-    global INSTRUCT_SOLVE_DF
-    global HARD_COMPLETE_SOLVE_DF
-    global HARD_INSTRUCT_SOLVE_DF
-
-    if NEW_DATA_ON_LEADERBOARD:
-        print("Leaderboard updated at reload!")
-        leaderboard_dataset = datasets.load_dataset(
-            RESULT_REPO, 
-            "default", 
-            split="train", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        )
-        LEADERBOARD_DF = get_leaderboard_df(
-            leaderboard_dataset=leaderboard_dataset, 
-            cols=COLS,
-        )
-        hard_leaderboard_dataset = datasets.load_dataset(
-            HARD_RESULT_REPO, 
-            "default", 
-            split="train", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        )
-        hard_leaderboard_df = get_leaderboard_df(
-            leaderboard_dataset=hard_leaderboard_dataset, 
-            cols=COLS,
-        )
-        HARD_LEADERBOARD_DF = hard_leaderboard_df
-        
-        elo_task_df = datasets.load_dataset(
-            ELO_REPO,
-            "default", 
-            split="task_no_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        elo_bench_df = datasets.load_dataset(
-            ELO_REPO,
-            "default", 
-            split="benchmark_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        ELO_TASK_DF = elo_task_df
-        ELO_BENCH_DF = elo_bench_df
+    with ProcessPoolExecutor(max_workers=n_workers) as executor:
+        futures = []
+        n_samples = 0
+        expected_time = dict()
         
-        hard_elo_task_df = datasets.load_dataset(
-            HARD_ELO_REPO,
-            "default", 
-            split="task_no_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        hard_elo_bench_df = datasets.load_dataset(
-            HARD_ELO_REPO,
-            "default", 
-            split="benchmark_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        HARD_ELO_TASK_DF = hard_elo_task_df
-        HARD_ELO_BENCH_DF = hard_elo_bench_df
-        
-        complete_solve_df = datasets.load_dataset(
-            SOLVE_REPO,
-            "default",
-            split="complete",
-            cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
-            verification_mode="no_checks"
-        ).to_pandas()
-        instruct_solve_df = datasets.load_dataset(
-            SOLVE_REPO,
-            "default",
-            split="instruct",
-            cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
-            verification_mode="no_checks"
-        ).to_pandas()
-        COMPLETE_SOLVE_DF = complete_solve_df
-        INSTRUCT_SOLVE_DF = instruct_solve_df
-        
-        hard_complete_solve_df = datasets.load_dataset(
-            HARD_SOLVE_REPO,
-            "default",
-            split="complete",
-            cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
-            verification_mode="no_checks"
-        ).to_pandas()
-        hard_instruct_solve_df = datasets.load_dataset(
-            HARD_SOLVE_REPO,
-            "default",
-            split="instruct",
-            cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
-            verification_mode="no_checks"
-        ).to_pandas()        
-        HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
-        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
-        
-        NEW_DATA_ON_LEADERBOARD = False
+        for problem in problems.values():
+            args = (
+                problem["complete_prompt"] + "\n" + problem["canonical_solution"],
+                problem["test"],
+                problem["task_id"],
+                max_as_limit,
+                max_data_limit,
+                max_stack_limit,
+                min_time_limit,
+            )
+            
+            futures.append(executor.submit(trusted_check, *args))
+            n_samples += 1
 
-    else:
-        LEADERBOARD_DF = leaderboard_initial_df
-        # HARD_LEADERBOARD_DF = hard_leaderboard_initial_df
-        ELO_TASK_DF = elo_task_df
-        # ELO_BENCH_DF = elo_bench_df
-        # HARD_ELO_TASK_DF = hard_elo_task_df
-        HARD_ELO_BENCH_DF = hard_elo_bench_df
-        COMPLETE_SOLVE_DF = complete_solve_df
-        # INSTRUCT_SOLVE_DF = instruct_solve_df
-        # HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
-        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
-        
-    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
-    # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
-
-
-def init_space():
-    """Initializes the application space, loading only necessary data."""
-
-    # Always redownload the leaderboard DataFrame
-    global LEADERBOARD_DF
-    global HARD_LEADERBOARD_DF
-    global ELO_TASK_DF
-    global ELO_BENCH_DF
-    global HARD_ELO_TASK_DF
-    global HARD_ELO_BENCH_DF
-    global COMPLETE_SOLVE_DF
-    global INSTRUCT_SOLVE_DF
-    global HARD_COMPLETE_SOLVE_DF
-    global HARD_INSTRUCT_SOLVE_DF
+        for future in tqdm(as_completed(futures), total=n_samples):
+            result = future.result()
+            expected_time[result["task_id"]] = result["time"]
     
-    LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
-    # HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
-
-    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
-    # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
-
-# Initialize VoteManager
-# vote_manager = VoteManager(VOTES_PATH, EVAL_REQUESTS_PATH, VOTES_REPO)
-
-
-# Schedule the upload_votes method to run every 15 minutes
-# schedule.every(15).minutes.do(vote_manager.upload_votes)
-
-# Start the scheduler in a separate thread
-# scheduler_thread = Thread(target=run_scheduler, args=(vote_manager,), daemon=True)
-# scheduler_thread.start()
-
-# Calls the init_space function with the `full_init` parameter determined by the `do_full_init` variable.
-# This initializes various DataFrames used throughout the application, with the level of initialization detail controlled by the `do_full_init` flag.
-LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, \
-ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, \
-COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, \
-HARD_INSTRUCT_SOLVE_DF = init_space()
-# HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = init_space()
-
-# Data processing for plots now only on demand in the respective Gradio tab
-# def load_and_create_plots():
-#     plot_df = create_plot_df(create_scores_df(LEADERBOARD_DF))
-#     return plot_df
-
-# Function to check if a user is logged in
-def check_login(profile: gr.OAuthProfile | None) -> bool:
-    if profile is None:
-        return False
-    return True
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.type.name, type="checkboxgroup", label="Model Types"),
-            ColumnFilter(AutoEvalColumn.openness.name, type="checkboxgroup", label="Openness"),
-            ColumnFilter(AutoEvalColumn.size_range.name, type="dropdown", label="Model Size"),
-            ColumnFilter(AutoEvalColumn.moe.name, type="checkboxgroup", label="Model Architecture"),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-        )
+    print(f"Expected outputs computed in {time.time() - tbegin:.2f}s")
+    
+    if any(expected_time.values()):
+        with open(cache_file, "wb") as f:
+            pickle.dump(expected_time, f)
+
+    return expected_time
+
+
+def check_correctness(
+    completion_id: int,
+    problem: Dict[str, Any],
+    solution: str,
+    max_as_limit: float,
+    max_data_limit: float,
+    max_stack_limit: float,
+    identifier=None,
+    min_time_limit: float = 0.1,
+    gt_time_limit: float = 2.0,
+) -> Dict[str, Result]:  
+    ret = {
+        "completion_id": completion_id,
+        "task_id": problem["task_id"],
+        "_identifier": identifier,
+        "solution": solution,
+    }
+    ret["base"] = untrusted_check(
+        solution,
+        problem["test"],
+        problem["entry_point"],
+        max_as_limit,
+        max_data_limit,
+        max_stack_limit,
+        min_time_limit,
+        gt_time_limit,
+    )
+    return ret
+
+
+def evaluate(
+    split: str,
+    subset: str,
+    samples: str,
+    pass_k: str="1,5,10",
+    parallel: int = None,
+    min_time_limit: float = 1,
+    max_as_limit: int = 30 * 1024,
+    max_data_limit: int = 30 * 1024,
+    max_stack_limit: int = 10,
+    check_gt_only: bool = False,
+    no_gt: bool = False,
+):
+    pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
+    if parallel is None:
+        n_workers = max(1, multiprocessing.cpu_count() // 2)
+    else:
+        n_workers = parallel
 
+    if check_gt_only:
+        samples = "__dummy__.jsonl"
 
-def init_others(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Gradio DataFrame is empty or None.")
-    return gr.Dataframe(dataframe, visible=False)
+    extra = subset + "_" if subset != "full" else ""
+    if os.path.isdir(samples):
+        result_path = os.path.join(samples, f"{extra}eval_results.json")
+    else:
+        assert samples.endswith(".jsonl")
+        result_path = samples.replace(".jsonl", f"_{extra}eval_results.json")
 
-main_block = gr.Blocks(css=custom_css)
-with main_block as demo:
-    with gr.Row(elem_id="header-row"):
-        gr.HTML(TITLE + "<p>Total models: " + str(len(HARD_LEADERBOARD_DF))+ "</p>")
+    problems = get_bigcodebench(subset=subset)
+    dataset_hash = get_bigcodebench_hash(subset=subset)
+    
+    if not no_gt:
+        expected_time = get_groundtruth(n_workers, problems, dataset_hash, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit)
+    else:
+        expected_time = {task_id: None for task_id in problems}
     
-    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")    
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.Tab("💎 Hard Set") as hard_tabs:
-            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="hard_bench"):
-                hard_leaderboard = init_leaderboard(HARD_LEADERBOARD_DF)
-                gr.Markdown(
-                    """
-                **Notes:**
-                - For the efficiency reasons, we only display the Hard Set leaderboard.
-                - _Hard Set_ vs _Full Set_:
-                    - <u>Hard Set</u>: A subset of ~150 BigCodeBench tasks which is more user-facing and challenging.
-                    - <u>Full Set</u>: The full set of 1140 BigCodeBench tasks.
-                - _Complete_ vs _Instruct_:
-                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This split tests if the models are good at coding.
-                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This split tests if the models are really capable enough to understand human intents to code.
-                - `Complete` and `Instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark splits.
-                - `Average` is the average of `Complete` and `Instruct` when both are available.
-                - `Elo Rating` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the Complete + Instruct splits. The rating starts from 1000 and is bootstrapped 500 times. We only consider the models having both `Complete` and `Instruct` scores.
-                - `#Act Params (B)` is the number of activated model parameters during inference.
-                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
-                - For more details check the 📝 About section.
-                """,
-                    elem_classes="markdown-text",
+    gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems])
+    failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems]
+    
+    if os.path.isfile(result_path):
+        with open(result_path, "r") as f:
+            results = json.load(f)
+        results = compatible_eval_result(results)
+    else:
+        if check_gt_only:
+            if gt_pass_rate > 0.99:
+                cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green")
+            else:
+                cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red")
+            if len(failed_tasks) > 0:
+                cprint(f"Failed tasks: {failed_tasks}", "red")
+            return {"gt_pass_rate":float(gt_pass_rate), "failed_tasks": failed_tasks}
+        
+        results = {
+            "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
+            "eval": {},
+        }
+
+        with ProcessPoolExecutor(max_workers=n_workers) as executor:
+            futures = []
+            completion_id = Counter()
+            n_samples = 0
+            eval_results = defaultdict(list)  # task_id ->
+            remainings = set()
+
+            print("Reading samples...")
+            for sample in tqdm(load_solutions(samples)):
+                task_id = sample["task_id"]
+                
+                if task_id not in problems:
+                    warn(
+                        f"Task {task_id} is found in the samples but not found in the dataset"
+                    )
+                    continue
+                solution = (
+                    sample["solution"]
+                    if "solution" in sample
+                    else problems[task_id]["complete_prompt"] + sample["completion"]
                 )
-            
-            with gr.TabItem("📊 Elo Rating", id="hard_elo"):
-                with gr.Column():
-                    with gr.Group():
-                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
-                        hard_task_elo_map = gr.Plot()
-                        hard_elo_task_gr = init_others(HARD_ELO_TASK_DF)
-                        demo.load(plot_elo_mle, [hard_elo_task_gr],
-                                    hard_task_elo_map)
-                    with gr.Group():
-                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
-                        hard_bench_elo_map = gr.Plot()
-                        hard_elo_bench_gr = init_others(HARD_ELO_BENCH_DF)
-                        demo.load(plot_elo_mle, [hard_elo_bench_gr],
-                                    hard_bench_elo_map)
-                        
-            with gr.TabItem("🧩 Solve Rate", id="hard_solve"):
-                with gr.Column():
-                    hard_complete_map = gr.Plot()
-                    hard_complete_solve_gr = init_others(HARD_COMPLETE_SOLVE_DF)
-                    demo.load(plot_solve_rate, [hard_complete_solve_gr,
-                                                gr.Textbox("Complete", visible=False),
-                                                gr.Number(10, visible=False),
-                                                gr.Number(16, visible=False),
-                                                ], hard_complete_map)
-                    hard_instruct_map = gr.Plot()
-                    hard_instruct_solve_gr = init_others(HARD_INSTRUCT_SOLVE_DF)
-                    demo.load(plot_solve_rate, [hard_instruct_solve_gr,
-                                                gr.Textbox("Instruct", visible=False),
-                                                gr.Number(10, visible=False),
-                                                gr.Number(16, visible=False),
-                                                ], hard_instruct_map)
-        with gr.Tab("🎯 Full Set") as full_tabs:
-            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="full_bench"):
-                leaderboard = init_leaderboard(LEADERBOARD_DF)
-                gr.Markdown(
-                    """
-                **Notes:**
-                - _Complete_ vs _Instruct_:
-                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding.
-                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code.
-                - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
-                - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the BigCodeBench-Complete split. The rating starts from 1000 and is bootstrapped 500 times.
-                - `size` is the amount of activated model weight during inference.
-                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
-                - For more details check the 📝 About section.
-                """,
-                    elem_classes="markdown-text",
+                if "sanitized-calibrated" in samples:
+                    solution = problems[task_id]["code_prompt"] + "\n    pass\n" + solution
+                remainings.add(sample["_identifier"])
+                args = (
+                    completion_id[task_id],
+                    problems[task_id],
+                    solution,
+                    max_as_limit,
+                    max_data_limit,
+                    max_stack_limit,
+                    sample["_identifier"],
+                    min_time_limit,
+                    expected_time[task_id] if expected_time[task_id] else 20
+                )
+                futures.append(executor.submit(check_correctness, *args))
+                completion_id[task_id] += 1
+                n_samples += 1
+
+            assert n_samples == len(remainings), "Missing problems in unfinished"
+            assert len(completion_id) == len(problems), "Missing problems in samples"
+
+            def stucking_checker():
+                not_done = futures
+                while len(not_done) > 0:
+                    done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED)
+
+                    if len(done) == 0:
+                        warn("No samples have finished testing in the last 240s")
+                        warn(f"{len(remainings)} samples to be tested: {remainings}")
+
+            threading.Thread(target=stucking_checker).start()
+
+            for future in tqdm(as_completed(futures), total=n_samples):
+                result = future.result()
+                remainings.remove(result["_identifier"])
+                eval_results[result["task_id"]].append(result)
+
+
+        # sort the results for each problem by completion_id
+        for task_id, task_results in eval_results.items():
+            task_results.sort(key=lambda x: x["completion_id"])
+            results["eval"][task_id] = []
+            for res in task_results:
+                stat, details = res["base"]
+                results["eval"][task_id].append(
+                    {
+                        "task_id": task_id,
+                        "solution": res["solution"],
+                        "status": stat,
+                        "details": details,
+                    }
                 )
-            
-            with gr.TabItem("📊 Elo Rating", id="full_elo"):
-                with gr.Column():
-                    with gr.Group():
-                        
-                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
-                        task_elo_map = gr.Plot()
-                        elo_task_gr = init_others(ELO_TASK_DF)
-                        demo.load(plot_elo_mle, [elo_task_gr], task_elo_map)
-                    with gr.Group():
-                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
-                        bench_elo_map = gr.Plot()
-                        elo_bench_gr = init_others(ELO_BENCH_DF)
-                        demo.load(plot_elo_mle, [elo_bench_gr], bench_elo_map)
-                    
-            with gr.TabItem("🧩 Solve Rate", id="full_solve"):
-                with gr.Column():
-                    complete_map = gr.Plot()
-                    complete_solve_gr = init_others(COMPLETE_SOLVE_DF)
-                    demo.load(plot_solve_rate, [complete_solve_gr,
-                                                gr.Textbox("Complete", visible=False),
-                                                ], complete_map)
-                    instruct_map = gr.Plot()
-                    instruct_solve_gr = init_others(INSTRUCT_SOLVE_DF)
-                    demo.load(plot_solve_rate, [instruct_solve_gr,
-                                                gr.Textbox("Instruct", visible=False),
-                                                ], instruct_map)
-        with gr.TabItem("📝 About", id=3):
-            gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
-        with gr.TabItem("🔎 Data Viewer", id="viewer"):
-            search_input = gr.Textbox(label="Search by keyword")
-            count_output = gr.Number(label="Number of filtered items")
-            index_slider = gr.Slider(minimum=0, maximum=len(DATA)-1, step=1, label="Select Index")
-            # show_solution = gr.Checkbox(label="Show Solution")
-            show_test = gr.Checkbox(label="Show Test Cases")
-            update_button = gr.Button("Update")
-            
-            task_id_output = gr.Textbox(label="Task ID")
-            code_completion = gr.Code(language="python", label="Code Completion")
-            nl_instruction = gr.Code(language="markdown", label="Natural Language Instruction")
-            # solution = gr.Code(language="python", label="Solution")
-            test_cases = gr.Code(language="python", label="Test Cases")
-            
-            update_button.click(
-                update_display, 
-                inputs=[search_input, index_slider, show_test],
-                outputs=[task_id_output, code_completion, nl_instruction, test_cases, count_output, index_slider]
-            )
-
-            # Initial load
-            demo.load(
-                update_display, 
-                inputs=[search_input, index_slider, show_test],
-                outputs=[task_id_output, code_completion, nl_instruction, test_cases, count_output, index_slider]
-            )
 
-        with gr.TabItem("🚀 Request", id=4):
-            gr.Markdown(SUBMISSION_TEXT_3)
-            
-        with gr.TabItem("🛠️ Execute", id=5):
-            gr.Markdown("# BigCodeBench Evaluator")
-            
-            with gr.Row():
-                jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
-                split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
-                subset = gr.Dropdown(choices=["hard", "full"], label="Subset", value="hard")
-            
-            with gr.Row():
-                parallel = gr.Number(label="Parallel (optional)", precision=0)
-                min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
-                max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
-            
-            with gr.Row():
-                max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
-                max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
-                check_gt_only = gr.Checkbox(label="Check GT Only")
-                no_gt = gr.Checkbox(label="No GT")
-            
-            command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
-            with gr.Row():
-                submit_btn = gr.Button("Run Evaluation")
-                download_btn = gr.DownloadButton(label="Download Result")
-            log_output = gr.Textbox(label="Execution Logs", lines=20)
-            
-            input_components = [
-                jsonl_file, split, subset, parallel,
-                min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
-                check_gt_only, no_gt
-            ]
-            
-            for component in input_components:
-                component.change(generate_command, inputs=input_components, outputs=command_output)
-                
-            
-            def start_evaluation(command, jsonl_file, subset, split):
-                extra = subset + "_" if subset != "full" else ""
-                if jsonl_file is not None:
-                    result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
-                else:
-                    result_path = None
-
-                for log in stream_logs(command, jsonl_file):
-                    if jsonl_file is not None:
-                        yield log, gr.update(value=result_path, label=result_path), gr.update()
-                    else:
-                        yield log, gr.update(), gr.update()
-                result_file = find_result_file()
-                if result_file:
-                    return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
-                            # gr.Button(visible=False)#,
-                            # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
-                else:
-                    return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
-                            # gr.Button("Run Evaluation", visible=True),
-                            # gr.DownloadButton(visible=False))
-            submit_btn.click(start_evaluation,
-                        inputs=[command_output, jsonl_file, subset, split],
-                        outputs=[log_output, download_btn])
+    # Calculate pass@k.
+    total = np.array([len(r) for k, r in results["eval"].items() if k in problems])
+    base_correct = []
+
+    for key, res in results["eval"].items():
+        if key not in problems:
+            continue
+        bc = sum([r["status"] == PASS for r in res])
+        base_correct.append(bc)
+
+    base_correct = np.array(base_correct)
+
+    pass_at_k = {
+        f"pass@{k}": float(estimate_pass_at_k(total, base_correct, k).mean())
+        for k in pass_k
+        if total.min() >= k
+    }
+    pass_at_k["gt_pass_rate"] = float(gt_pass_rate)
+    pass_at_k["failed_tasks"] = failed_tasks
+    return pass_at_k
+
+    # mode = "-calibrated" if "sanitized-calibrated" in samples else ""
+    # extra = subset.capitalize()
+    # split = split.capitalize()
+    # cprint(f"BigCodeBench-{split}{mode} ({extra})", "green")
+        
+    # if no_gt:
+    #     cprint(f"Groundtruth is not checked", "yellow")
+    # else:
+    #     if gt_pass_rate > 0.99:
+    #         cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}", "green")
+    #     else:
+    #         cprint(f"Groundtruth pass rate: {gt_pass_rate:.3f}\nPlease be cautious!", "red")
+        
+    #     if len(failed_tasks) > 0:
+    #         cprint(f"Failed tasks: {failed_tasks}", "red")
     
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    # for k, v in pass_at_k.items():
+    #     cprint(f"{k}:\t{v:.3f}", "green")
+
+    # # save results
+    # if os.path.isfile(result_path):
+    #     decision = ""
+    #     while decision.lower() not in ["y", "n"]:
+    #         print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...")
+    #         decision = input()
+
+    #     if decision.lower() == "y":
+    #         # mv the file to a backup
+    #         new_path = result_path + ".bak"
+    #         while os.path.isfile(new_path):
+    #             new_path += ".bak"
+    #         os.rename(result_path, new_path)
+    #         print(f"Backup {result_path} to {new_path}")
+
+    # if not os.path.isfile(result_path):
+    #     with open(result_path, "w") as f:
+    #         json.dump(results, f, indent=2)
+
+    # if save_pass_rate:
+    #     pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json")
+    #     pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0]
+    #     pass_at_k["calibrated"] = "sanitized-calibrated" in samples
+    #     pass_at_k["subset"] = subset
+
+    #     def save_pass_at_k():
+    #         with open(pass_at_k_path, "w") as f:
+    #             json.dump(pass_at_k, f, indent=2)
+
+    #     if os.path.isfile(pass_at_k_path):
+    #         saved_pass_at_k = json.load(open(pass_at_k_path, "r"))
+    #         # compare saved_pass_at_k with pass_at_k
+    #         for k in saved_pass_at_k.keys():
+    #             if pass_at_k[k] != saved_pass_at_k[k]:
+    #                 cprint(f"Warning: {k} is different from the saved one", "yellow")
                     
-    main_block.load(fn=get_latest_data_leaderboard, inputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
-    # main_block.load(fn=get_latest_data_leaderboard, inputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
-    # leaderboard.change(fn=get_latest_data_queue, inputs=None, outputs=[finished_eval_table, running_eval_table, pending_eval_table])
-    # pending_eval_table.change(fn=vote_manager.create_request_vote_df, inputs=[pending_eval_table], outputs=[pending_eval_table_votes])
-
-main_block.queue(default_concurrency_limit=100)
-
-
-def enable_space_ci_and_return_server(ui: gr.Blocks) -> WebhooksServer:
-    # Taken from https://huggingface.co/spaces/Wauplin/gradio-space-ci/blob/075119aee75ab5e7150bf0814eec91c83482e790/src/gradio_space_ci/webhook.py#L61
-    # Compared to original, this one do not monkeypatch Gradio which allows us to define more webhooks.
-    # ht to Lucain!
-    if SPACE_ID is None:
-        print("Not in a Space: Space CI disabled.")
-        return WebhooksServer(ui=main_block)
-
-    if IS_EPHEMERAL_SPACE:
-        print("In an ephemeral Space: Space CI disabled.")
-        return WebhooksServer(ui=main_block)
-
-    card = RepoCard.load(repo_id_or_path=SPACE_ID, repo_type="space")
-    config = card.data.get("space_ci", {})
-    print(f"Enabling Space CI with config from README: {config}")
-
-    return configure_space_ci(
-        blocks=ui,
-        trusted_authors=config.get("trusted_authors"),
-        private=config.get("private", "auto"),
-        variables=config.get("variables", "auto"),
-        secrets=config.get("secrets"),
-        hardware=config.get("hardware"),
-        storage=config.get("storage"),
+    #         # ask user whether to save the pass@k
+    #         decision = ""
+    #         while decision.lower() not in ["y", "n"]:
+    #             print(f"Save pass@k to {pass_at_k_path}? [Y/N]")
+    #             decision = input()
+    #         if decision.lower() == "y":
+    #             save_pass_at_k()
+                
+    #     else:
+    #         save_pass_at_k()
+
+def run_gradio():
+    interface = gr.Interface(
+        fn=evaluate,
+        inputs=[
+            gr.Dropdown(["complete", "instruct"], label="Split"),
+            gr.Dropdown(["full", "hard"], label="Subset"),
+            gr.File(label="Samples Path (.jsonl)"),
+            gr.Textbox(label="Pass k Values (comma-separated)", value="1,5,10"),
+            gr.Slider(1, multiprocessing.cpu_count(), step=1, label="Parallel Workers"),
+            gr.Slider(0.1, 10, step=0.1, label="Min Time Limit", value=1),
+            gr.Slider(1, 100 * 1024, step=1024, label="Max AS Limit", value=30 * 1024),
+            gr.Slider(1, 100 * 1024, step=1024, label="Max Data Limit", value=30 * 1024),
+            gr.Slider(1, 100, step=1, label="Max Stack Limit", value=10),
+            gr.Checkbox(label="Check GT Only"),
+            gr.Checkbox(label="No GT"),
+        ],
+        outputs="text",
+        # concurrency_limit=None
     )
+    interface.queue(default_concurrency_limit=None)
+    interface.launch(show_error=True)
 
-# Create webhooks server (with CI url if in Space and not ephemeral)
-webhooks_server = enable_space_ci_and_return_server(ui=main_block)
-
-# Add webhooks
-@webhooks_server.add_webhook
-def update_leaderboard(payload: WebhookPayload) -> None:
-    """Redownloads the leaderboard dataset each time it updates"""
-    if payload.repo.type == "dataset" and payload.event.action == "update":
-        global NEW_DATA_ON_LEADERBOARD
-        if NEW_DATA_ON_LEADERBOARD:
-            return
-        NEW_DATA_ON_LEADERBOARD = True
-
-        for repo in [RESULT_REPO, HARD_RESULT_REPO, ELO_REPO, HARD_ELO_REPO, SOLVE_REPO, HARD_SOLVE_REPO]:
-            datasets.load_dataset(
-                repo, 
-                "default", 
-                cache_dir=HF_HOME, 
-                download_mode=datasets.DownloadMode.FORCE_REDOWNLOAD, 
-                verification_mode="no_checks"
-            )
-        
-        
-
-webhooks_server.launch()
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
-scheduler.start()
+if __name__ == "__main__":
+    run_gradio()
+    # evaluate("complete", "hard", "meta-llama--Llama-3.2-3B-Instruct--bigcodebench-instruct--vllm-0-1.jsonl")