File size: 9,380 Bytes
aca9a0c 1ad7e58 aca9a0c 55f4d70 aca9a0c 828190b aca9a0c b42c9ff aca9a0c 4211404 aca9a0c ab1a0c8 4211404 aca9a0c 9e24019 aca9a0c de45929 aca9a0c a2d883f aca9a0c 2480d21 9fd6b97 aca9a0c 9fd6b97 aca9a0c 5a5b2b2 aca9a0c eee2e10 aca9a0c 0ee99b8 aca9a0c 0ee99b8 aca9a0c 0ee99b8 aca9a0c de45929 aca9a0c 7eeb535 89eb067 3204d18 7eeb535 9fd6b97 7eeb535 9fd6b97 7eeb535 b42c9ff 7eeb535 aca9a0c 0ee99b8 7eeb535 aca9a0c 7eeb535 aca9a0c 7eeb535 aca9a0c 7eeb535 4c2c8e2 3204d18 aca9a0c 7eeb535 3204d18 aca9a0c 4211404 0ee99b8 ab1a0c8 c12bb85 f9ea51a c12bb85 ab1a0c8 4211404 0fc9cc1 f71fb26 0fc9cc1 b18ded7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 |
import gradio as gr
import json
import logging
import multiprocessing
import os
import pickle
import threading
import time
from collections import Counter, defaultdict
from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED
from datetime import datetime
from typing import Any, Dict, List, Tuple
from warnings import warn
import gc
import numpy as np
from huggingface_hub import HfApi
from bigcodebench.data import get_bigcodebench, get_bigcodebench_hash, load_solutions
from bigcodebench.data.utils import CACHE_DIR
from bigcodebench.eval import PASS, compatible_eval_result, estimate_pass_at_k, untrusted_check
from bigcodebench.gen.util import trusted_check
from apscheduler.schedulers.background import BackgroundScheduler
REPO_ID = "bigcode/bigcodebench-evaluator"
HF_TOKEN = os.environ.get("HF_TOKEN", None)
API = HfApi(token=HF_TOKEN)
Result = Tuple[str, List[bool]]
def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit):
cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl")
if os.path.exists(cache_file):
with open(cache_file, "rb") as f:
return pickle.load(f)
os.makedirs(CACHE_DIR, exist_ok=True)
tbegin = time.time()
with ProcessPoolExecutor(max_workers=n_workers) as executor:
futures = []
n_samples = 0
expected_time = dict()
for problem in problems.values():
args = (
problem["complete_prompt"] + "\n" + problem["canonical_solution"],
problem["test"],
problem["task_id"],
max_as_limit,
max_data_limit,
max_stack_limit,
min_time_limit,
)
futures.append(executor.submit(trusted_check, *args))
n_samples += 1
for future in as_completed(futures):
result = future.result()
expected_time[result["task_id"]] = result["time"]
if any(expected_time.values()):
with open(cache_file, "wb") as f:
pickle.dump(expected_time, f)
return expected_time
def check_correctness(
completion_id: int,
problem: Dict[str, Any],
solution: str,
max_as_limit: float,
max_data_limit: float,
max_stack_limit: float,
identifier=None,
min_time_limit: float = 0.1,
gt_time_limit: float = 2.0,
) -> Dict[str, Result]:
ret = {
"completion_id": completion_id,
"task_id": problem["task_id"],
"_identifier": identifier,
"solution": solution,
}
ret["base"] = untrusted_check(
solution,
problem["test"],
problem["entry_point"],
max_as_limit,
max_data_limit,
max_stack_limit,
min_time_limit,
gt_time_limit,
)
return ret
def evaluate(
split: str,
subset: str,
samples: str,
pass_k: str="1,5,10",
parallel: int = -1,
min_time_limit: float = 1,
max_as_limit: int = 30 * 1024,
max_data_limit: int = 30 * 1024,
max_stack_limit: int = 10,
check_gt_only: bool = False,
no_gt: bool = False,
):
pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
if parallel < 1:
n_workers = max(1, multiprocessing.cpu_count() // 2)
else:
n_workers = parallel
if check_gt_only:
samples = "__dummy__.jsonl"
extra = subset + "_" if subset != "full" else ""
problems = get_bigcodebench(subset=subset)
dataset_hash = get_bigcodebench_hash(subset=subset)
if not no_gt:
expected_time = get_groundtruth(n_workers, problems, dataset_hash, check_gt_only, max_as_limit, max_data_limit, max_stack_limit, min_time_limit)
else:
expected_time = {task_id: None for task_id in problems}
gt_pass_rate = np.mean([1 if v is not None else 0 for k, v in expected_time.items() if k in problems])
failed_tasks = [k for k, v in expected_time.items() if v is None and k in problems]
pass_at_k = dict()
results = {
"date": datetime.now().strftime("%Y-%m-%d %H:%M"),
"eval": {},
}
if not check_gt_only:
with ProcessPoolExecutor(max_workers=n_workers) as executor:
futures = []
completion_id = Counter()
n_samples = 0
eval_results = defaultdict(list) # task_id ->
remainings = set()
for sample in load_solutions(samples):
task_id = sample["task_id"]
if task_id not in problems:
continue
solution = (
sample["solution"]
if "solution" in sample
else problems[task_id]["complete_prompt"] + sample["completion"]
)
if "sanitized-calibrated" in samples:
solution = problems[task_id]["code_prompt"] + "\n pass\n" + solution
remainings.add(sample["_identifier"])
args = (
completion_id[task_id],
problems[task_id],
solution,
max_as_limit,
max_data_limit,
max_stack_limit,
sample["_identifier"],
min_time_limit,
expected_time[task_id] if expected_time[task_id] else 20
)
futures.append(executor.submit(check_correctness, *args))
completion_id[task_id] += 1
n_samples += 1
assert n_samples == len(remainings), "Missing problems in unfinished"
assert len(completion_id) == len(problems), "Missing problems in samples"
for future in as_completed(futures):
result = future.result()
remainings.remove(result["_identifier"])
eval_results[result["task_id"]].append(result)
del future, result
gc.collect()
# sort the results for each problem by completion_id
for task_id, task_results in eval_results.items():
task_results.sort(key=lambda x: x["completion_id"])
results["eval"][task_id] = []
for res in task_results:
stat, details = res["base"]
results["eval"][task_id].append(
{
"task_id": task_id,
"solution": res["solution"],
"status": stat,
"details": details,
}
)
# Calculate pass@k.
total = np.array([len(r) for k, r in results["eval"].items() if k in problems])
base_correct = []
for key, res in results["eval"].items():
if key not in problems:
continue
bc = sum([r["status"] == PASS for r in res])
base_correct.append(bc)
base_correct = np.array(base_correct)
pass_at_k.update({
f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
for k in pass_k
if total.min() >= k
})
del problems, futures
gc.collect()
pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0]
pass_at_k["split"] = split
pass_at_k["subset"] = subset
pass_at_k["calibrated"] = "sanitized-calibrated" in samples
pass_at_k["gt_pass_rate"] = gt_pass_rate
pass_at_k["failed_tasks"] = failed_tasks
return results, pass_at_k
# def run_gradio():
interface = gr.Interface(
fn=evaluate,
inputs=[
gr.Dropdown(["complete", "instruct"], label="BigCodeBench Split"),
gr.Dropdown(["full", "hard"], label="BigCodeBench Subset"),
gr.File(label="Samples Path (.jsonl)"),
gr.Textbox(label="Pass k Values (comma-separated)", value="1,5,10"),
gr.Slider(-1, multiprocessing.cpu_count(), step=1, label="Parallel Workers", value=-1),
gr.Slider(0.1, 10, step=0.1, label="Min Time Limit", value=1),
gr.Slider(1, 100 * 1024, step=1024, label="Max AS Limit", value=30 * 1024),
gr.Slider(1, 100 * 1024, step=1024, label="Max Data Limit", value=30 * 1024),
gr.Slider(1, 100, step=1, label="Max Stack Limit", value=10),
gr.Checkbox(label="Check GT Only"),
gr.Checkbox(label="No GT"),
],
outputs=[
gr.JSON(label="Results"),
gr.JSON(label="Eval Results"),
],
# concurrency_limit=None
)
interface.queue(default_concurrency_limit=None)
def preload_gt():
evaluate(split="complete", subset="full", samples="", check_gt_only=True)
evaluate(split="complete", subset="hard", samples="", check_gt_only=True)
def restart_space():
logging.info(f"Restarting space with repo ID: {REPO_ID}")
try:
# Now restart the space
API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
logging.info("Space restarted successfully.")
except Exception as e:
logging.error(f"Failed to restart space: {e}")
# if __name__ == "__main__":
preload_gt()
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", hours=5) # Restart every 5hs
scheduler.start()
interface.launch(show_error=True)
|