import gradio as gr import subprocess import sys import os import threading import time import uuid import glob import shutil from pathlib import Path class Logger: def __init__(self, filename): self.terminal = sys.stdout self.log = open(filename, "w") def write(self, message): self.terminal.write(message) self.log.write(message) self.log.flush() def flush(self): self.terminal.flush() self.log.flush() def isatty(self): return False default_command = "bigcodebench.evaluate" is_running = False def generate_command( jsonl_file, split, subset, save_pass_rate, parallel, min_time_limit, max_as_limit, max_data_limit, max_stack_limit, check_gt_only, no_gt ): command = [default_command] if jsonl_file is not None: # Copy the uploaded file to the current directory local_filename = os.path.basename(jsonl_file.name) shutil.copy(jsonl_file.name, local_filename) command.extend(["--samples", local_filename]) command.extend(["--split", split, "--subset", subset]) if save_pass_rate: command.append("--save_pass_rate") if parallel is not None and parallel != 0: command.extend(["--parallel", str(int(parallel))]) command.extend([ "--min-time-limit", str(min_time_limit), "--max-as-limit", str(int(max_as_limit)), "--max-data-limit", str(int(max_data_limit)), "--max-stack-limit", str(int(max_stack_limit)) ]) if check_gt_only: command.append("--check-gt-only") if no_gt: command.append("--no-gt") return " ".join(command) def cleanup_previous_files(jsonl_file=None): for file in glob.glob("*.json") + glob.glob("*.log") + glob.glob("*.jsonl"): try: if jsonl_file is not None and file == jsonl_file: continue os.remove(file) except Exception as e: print(f"Error during cleanup of {file}: {e}") def find_result_file(): json_files = glob.glob("*.json") if json_files: return max(json_files, key=os.path.getmtime) return None def run_bigcodebench(command): global is_running is_running = True yield f"Executing command: {command}\n" process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) for line in process.stdout: yield line process.wait() if process.returncode != 0: yield f"Error: Command exited with status {process.returncode}\n" cleanup_command = "pids=$(ps -u $(id -u) -o pid,comm | grep 'bigcodebench' | awk '{print $1}'); if [ -n \"$pids\" ]; then echo $pids | xargs -r kill; fi; rm -rf /tmp/*" subprocess.run(cleanup_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) is_running = False yield "Evaluation completed.\n" result_file = find_result_file() if result_file: yield f"Result file found: {result_file}\n" else: yield "No result file found.\n" def stream_logs(command, jsonl_file=None): global is_running if jsonl_file is not None: local_filename = os.path.basename(jsonl_file.name) if is_running: yield "A command is already running. Please wait for it to finish.\n" return cleanup_previous_files(local_filename) yield "Cleaned up previous files.\n" log_content = [] for log_line in run_bigcodebench(command): log_content.append(log_line) yield "".join(log_content) with gr.Blocks() as demo: gr.Markdown("# BigCodeBench Evaluator") with gr.Row(): jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"]) split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete") subset = gr.Dropdown(choices=["full", "hard"], label="Subset", value="hard") with gr.Row(): save_pass_rate = gr.Checkbox(label="Save Pass Rate") parallel = gr.Number(label="Parallel (optional)", precision=0) min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1) max_as_limit = gr.Number(label="Max AS Limit", value=30*1024, precision=0) with gr.Row(): max_data_limit = gr.Number(label="Max Data Limit", value=30*1024, precision=0) max_stack_limit = gr.Number(label="Max Stack Limit", value=20, precision=0) check_gt_only = gr.Checkbox(label="Check GT Only") no_gt = gr.Checkbox(label="No GT") command_output = gr.Textbox(label="Command", value=default_command, interactive=False) with gr.Row(): submit_btn = gr.Button("Run Evaluation") download_btn = gr.DownloadButton(label="Download Result", visible=False) log_output = gr.Textbox(label="Execution Logs", lines=20) # Hidden component to store the result file path # result_file_path = gr.State("") def update_command(*args): return generate_command(*args) input_components = [ jsonl_file, split, subset, save_pass_rate, parallel, min_time_limit, max_as_limit, max_data_limit, max_stack_limit, check_gt_only, no_gt ] for component in input_components: component.change(update_command, inputs=input_components, outputs=command_output) def start_evaluation(command, jsonl_file): for log in stream_logs(command, jsonl_file): yield log, gr.update(), gr.update() result_file = find_result_file() if result_file: print(f"Result file: {result_file}") return (gr.update(label="Evaluation completed. Result file found."), gr.Button(visible=False), gr.DownloadButton(label="Download Result", value=result_file)) else: return (gr.update(label="Evaluation completed. No result file found."), gr.Button("Run Evaluation"), gr.DownloadButton(visible=False)) submit_btn.click(start_evaluation, inputs=[command_output, jsonl_file], outputs=[log_output, submit_btn, download_btn]) # def update_logs(session_log_file): # return read_logs(session_log_file) # demo.load(update_logs, inputs=[session_log_file], outputs=[log_output], every=1) if __name__ == "__main__": demo.queue(max_size=300).launch(server_name="0.0.0.0", server_port=7860)