import json import logging import os import subprocess import time import pandas as pd from huggingface_hub import snapshot_download from src.envs import EVAL_RESULTS_PATH # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') def time_diff_wrapper(func): def wrapper(*args, **kwargs): start_time = time.time() result = func(*args, **kwargs) end_time = time.time() diff = end_time - start_time logging.info(f"Time taken for {func.__name__}: {diff} seconds") return result return wrapper @time_diff_wrapper def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5): """Download dataset with exponential backoff retries.""" attempt = 0 while attempt < max_attempts: try: logging.info(f"Downloading {repo_id} to {local_dir}") snapshot_download( repo_id=repo_id, local_dir=local_dir, repo_type=repo_type, tqdm_class=None, token=os.environ.get("HF_TOKEN_PRIVATE"), etag_timeout=30, max_workers=8, ) logging.info("Download successful") return except Exception as e: wait_time = backoff_factor ** attempt logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s") time.sleep(wait_time) attempt += 1 logging.error(f"Failed to download {repo_id} after {max_attempts} attempts") def build_leadearboard_df(): """Initializes the application space, loading only necessary data.""" # Check ENV LEADERBOARD_DOWNLOAD if wee need to download the leaderboard if os.getenv("LEADERBOARD_DOWNLOAD", "True") == "True": # These downloads only occur on full initialization # try: # download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH) # download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH) download_dataset("Vikhrmodels/openbench-eval", EVAL_RESULTS_PATH) # print(subprocess.Popen('ls src')) subprocess.run(['rsync', '-avzP', '--ignore-existing', f'{EVAL_RESULTS_PATH[2:]}/external/*', 'src/gen/data/arena-hard-v0.1/model_answer/'], check=False) subprocess.run(['rsync', '-avzP', '--ignore-existing', f'{EVAL_RESULTS_PATH[2:]}/model_judgment/*', 'src/gen/data/arena-hard-v0.1/model_judgement/'], check=False) # except Exception: # restart_space() # Always retrieve the leaderboard DataFrame leaderboard_df = pd.DataFrame.from_records(json.load(open('eval-results/evals/upd.json','r'))) return leaderboard_df.copy()