Spaces:
Paused
Paused
import logging | |
import pprint | |
from huggingface_hub import snapshot_download | |
logging.getLogger("openai").setLevel(logging.WARNING) | |
from src.backend.run_eval_suite_lighteval import run_evaluation | |
from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request, set_requests_seen | |
from src.backend.sort_queue import sort_models_by_priority | |
from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, API, LIMIT, TOKEN, ACCELERATOR, VENDOR, REGION | |
from src.about import TASKS_LIGHTEVAL | |
logging.basicConfig(level=logging.ERROR) | |
pp = pprint.PrettyPrinter(width=80) | |
PENDING_STATUS = "PENDING" | |
RUNNING_STATUS = "RUNNING" | |
FINISHED_STATUS = "FINISHED" | |
FAILED_STATUS = "FAILED" | |
snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN) | |
snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN) | |
def run_auto_eval(): | |
current_pending_status = [PENDING_STATUS] | |
# pull the eval dataset from the hub and parse any eval requests | |
# check completed evals and set them to finished | |
check_completed_evals( | |
api=API, | |
checked_status=RUNNING_STATUS, | |
completed_status=FINISHED_STATUS, | |
failed_status=FAILED_STATUS, | |
hf_repo=QUEUE_REPO, | |
local_dir=EVAL_REQUESTS_PATH_BACKEND, | |
hf_repo_results=RESULTS_REPO, | |
local_dir_results=EVAL_RESULTS_PATH_BACKEND | |
) | |
# Get all eval request that are PENDING, if you want to run other evals, change this parameter | |
eval_requests, requests_seen = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND) | |
# Sort the evals by priority (first submitted first run) | |
eval_requests = sort_models_by_priority(api=API, models=eval_requests) | |
print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests") | |
if len(eval_requests) == 0: | |
return | |
eval_request = eval_requests[0] | |
pp.pprint(eval_request) | |
# For GPU | |
if not eval_request or eval_request.params < 0: | |
raise ValueError("Couldn't detect number of params, please make sure the metadata is available") | |
elif eval_request.params < 4: | |
instance_size, instance_type, cap = "small", "g4dn.xlarge", 20 | |
elif eval_request.params < 9: | |
instance_size, instance_type, cap = "medium", "g5.2xlarge", 35 | |
elif eval_request.params < 24: | |
instance_size, instance_type, cap = "xxlarge", "g5.12xlarge", 15 | |
else: | |
raise ValueError("Number of params too big, can't run this model") | |
counter_key = f'count_{instance_type}' | |
if not counter_key in requests_seen: | |
requests_seen[counter_key] = 0 | |
if requests_seen[counter_key] >= cap: | |
set_eval_request( | |
api=API, | |
eval_request=eval_request, | |
set_to_status=FAILED_STATUS, | |
hf_repo=QUEUE_REPO, | |
local_dir=EVAL_REQUESTS_PATH_BACKEND, | |
) | |
pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap)) | |
return | |
# next, check to see who made the last commit to this repo - keep track of that. One person shouldn't commit more | |
# than 4 models in one month. | |
commits = API.list_repo_commits(eval_request.model, revision=eval_request.revision) | |
users = commits[0].authors | |
for user in users: | |
if user in requests_seen and len(requests_seen[user]) >= 4: | |
set_eval_request( | |
api=API, | |
eval_request=eval_request, | |
set_to_status=FAILED_STATUS, | |
hf_repo=QUEUE_REPO, | |
local_dir=EVAL_REQUESTS_PATH_BACKEND, | |
) | |
pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user)) | |
return | |
if not user in requests_seen: | |
requests_seen[user] = [] | |
requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision)) | |
requests_seen[counter_key] += 1 | |
set_requests_seen( | |
api=API, | |
requests_seen=requests_seen, | |
hf_repo=QUEUE_REPO, | |
local_dir=EVAL_REQUESTS_PATH_BACKEND | |
) | |
set_eval_request( | |
api=API, | |
eval_request=eval_request, | |
set_to_status=RUNNING_STATUS, | |
hf_repo=QUEUE_REPO, | |
local_dir=EVAL_REQUESTS_PATH_BACKEND, | |
) | |
run_evaluation( | |
eval_request=eval_request, | |
task_names=TASKS_LIGHTEVAL, | |
local_dir=EVAL_RESULTS_PATH_BACKEND, | |
batch_size=25, | |
accelerator=ACCELERATOR, | |
region=REGION, | |
vendor=VENDOR, | |
instance_size=instance_size, | |
instance_type=instance_type, | |
limit=LIMIT | |
) | |
if __name__ == "__main__": | |
run_auto_eval() |