from glob import glob import pandas as pd from huggingface_hub import snapshot_download import json from tqdm.auto import tqdm import os import traceback from functions import pr_already_exists, commit QUEUE_REPO = "eduagarcia-temp/llm_pt_leaderboard_requests" EVAL_REQUESTS_PATH = "./eval-queue/" def run_pr_worker(): snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30) for filepath in glob(os.path.join(EVAL_REQUESTS_PATH, '**/*.json'), recursive=True): with open(filepath, 'r') as f: model_data = json.load(f) if model_data['status'] != 'FINISHED': continue if 'main_language' not in model_data: continue if model_data['main_language'] != "Portuguese": continue if model_data['result_metrics_average'] < 0.25: continue if 'ptpt' in model_data['model']: continue try: if not pr_already_exists(model_data['model']): print(f"Opening PR for {model_data['model']}") commit(model_data['model'], check_if_pr_exists=True) except Exception as e: traceback.print_exc() print(f"Error on {model_data['model']}: {str(e)}") if __name__ == "__main__": run_pr_worker()