import os from huggingface_hub import CommitOperationAdd, create_commit, RepoUrl from huggingface_hub import EvalResult, ModelCard from huggingface_hub.repocard_data import eval_results_to_model_index import time from pytablewriter import MarkdownTableWriter import gradio as gr from openllm import get_json_format_data, get_datas import pandas as pd import traceback from huggingface_hub import HfApi BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN') #data = get_json_format_data() #finished_models = get_datas(data) #df = pd.DataFrame(finished_models) df = None source_name = "Open Portuguese LLM Leaderboard" default_pull_request_title = "Adding the Open Portuguese LLM Leaderboard Evaluation Results" desc = """ This is an automated PR created with https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard The purpose of this PR is to add evaluation results from the [🚀 Open Portuguese LLM Leaderboard](https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard) to your model card. If you encounter any issues, please report them to https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard/discussions """ def search(query_df, value): global df if df is None and query_df is None: data = get_json_format_data() finished_models = get_datas(data) df = pd.DataFrame(finished_models) if query_df is not None: df = query_df result_df = df[df["Model Name"] == value] return result_df.iloc[0].to_dict() if not result_df.empty else None def get_details_url(repo): #author, model = repo.split("/") return f"https://huggingface.co/datasets/eduagarcia-temp/llm_pt_leaderboard_raw_results/tree/main/{repo}" def get_query_url(repo): return f"https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query={repo}" def get_task_summary(results): return { "ENEM": {"dataset_type":"eduagarcia/enem_challenge", "dataset_name":"ENEM Challenge (No Images)", "metric_type":"acc", "metric_value":results["ENEM"], "dataset_config": None, "dataset_split":"train", "dataset_revision":None, "dataset_args":{"num_few_shot": 3}, "metric_name":"accuracy" }, "BLUEX": {"dataset_type":"eduagarcia-temp/BLUEX_without_images", "dataset_name":"BLUEX (No Images)", "metric_type":"acc", "metric_value":results["BLUEX"], "dataset_config": None, "dataset_split":"train", "dataset_revision":None, "dataset_args":{"num_few_shot": 3}, "metric_name":"accuracy" }, "OAB Exams": {"dataset_type":"eduagarcia/oab_exams", "dataset_name":"OAB Exams", "metric_type":"acc", "metric_value":results["OAB Exams"], "dataset_config": None, "dataset_split":"train", "dataset_revision":None, "dataset_args":{"num_few_shot": 3}, "metric_name":"accuracy" }, "ASSIN2 RTE": {"dataset_type":"assin2", "dataset_name":"Assin2 RTE", "metric_type":"f1_macro", "metric_value":results["ASSIN2 RTE"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 15}, "metric_name":"f1-macro" }, "ASSIN2 STS": {"dataset_type":"eduagarcia/portuguese_benchmark", "dataset_name":"Assin2 STS", "metric_type":"pearson", "metric_value":results["ASSIN2 STS"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 15}, "metric_name":"pearson" }, "FAQUAD NLI": {"dataset_type":"ruanchaves/faquad-nli", "dataset_name":"FaQuAD NLI", "metric_type":"f1_macro", "metric_value":results["FAQUAD NLI"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 15}, "metric_name":"f1-macro" }, "HateBR": {"dataset_type":"ruanchaves/hatebr", "dataset_name":"HateBR Binary", "metric_type":"f1_macro", "metric_value":results["HateBR"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 25}, "metric_name":"f1-macro" }, "PT Hate Speech": {"dataset_type":"hate_speech_portuguese", "dataset_name":"PT Hate Speech Binary", "metric_type":"f1_macro", "metric_value":results["PT Hate Speech"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 25}, "metric_name":"f1-macro" }, "tweetSentBR": {"dataset_type":"eduagarcia/tweetsentbr_fewshot", "dataset_name":"tweetSentBR", "metric_type":"f1_macro", "metric_value":results["tweetSentBR"], "dataset_config": None, "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 25}, "metric_name":"f1-macro" } } def get_eval_results(repo): results = search(df, repo) task_summary = get_task_summary(results) md_writer = MarkdownTableWriter() md_writer.headers = ["Metric", "Value"] md_writer.value_matrix = [["Average", f"**{results['Average ⬆️']}**"]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()] text = f""" # Open Portuguese LLM Leaderboard Evaluation Results Detailed results can be found [here]({get_details_url(repo)}) and on the [🚀 Open Portuguese LLM Leaderboard](https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard) {md_writer.dumps()} """ return text def get_edited_yaml_readme(repo, token: str | None): card = ModelCard.load(repo, token=token) results = search(df, repo) common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": source_name, "source_url": get_query_url(repo)} tasks_results = get_task_summary(results) if not card.data['eval_results']: # No results reported yet, we initialize the metadata card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()]) else: # We add the new evaluations for task in tasks_results.values(): cur_result = EvalResult(**task, **common) if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']): continue card.data['eval_results'].append(cur_result) return str(card) def pr_already_exists(repo, token: str | None = None): card = ModelCard.load(repo, token=token) if 'eval_results' in card.data and card.data['eval_results']: for x in card.data['eval_results']: if x.source_name == source_name: return True if 'Open Portuguese LLM Leaderboard' in card.content: return True if 'Open PT LLM Leaderboard' in card.content: return True api = HfApi(token=token) for x in api.get_repo_discussions(repo): if x.title == default_pull_request_title: return True if x.author == "leaderboard-pt-pr-bot": return True if x.author == "eduagarcia" and x.is_pull_request: return True return False def commit(repo, pr_number=None, message=default_pull_request_title, oauth_token: gr.OAuthToken | None = None, check_if_pr_exists=False): # specify pr number if you want to edit it, don't if you don't want if oauth_token is None: gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.") token = BOT_HF_TOKEN elif oauth_token.expires_at < time.time(): raise gr.Error("Token expired. Logout and try again.") else: token = oauth_token.token if repo.startswith("https://huggingface.co/"): try: repo = RepoUrl(repo).repo_id except Exception: raise gr.Error(f"Not a valid repo id: {str(repo)}") if check_if_pr_exists or token == BOT_HF_TOKEN: if pr_already_exists(repo, token): return "PR already exists, Login to make a duplicate PR" edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True} try: try: # check if there is a readme already readme_text = get_edited_yaml_readme(repo, token=token).rstrip() + '\n\n' + get_eval_results(repo) except Exception as e: if "Repo card metadata block was not found." in str(e): # There is no readme readme_text = get_edited_yaml_readme(repo, token=token) else: traceback.print_exc() print(f"Something went wrong: {e}") liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url) return commit except Exception as e: if "Discussions are disabled for this repo" in str(e): return "Discussions disabled" elif "Cannot access gated repo" in str(e): return "Gated repo" elif "Repository Not Found" in str(e): return "Repository Not Found" else: return e if __name__ == "__main__": print(get_eval_results("Qwen/Qwen1.5-72B-Chat"))