Spaces:
Runtime error
Runtime error
import os | |
import time | |
os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py") | |
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, RepoUrl | |
from huggingface_hub import ModelCardData, EvalResult, ModelCard | |
from huggingface_hub.repocard_data import eval_results_to_model_index | |
from huggingface_hub.repocard import RepoCard | |
from openllm import get_json_format_data, get_datas | |
from tqdm import tqdm | |
import time | |
import requests | |
import pandas as pd | |
from pytablewriter import MarkdownTableWriter | |
import threading | |
import gradio as gr | |
from gradio_space_ci import enable_space_ci | |
enable_space_ci() | |
HF_TOKEN = os.getenv('HF_TOKEN') | |
BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN') | |
api = HfApi() | |
fs = HfFileSystem() | |
data = get_json_format_data() | |
finished_models = get_datas(data) | |
df = pd.DataFrame(finished_models) | |
def refresh(how_much=3600): # default to 1 hour | |
global data, finished_models, df | |
time.sleep(how_much) | |
try: | |
data = get_json_format_data() | |
finished_models = get_datas(data) | |
df = pd.DataFrame(finished_models) | |
except Exception as e: | |
print(f"Error while scraping leaderboard, trying again... {e]") | |
refresh(600) # 10 minutes if any error happens | |
def search(df, value): | |
result_df = df[df["Model"] == value] | |
return result_df.iloc[0].to_dict() if not result_df.empty else None | |
def get_details_url(repo): | |
author, model = repo.split("/") | |
return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}" | |
def get_query_url(repo): | |
return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}" | |
desc = """ | |
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr | |
The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card. | |
If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions | |
""" | |
def get_task_summary(results): | |
return { | |
"ARC": | |
{"dataset_type":"ai2_arc", | |
"dataset_name":"AI2 Reasoning Challenge (25-Shot)", | |
"metric_type":"acc_norm", | |
"metric_value":results["ARC"], | |
"dataset_config":"ARC-Challenge", | |
"dataset_split":"test", | |
"dataset_revision":None, | |
"dataset_args":{"num_few_shot": 25}, | |
"metric_name":"normalized accuracy" | |
}, | |
"HellaSwag": | |
{"dataset_type":"hellaswag", | |
"dataset_name":"HellaSwag (10-Shot)", | |
"metric_type":"acc_norm", | |
"metric_value":results["HellaSwag"], | |
"dataset_config":None, | |
"dataset_split":"validation", | |
"dataset_revision":None, | |
"dataset_args":{"num_few_shot": 10}, | |
"metric_name":"normalized accuracy" | |
}, | |
"MMLU": | |
{ | |
"dataset_type":"cais/mmlu", | |
"dataset_name":"MMLU (5-Shot)", | |
"metric_type":"acc", | |
"metric_value":results["MMLU"], | |
"dataset_config":"all", | |
"dataset_split":"test", | |
"dataset_revision":None, | |
"dataset_args":{"num_few_shot": 5}, | |
"metric_name":"accuracy" | |
}, | |
"TruthfulQA": | |
{ | |
"dataset_type":"truthful_qa", | |
"dataset_name":"TruthfulQA (0-shot)", | |
"metric_type":"mc2", | |
"metric_value":results["TruthfulQA"], | |
"dataset_config":"multiple_choice", | |
"dataset_split":"validation", | |
"dataset_revision":None, | |
"dataset_args":{"num_few_shot": 0}, | |
"metric_name":None | |
}, | |
"Winogrande": | |
{ | |
"dataset_type":"winogrande", | |
"dataset_name":"Winogrande (5-shot)", | |
"metric_type":"acc", | |
"metric_value":results["Winogrande"], | |
"dataset_config":"winogrande_xl", | |
"dataset_split":"validation", | |
"dataset_args":{"num_few_shot": 5}, | |
"metric_name":"accuracy" | |
}, | |
"GSM8K": | |
{ | |
"dataset_type":"gsm8k", | |
"dataset_name":"GSM8k (5-shot)", | |
"metric_type":"acc", | |
"metric_value":results["GSM8K"], | |
"dataset_config":"main", | |
"dataset_split":"test", | |
"dataset_args":{"num_few_shot": 5}, | |
"metric_name":"accuracy" | |
} | |
} | |
def get_eval_results(repo): | |
results = search(df, repo) | |
task_summary = get_task_summary(results) | |
md_writer = MarkdownTableWriter() | |
md_writer.headers = ["Metric", "Value"] | |
md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()] | |
text = f""" | |
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) | |
Detailed results can be found [here]({get_details_url(repo)}) | |
{md_writer.dumps()} | |
""" | |
return text | |
def get_edited_yaml_readme(repo, token: str | None): | |
card = ModelCard.load(repo, token=token) | |
results = search(df, repo) | |
common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"} | |
tasks_results = get_task_summary(results) | |
if not card.data['eval_results']: # No results reported yet, we initialize the metadata | |
card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()]) | |
else: # We add the new evaluations | |
for task in tasks_results.values(): | |
cur_result = EvalResult(**task, **common) | |
if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']): | |
continue | |
card.data['eval_results'].append(cur_result) | |
return str(card) | |
def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want | |
if oauth_token is None: | |
raise gr.Error("You must be logged in to open a PR. Click on 'Sign in with Huggingface' first.") | |
if oauth_token.expires_at < time.time(): | |
raise gr.Error("Token expired. Logout and try again.") | |
token = oauth_token.token | |
if repo.startswith("https://huggingface.co/"): | |
try: | |
repo = RepoUrl(repo).repo_id | |
except Exception: | |
raise gr.Error(f"Not a valid repo id: {str(repo)}") | |
edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True} | |
try: | |
try: # check if there is a readme already | |
readme_text = get_edited_yaml_readme(repo, token=token) + get_eval_results(repo) | |
except Exception as e: | |
if "Repo card metadata block was not found." in str(e): # There is no readme | |
readme_text = get_edited_yaml_readme(repo, token=token) | |
else: | |
print(f"Something went wrong: {e}") | |
liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] | |
commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url) | |
return commit | |
except Exception as e: | |
if "Discussions are disabled for this repo" in str(e): | |
return "Discussions disabled" | |
elif "Cannot access gated repo" in str(e): | |
return "Gated repo" | |
elif "Repository Not Found" in str(e): | |
return "Repository Not Found" | |
else: | |
return e | |
gradio_title="🧐 Open LLM Leaderboard Results PR Opener" | |
gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card. | |
## 💭 What Does This Tool Do: | |
- This tool adds the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) result of your model at the end of your model card. | |
- This tool also adds evaluation results as your model's metadata to showcase the evaluation results as a widget. | |
## 🛠️ Backend | |
The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api). | |
## 🤝 Acknowledgements | |
- Special thanks to [Clémentine Fourrier (clefourrier)](https://huggingface.co/clefourrier) for her help and contributions to the code. | |
- Special thanks to [Lucain Pouget (Wauplin)](https://huggingface.co/Wauplin) for assisting with the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api). | |
""" | |
with gr.Blocks() as demo: | |
gr.HTML(f"""<h1 align="center" id="space-title">{gradio_title}</h1>""") | |
gr.Markdown(gradio_desc) | |
with gr.Row(equal_height=False): | |
with gr.Column(): | |
model_id = gr.Textbox(label="Model ID or URL", lines=1) | |
gr.LoginButton() | |
with gr.Column(): | |
output = gr.Textbox(label="Output", lines=1) | |
gr.LogoutButton() | |
submit_btn = gr.Button("Submit", variant="primary") | |
submit_btn.click(commit, model_id, output) | |
threading.Thread(target=refresh).start() | |
demo.launch() |