Spaces:
Runtime error
Runtime error
File size: 7,635 Bytes
a74897e 1982aac b73db7d 1982aac b73db7d 2e83d59 1982aac b73db7d 1982aac b73db7d 63601dd 1982aac b73db7d 1982aac b73db7d 1982aac b73db7d 1982aac b73db7d 1982aac b73db7d 1982aac b73db7d 007681b b73db7d 1982aac b73db7d 1982aac b73db7d 1982aac b73db7d 1982aac 52101ad 250d5a2 52101ad 250d5a2 52101ad 1982aac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 |
import os
os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login
from huggingface_hub import ModelCardData, EvalResult, ModelCard
from huggingface_hub.repocard_data import eval_results_to_model_index
from huggingface_hub.repocard import RepoCard
from openllm import get_json_format_data, get_datas
from tqdm import tqdm
import time
import requests
import pandas as pd
from pytablewriter import MarkdownTableWriter
import gradio as gr
api = HfApi()
fs = HfFileSystem()
data = get_json_format_data()
finished_models = get_datas(data)
df = pd.DataFrame(finished_models)
def search(df, value):
result_df = df[df["Model"] == value]
return result_df.iloc[0].to_dict() if not result_df.empty else None
def get_details_url(repo):
author, model = repo.split("/")
return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
def get_query_url(repo):
return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
desc = """
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
"""
def get_task_summary(results):
return {
"ARC":
{"dataset_type":"ai2_arc",
"dataset_name":"AI2 Reasoning Challenge (25-Shot)",
"metric_type":"acc_norm",
"metric_value":results["ARC"],
"dataset_config":"ARC-Challenge",
"dataset_split":"test",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 25},
"metric_name":"normalized accuracy"
},
"HellaSwag":
{"dataset_type":"hellaswag",
"dataset_name":"HellaSwag (10-Shot)",
"metric_type":"acc_norm",
"metric_value":results["HellaSwag"],
"dataset_config":None,
"dataset_split":"validation",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 10},
"metric_name":"normalized accuracy"
},
"MMLU":
{
"dataset_type":"cais/mmlu",
"dataset_name":"MMLU (5-Shot)",
"metric_type":"acc",
"metric_value":results["MMLU"],
"dataset_config":"all",
"dataset_split":"test",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
},
"TruthfulQA":
{
"dataset_type":"truthful_qa",
"dataset_name":"TruthfulQA (0-shot)",
"metric_type":"mc2",
"metric_value":results["TruthfulQA"],
"dataset_config":"multiple_choice",
"dataset_split":"validation",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 0},
"metric_name":None
},
"Winogrande":
{
"dataset_type":"winogrande",
"dataset_name":"Winogrande (5-shot)",
"metric_type":"acc",
"metric_value":results["Winogrande"],
"dataset_config":"winogrande_xl",
"dataset_split":"validation",
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
},
"GSM8K":
{
"dataset_type":"gsm8k",
"dataset_name":"GSM8k (5-shot)",
"metric_type":"acc",
"metric_value":results["GSM8K"],
"dataset_config":"main",
"dataset_split":"test",
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
}
}
def get_eval_results(repo):
results = search(df, repo)
task_summary = get_task_summary(results)
md_writer = MarkdownTableWriter()
md_writer.headers = ["Metric", "Value"]
md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_short_name"], v["metric_value"]] for v in task_summary.values()]
text = f"""
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here]({get_details_url(repo)})
{md_writer.dumps()}
"""
return text
def get_edited_yaml_readme(repo):
card = ModelCard.load(repo)
results = search(df, repo)
common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}
tasks_results = get_task_summary(results)
if not card.data['eval_results']: # No results reported yet, we initialize the metadata
card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
else: # We add the new evaluations
for task in tasks_results.values():
cur_result = EvalResult(**task, **common)
if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
continue
card.data['eval_results'].append(cur_result)
return str(card)
def commit(hf_token, repo, pr_number=None, message="Adding Evaluation Results"): # specify pr number if you want to edit it, don't if you don't want
login(hf_token)
edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
try:
try: # check if there is a readme already
readme_text = get_edited_yaml_readme(repo) + get_eval_results(repo)
except Exception as e:
if "Repo card metadata block was not found." in str(e): # There is no readme
readme_text = get_edited_yaml_readme(repo)
else:
print(f"Something went wrong: {e}")
liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
commit = (create_commit(repo_id=repo, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)
return commit
except Exception as e:
if "Discussions are disabled for this repo" in str(e):
return "Discussions disabled"
elif "Cannot access gated repo" in str(e):
return "Gated repo"
elif "Repository Not Found" in str(e):
return "Repository Not Found"
else:
return e
gradio_title="🧐 Open LLM Leaderboard Results PR Opener"
gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card.
## 💭 What Does This Tool Do:
- This tool adds the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) result of your model at the end of your model card.
- This tool also adds evaluation results as your model's metadata to showcase the evaluation results as a widget.
## 🛠️ Backend
The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
## 🤝 Acknowledgements
- Special thanks to [Clémentine Fourrier (clefourrier)](https://huggingface.co/clefourrier) for her help and contributions to the code.
- Special thanks to [Lucain Pouget (Wauplin)](https://huggingface.co/Wauplin) for assisting with the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
"""
demo = gr.Interface(title=gradio_title, description=gradio_desc, fn=commit, inputs=["text", "text"], outputs="text")
demo.launch() |