File size: 2,487 Bytes
1982aac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login
from huggingface_hub.utils import RepositoryNotFoundError as deneme
from openllm import *
import gradio as gr
import requests
import pandas as pd
api = HfApi()
fs = HfFileSystem()
data = get_json_format_data()
finished_models = get_datas(data)
df = pd.DataFrame(finished_models)
def search(df, value):
result_df = df[df["Model"] == value]
return result_df.iloc[0].to_dict() if not result_df.empty else None
def get_details_url(repo):
author, model = repo.split("/")
return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
def get_eval_results(repo):
results = search(df, repo)
text = f"""
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here]({get_details_url(repo)})
| Metric | Value |
|-----------------------|---------------------------|
| Avg. | {results['Average ⬆️']} |
| ARC (25-shot) | {results['ARC']} |
| HellaSwag (10-shot) | {results['HellaSwag']} |
| MMLU (5-shot) | {results['MMLU']} |
| TruthfulQA (0-shot) | {results['TruthfulQA']} |
| Winogrande (5-shot) | {results['Winogrande']} |
| GSM8K (5-shot) | {results['GSM8K']} |
| DROP (3-shot) | {results['DROP']} |
"""
return text
desc = """
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
"""
def commit(hf_token, repo):
try:
try: # check if there is a readme already
readme_text = fs.read_text(f"{repo}/README.md") + get_eval_results(repo)
except:
readme_text = get_eval_results(repo)
liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
commit = (create_commit(repo_id=repo, operations=liste, commit_message=f"Adding Evaluation Results", commit_description=desc, repo_type="model", create_pr=True).__dict__['pr_url'])
return commit
except: # unexpected error
return "Error"
demo = gr.Interface(fn=commit, inputs=["text", "text"], outputs="text")
demo.launch() |