|
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login |
|
from huggingface_hub.utils import RepositoryNotFoundError as deneme |
|
from openllm import * |
|
import gradio as gr |
|
import requests |
|
import pandas as pd |
|
|
|
api = HfApi() |
|
fs = HfFileSystem() |
|
|
|
data = get_json_format_data() |
|
finished_models = get_datas(data) |
|
df = pd.DataFrame(finished_models) |
|
|
|
|
|
def search(df, value): |
|
result_df = df[df["Model"] == value] |
|
return result_df.iloc[0].to_dict() if not result_df.empty else None |
|
|
|
|
|
def get_details_url(repo): |
|
author, model = repo.split("/") |
|
return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}" |
|
|
|
|
|
def get_eval_results(repo): |
|
results = search(df, repo) |
|
|
|
text = f""" |
|
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) |
|
Detailed results can be found [here]({get_details_url(repo)}) |
|
|
|
| Metric | Value | |
|
|-----------------------|---------------------------| |
|
| Avg. | {results['Average ⬆️']} | |
|
| ARC (25-shot) | {results['ARC']} | |
|
| HellaSwag (10-shot) | {results['HellaSwag']} | |
|
| MMLU (5-shot) | {results['MMLU']} | |
|
| TruthfulQA (0-shot) | {results['TruthfulQA']} | |
|
| Winogrande (5-shot) | {results['Winogrande']} | |
|
| GSM8K (5-shot) | {results['GSM8K']} | |
|
| DROP (3-shot) | {results['DROP']} | |
|
""" |
|
return text |
|
|
|
|
|
desc = """ |
|
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr |
|
|
|
The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card. |
|
|
|
If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions |
|
""" |
|
|
|
|
|
def commit(hf_token, repo): |
|
try: |
|
try: |
|
readme_text = fs.read_text(f"{repo}/README.md") + get_eval_results(repo) |
|
except: |
|
readme_text = get_eval_results(repo) |
|
|
|
liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] |
|
commit = (create_commit(repo_id=repo, operations=liste, commit_message=f"Adding Evaluation Results", commit_description=desc, repo_type="model", create_pr=True).__dict__['pr_url']) |
|
return commit |
|
except: |
|
return "Error" |
|
|
|
|
|
demo = gr.Interface(fn=commit, inputs=["text", "text"], outputs="text") |
|
|
|
demo.launch() |
