|
import os |
|
import pandas as pd |
|
import requests |
|
import huggingface_hub |
|
import gradio as gr |
|
|
|
data = pd.read_csv("data.csv") |
|
webhook_url = os.environ.get("WEBHOOK_URL") |
|
|
|
def filter_table(name, type, arch, license): |
|
tmp = data |
|
|
|
if name: |
|
tmp = tmp[tmp["Name"].str.contains(name)] |
|
if type: |
|
tmp = tmp[tmp["Type"].isin(type)] |
|
if arch: |
|
tmp = tmp[tmp["Architecture"].isin(arch)] |
|
if license: |
|
tmp = tmp[tmp["License"].isin(license)] |
|
|
|
tmp["Name"] = tmp["Name"].apply(lambda x: f'<a target="_blank" href="https://huggingface.co/{x}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{x}</a>') |
|
return tmp |
|
|
|
def submit_model(name): |
|
try: |
|
huggingface_hub.hf_hub_download(repo_id=name, filename="config.json") |
|
except huggingface_hub.utils._errors.EntryNotFoundError: |
|
return "# ERROR: Model does not have a config.json file!" |
|
except huggingface_hub.utils._errors.RepositoryNotFoundError: |
|
return "# ERROR: Model could not be found on the Hugging Face Hub!" |
|
except requests.exceptions.HTTPError: |
|
return "# ERROR: Network error while validating model. Please try again later." |
|
except Exception as e: |
|
print(e) |
|
return "ERROR: Unexpected error. Please try again later." |
|
|
|
try: |
|
result = requests.post(webhook_url, json={"content":name}) |
|
except requests.exceptions.HTTPError: |
|
return "# ERROR: Network error while contacting queue. Please try again in a few minutes." |
|
except Exception as e: |
|
print(e) |
|
return "ERROR: Unexpected error. Please try again later." |
|
|
|
return "# SUCCESS: Please wait up to 24 hours for your model to be added to the queue." |
|
|
|
with gr.Blocks() as demo: |
|
gr.HTML('<style>.tab-buttons button{font-size:1.3em}</style><h1 style="text-align:center">Subquadratic LLM Leaderboard</h1>') |
|
|
|
with gr.Tabs(elem_classes="tab-buttons") as tabs: |
|
with gr.Tab("🏅 LLM Benchmark"): |
|
with gr.Row(): |
|
namefilter = model_name = gr.Textbox(max_lines=1, label="Search by model name and hit Enter") |
|
typefilter = gr.CheckboxGroup(label="Filter by model type", choices=list(data["Type"].unique()), value=[n for n in data["Type"].unique() if n not in ["Pending"]]) |
|
archfilter = gr.CheckboxGroup(label="Filter by model architecture", choices=list(data["Architecture"].unique()), value=list(data["Architecture"].unique())) |
|
lcnsfilter = gr.CheckboxGroup(label="Filter by model license", choices=list(data["License"].unique()), value=list(data["License"].unique())) |
|
|
|
table = gr.Dataframe(datatype="markdown") |
|
|
|
namefilter.submit(filter_table, [namefilter,typefilter,archfilter,lcnsfilter], table) |
|
|
|
for filter in [typefilter, archfilter, lcnsfilter]: |
|
filter.input(filter_table, [namefilter,typefilter,archfilter,lcnsfilter], table) |
|
|
|
demo.load(fn=filter_table, inputs=[namefilter,typefilter,archfilter,lcnsfilter], outputs=table) |
|
|
|
with gr.Tab("📝 About"): |
|
gr.Markdown(""" |
|
The **Subquadratic LLM Leaderboard** evaluates LLMs with subquadratic architectures (ie RWKV & Mamba) with the goal of providing open evaluation results while the architectures themselves are pending inclusion in 🤗 Transformers. |
|
The metrics are the same as the Open LLM Leaderboard: ARC 25-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA zeroshot, Winogrande 5-shot, and GSM8K 5-shot. |
|
This leaderboard is maintained by Devin Gulliver and is still under construction, check back regularly for further improvements! |
|
""") |
|
|
|
with gr.Tab("🚀 Submit here!"): |
|
with gr.Group(): |
|
with gr.Row(): |
|
model_name = gr.Textbox(max_lines=1, label="Model Name", scale=4) |
|
submit = gr.Button("Submit", variant="primary", scale=0) |
|
|
|
output = gr.Markdown("Enter a public HF repo id, then hit Submit to add it to the evaluation queue.") |
|
|
|
submit.click(fn=submit_model, inputs=model_name, outputs=output) |
|
|
|
demo.launch() |
