Spaces:
Runtime error
Runtime error
File size: 2,568 Bytes
8ea42fc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 |
# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
from dataclasses import dataclass
import plotly.graph_objects as go
from transformers import AutoConfig
# These classes are for user facing column names, to avoid having to change them
# all around the code when a modif is needed
@dataclass
class ColumnContent:
name: str
type: str
displayed_by_default: bool
hidden: bool = False
def fields(raw_class):
return [
v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
]
@dataclass(frozen=True)
class AutoEvalColumn: # Auto evals column
model_type_symbol = ColumnContent("T", "str", True)
model = ColumnContent("Models", "markdown", True)
ARC = ColumnContent("ARC", "number", True)
HellaSwag = ColumnContent("HellaSwag", "number", True)
MMLU = ColumnContent("MMLU", "number", True)
TruthfulQA = ColumnContent("TruthfulQA", "number", True)
Winogrande = ColumnContent("Winogrande", "number", True)
GSM8K = ColumnContent("GSM8K", "number", True)
dummy = ColumnContent("Models", "str", True)
def model_hyperlink(link, model_name):
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def make_clickable_names(df):
df["Models"] = df.apply(
lambda row: model_hyperlink(row["Links"], row["Models"]), axis=1
)
return df
def styled_error(error):
return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
def styled_warning(warn):
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
def styled_message(message):
return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
def has_no_nan_values(df, columns):
return df[columns].notna().all(axis=1)
def has_nan_values(df, columns):
return df[columns].isna().any(axis=1)
def is_model_on_hub(model_name: str, revision: str) -> bool:
try:
AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
return True, None
except ValueError:
return (
False,
"needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
)
except Exception as e:
print(f"Could not get the model config from the hub.: {e}")
return False, "was not found on hub!"
|