first

app.py

@@ -0,0 +1,295 @@
+"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space."""
+
+import ast
+import argparse
+import glob
+import pickle
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import pandas as pd
+
+
+def make_default_md():
+
+    leaderboard_md = f"""
+# 🏆 CZ-EVAL Leaderboard
+[Developer](https://me.hynky.name/) | [Twitter](https://twitter.com/HKydlicek)
+
+CZ-EVAL is a evaluation leadboard of Tasks in Czech for LLMs.
+
+It's evaluated on following datasets:
+
+- Math Problems Understanding [Klokan-QA](https://huggingface.co/datasets/hynky/klokan-qa)
+- Reasoning and General Knowledge [TSP-QA](https://huggingface.co/datasets/hynky/tsp-qa)
+
+💻 Code: The evaluation code can be found at [hynky1999/LLM-Eval](https://github.com/hynky1999/LLM-Eval). Model inference is done using [Open-Router](https://openrouter.ai/) or on cloud using [Modal Labs](https://modal.com/).
+"""
+    return leaderboard_md
+
+
+def make_arena_leaderboard_md(arena_df):
+    total_models = len(arena_df)
+
+    leaderboard_md = f"""
+Total #models: **{total_models}**. Last updated: Feb 15, 2024.
+"""
+    return leaderboard_md
+
+
+def make_full_leaderboard_md(elo_results):
+    leaderboard_md = f"""
+Three benchmarks are displayed: **Arena Elo**, **MT-Bench** and **MMLU**.
+- [Klokan-QA](https://huggingface.co/datasets/hynky/klokan-qa) - Mathematical competitions dataset
+- [TSP](https://huggingface.co/datasets/hynky/TSP) - Comprehensive dataset of 
+
+"""
+    return leaderboard_md
+
+
+# Combine all category accuracies into a single DataFrame
+
+
+def plot_spider(df, title):
+    categories = df.columns.tolist()[1:]
+    categories = [
+        *categories,
+        categories[0],
+    ]  # Ensure the graph is circular by appending the start to the end
+    colors = [
+        "#1f77b4",  # muted blue
+        "#ff7f0e",  # safety orange
+        "#2ca02c",  # cooked asparagus green
+        "#d62728",  # brick red
+        "#9467bd",  # muted purple
+        "#8c564b",  # chestnut brown
+        "#e377c2",  # raspberry yogurt pink
+        "#7f7f7f",  # middle gray
+        "#bcbd22",  # curry yellow-green
+        "#17becf",  # blue-teal
+    ]
+
+    # Setting for 1000x1000
+    fig_1000 = go.Figure()
+
+    for i, (idx, row) in enumerate(df.iterrows()):
+        name = row[0]
+        row = row.tolist()[1:]
+        row = row + [
+            row[0]
+        ]  # Ensure the graph is circular by appending the start to the end
+        color = colors[i]
+        fig_1000.add_trace(
+            go.Scatterpolar(
+                r=row,
+                theta=categories,
+                opacity=0.4,
+                name=name,
+                line=dict(
+                    color=color, width=4
+                ),  # Adjust line width for better visibility
+            )
+        )
+
+    fig_1000.update_layout(
+        width=600,
+        height=628,
+        polar=dict(
+            angularaxis=dict(
+                gridwidth=2,  # Increase line width for better visibility
+                rotation=90,
+                direction="clockwise",
+            ),
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],
+                angle=45,
+                tickangle=45,
+                tickvals=[0, 25, 50, 75, 100],
+                ticktext=["0%", "25%", "50%", "75%", "100%"],
+            ),
+        ),
+        title_text=title,
+        title_x=0.5,
+        title_y=0.97,
+        title_xanchor="center",
+        title_yanchor="top",
+        title_font_size=24,
+        title_font_color="#333333",
+        font=dict(family="Arial", size=16, color="#333333"),
+        legend=dict(
+            orientation="h", yanchor="bottom", y=-0.45, xanchor="center", x=0.5
+        ),
+    )
+    return fig_1000
+
+
+def openrouter_hyperlink(model_name):
+    return f'<a target="_blank" href="https://openrouter.ai/models/{model_name}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def get_full_table(model_table_df):
+    num_cols = ["klokan", "culture", "analytical", "critical", "verbal"]
+    # Multiply by 100 and round to 2 decimals
+    # Add average
+    model_table_df["average"] = model_table_df[num_cols].mean(axis=1)
+    model_table_df[num_cols + ["average"]] = model_table_df[
+        num_cols + ["average"]
+    ].apply(lambda x: round(x * 100, 2))
+
+    # Sort and add rank
+    model_table_df.sort_values(by="average", ascending=False, inplace=True)
+    model_table_df.insert(0, "rank", np.arange(1, len(model_table_df) + 1))
+
+    # Add link
+    model_table_df["model_name"] = model_table_df["model_name"].apply(
+        lambda x: openrouter_hyperlink(x)
+    )
+
+    model_table_df.rename(
+        columns={
+            "model_name": "🤖 Model",
+            "klokan": "🧮 Klokan-QA",
+            "culture": "🌍 TSP-Culture",
+            "analytical": "🔍 TSP-Analytical",
+            "critical": "💡 TSP-Critical",
+            "verbal": "📖 TSP-Verbal",
+            "average": "📊 Average",
+        },
+        inplace=True,
+    )
+
+    return model_table_df
+
+
+def build_leaderboard_tab(leaderboard_table_file, klokan_table_file, tsp_table_file):
+
+    results = pd.read_csv(leaderboard_table_file)
+    results = get_full_table(results)
+    # p1, p2 = get_grafs(pd.read_json(klokan_table_file), pd.read_json(tsp_table_file))
+    default_md = make_default_md()
+
+    md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
+    with gr.Tabs() as tabs:
+        # arena table
+        with gr.Tab("CZ-EVAL Leaderboard", id=0):
+            md = make_arena_leaderboard_md(results)
+            gr.Markdown(md, elem_id="leaderboard_markdown")
+            gr.Dataframe(
+                datatype=[
+                    "str",
+                    "markdown",
+                    "number",
+                    "number",
+                    "number",
+                    "number",
+                    "number",
+                    "number",
+                    "str",
+                    "str",
+                    "str",
+                ],
+                value=results,
+                elem_id="arena_leaderboard_dataframe",
+                height=700,
+                column_widths=[
+                    50,
+                    200,
+                    120,
+                    100,
+                    100,
+                    150,
+                    150,
+                    100,
+                    150,
+                    150,
+                    150,
+                ],
+                wrap=True,
+            )
+
+    p1 = plot_spider(pd.read_csv(klokan_table_file), "Klokan-QA - Acurracy")
+    p2 = plot_spider(pd.read_csv(tsp_table_file), "TSP - Accuracy")
+
+    gr.Markdown(
+        f"""## More Statistics for CZ-EVAL\n
+Below are figures for more statistics.
+""",
+        elem_id="leaderboard_markdown",
+    )
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+                "#### Figure 1: Performance of models on Klokan-QA per difficulty"
+            )
+            plot_1 = gr.Plot(p1, show_label=False)
+        with gr.Column():
+            gr.Markdown("#### Figure 2: Performance of models on TSP dataset")
+            plot_2 = gr.Plot(p2, show_label=False)
+
+    return [md_1, plot_1, plot_2]
+
+
+block_css = """
+#notice_markdown {
+    font-size: 104%
+}
+#notice_markdown th {
+    display: none;
+}
+#notice_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_markdown {
+    font-size: 104%
+}
+#leaderboard_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_dataframe td {
+    line-height: 0.1em;
+}
+footer {
+    display:none !important
+}
+.image-container {
+    display: flex;
+    align-items: center;
+    padding: 1px;
+}
+.image-container img {
+    margin: 0 30px;
+    height: 20px;
+    max-height: 100%;
+    width: auto;
+    max-width: 20%;
+}
+"""
+
+
+def build_demo(leadboard_table, klokan_table, tsp_table):
+    text_size = gr.themes.sizes.text_lg
+
+    with gr.Blocks(
+        title="CZ-EVAL Leaderboard",
+        theme=gr.themes.Base(text_size=text_size),
+        css=block_css,
+    ) as demo:
+        leader_components = build_leaderboard_tab(
+            leadboard_table, klokan_table, tsp_table
+        )
+    return demo
+
+
+demo = build_demo(
+    leadboard_table="./leaderboard/table.csv",
+    klokan_table="./leaderboard/klokan.csv",
+    tsp_table="./leaderboard/tsp.csv",
+)
+
+if __name__ == "__main__":
+    demo.launch()

leaderboard/klokan.csv

@@ -0,0 +1,6 @@
+,Elementary 2-3,Elementary 4-5,Elementary 6-7,Elementary 8-9,High School 1-2,High School 3-4
+anthropic/claude-2.1,43.96551724137931,50.35971223021583,39.87730061349693,39.75155279503105,33.33333333333333,14.772727272727273
+google/gemini-pro,25.0,28.05755395683453,22.699386503067483,20.496894409937887,24.691358024691358,19.318181818181817
+mistralai/mixtral-8x7b-instruct,34.48275862068966,25.899280575539567,25.766871165644172,25.465838509316768,20.98765432098765,19.318181818181817
+openai/gpt-3.5-turbo,37.06896551724138,41.007194244604314,33.74233128834356,29.81366459627329,26.543209876543212,17.045454545454543
+openai/gpt-4-1106-preview,66.37931034482759,62.589928057553955,50.306748466257666,40.993788819875775,32.71604938271605,36.36363636363637

leaderboard/table.csv

@@ -0,0 +1,6 @@
+model_name,analytical,critical,culture,verbal,klokan
+anthropic/claude-2.1,0.3804034582132565,0.6449912126537786,0.7981770833333334,0.6336336336336337,0.3823884197828709
+google/gemini-pro,0.2680115273775216,0.5992970123022847,0.7825520833333334,0.5765765765765766,0.23522316043425814
+mistralai/mixtral-8x7b-instruct,0.24495677233429394,0.4833040421792619,0.6432291666666666,0.36936936936936937,0.25331724969843183
+openai/gpt-3.5-turbo,0.27761767531219983,0.46572934973637964,0.6822916666666666,0.4084084084084084,0.3148371531966224
+openai/gpt-4-1106-preview,0.4793467819404419,0.7662565905096661,0.9166666666666666,0.7207207207207207,0.47889022919179736

leaderboard/tsp.csv

@@ -0,0 +1,6 @@
+,Analytical,Critical,Cultural,Verbal
+anthropic/claude-2.1,38.04034582132565,64.49912126537785,79.81770833333334,63.36336336336337
+google/gemini-pro,26.801152737752158,59.929701230228474,78.25520833333334,57.65765765765766
+mistralai/mixtral-8x7b-instruct,24.495677233429394,48.33040421792619,64.32291666666666,36.93693693693694
+openai/gpt-3.5-turbo,27.761767531219984,46.57293497363796,68.22916666666666,40.84084084084084
+openai/gpt-4-1106-preview,47.93467819404419,76.6256590509666,91.66666666666666,72.07207207207207

requirements.txt

@@ -0,0 +1,3 @@
+gradio==4.19.1
+pandas==2.2.0
+plotly==5.19.0