SEED Bench

app.py

@@ -0,0 +1,263 @@
+
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import json
+import pdb
+import tempfile
+
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+global data_component, filter_component
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def prediction_analyse(prediction_content):
+    # pdb.set_trace()
+    predictions = prediction_content.split("\n")
+
+    # 读取 ground_truth JSON 文件
+    with open("./file/SEED-Bench.json", "r") as file:
+        ground_truth_data = json.load(file)["questions"]
+
+    # 将 ground_truth 数据转换为以 question_id 为键的字典
+    ground_truth = {item["question_id"]: item for item in ground_truth_data}
+
+    # 初始化结果统计字典
+    results = {i: {"correct": 0, "total": 0} for i in range(1, 13)}
+
+    # 遍历 predictions，计算每个 question_type_id 的正确预测数和总预测数
+    for prediction in predictions:
+        prediction = json.loads(prediction)
+        question_id = prediction["question_id"]
+        gt_item = ground_truth[question_id]
+        question_type_id = gt_item["question_type_id"]
+
+        if prediction["prediction"] == gt_item["answer"]:
+            results[question_type_id]["correct"] += 1
+
+        results[question_type_id]["total"] += 1
+    
+    return results
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_type: str,
+    model_link: str,
+    LLM_type: str,
+    LLM_name_textbox: str,
+    Evaluation_dimension: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        content = input_file.decode("utf-8")
+        prediction = prediction_analyse(content)
+        each_task_accuracy = {i: round(prediction[i]["correct"] / prediction[i]["total"] * 100, 1) for i in range(1, 13)}
+
+        # count for average image\video\all
+        total_correct_image = sum(prediction[i]["correct"] for i in range(1, 10))
+        total_correct_video = sum(prediction[i]["correct"] for i in range(10, 13))
+
+        total_image = sum(prediction[i]["total"] for i in range(1, 10))
+        total_video = sum(prediction[i]["total"] for i in range(10, 13))
+
+        average_accuracy_image = round(total_correct_image / total_image * 100, 1)
+        average_accuracy_video = round(total_correct_video / total_video * 100, 1)
+        overall_accuracy = round((total_correct_image + total_correct_video) / (total_image + total_video) * 100, 1)
+
+        if LLM_type == 'other':
+            LLM_name = LLM_name_textbox
+        else:
+            LLM_name = LLM_type
+        # add new data
+        new_data = [
+            model_type, 
+            model_name_textbox, 
+            LLM_name, 
+            each_task_accuracy[1],
+            each_task_accuracy[2],
+            each_task_accuracy[3],
+            each_task_accuracy[4],
+            each_task_accuracy[5],
+            each_task_accuracy[6],
+            each_task_accuracy[7],
+            each_task_accuracy[8],
+            each_task_accuracy[9],
+            average_accuracy_image,
+            each_task_accuracy[10],
+            each_task_accuracy[11],
+            each_task_accuracy[12], 
+            average_accuracy_video, 
+            overall_accuracy]
+        # pdb.set_trace()
+        csv_data = pd.read_csv(CSV_DIR)
+        col = csv_data.shape[0]
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(CSV_DIR, index=False)
+    return 0
+
+def get_baseline_df():
+    df = pd.read_csv(CSV_DIR)
+    return df
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 SEED Benchmark", elem_id="seed-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                    ).style(show_copy_button=True)
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO,
+                value=TASK_INFO,
+                label="Select options",
+                interactive=True,
+            )
+
+            # 创建数据帧组件
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            def on_checkbox_group_change(selected_columns):
+                print("TEst")
+                # pdb.set_trace()
+                selected_columns = [item for item in TASK_INFO if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_baseline_df()[present_columns]
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                # data_component.update(value=updated_data, headers=updated_headers, datatype=update_datatype)
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+        
+                return filter_component.value
+
+            # 将复选框组关联到处理函数
+            checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
+
+        # table 2
+        with gr.TabItem("📝 About", elem_id="seed-benchmark-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="LLaMA-7B"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="LLaMA-7B"
+                    )
+                    model_type = gr.Dropdown(
+                        choices=[                         
+                            ModelType.PT.to_str(" : "),
+                            ModelType.FT.to_str(" : "),
+                            ModelType.IFT.to_str(" : "),
+                            ModelType.RL.to_str(" : "), 
+                        ], 
+                        label="Model type", 
+                        multiselect=False,
+                        value="LLaMA-7B",
+                        interactive=True,
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf"
+                    )
+
+                with gr.Column():
+                    
+                    LLM_type = gr.Dropdown(
+                        choices=["Vicuna-7B", "Flan-T5-XL", "LLaMA-7B", "other"],
+                        label="LLM type", 
+                        multiselect=False,
+                        value="LLaMA-7B",
+                        interactive=True,
+                    )
+                    LLM_name_textbox = gr.Textbox(
+                        label="LLM model (for other)",
+                        value="LLaMA-13B"
+                    )
+                    Evaluation_dimension = gr.Dropdown(
+                        choices=["All", "Image", "Video"],
+                        label="Evaluation dimension", 
+                        multiselect=False,
+                        value="All",
+                        interactive=True,
+                    )
+
+            with gr.Column():
+
+                input_file = gr.inputs.File(label = "Click to Upload a json File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_type,
+                        model_link,
+                        LLM_type,
+                        LLM_name_textbox,
+                        Evaluation_dimension,
+                    ],
+                    # outputs = submission_result,
+                )
+
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            get_baseline_df, outputs=data_component
+        )
+
+    # block.load(get_baseline_df, outputs=data_title)
+
+block.launch()

constants.py

@@ -0,0 +1,60 @@
+# this is .py for store constants 
+MODEL_INFO = ["Model Type", "Model", "Language Model"]
+TASK_INFO = ["Scene Understanding", "Instance Identity", "Instance Attributes", "Instance Localization", "Instance Counting", "Spatial Relation", "Instance Interaction", "Visual Reasoning", "Text Recognition", "Avg. Img", "Action Recognition", "Action Prediction", "Procedure Understanding", "Avg. Video", "Avg. All"]
+AVG_INFO = ["Avg. Img", "Avg. Video", "Avg. All"]
+DATA_TITILE_TYPE = ["markdown", "markdown", "markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"]
+CSV_DIR = "./file/result.csv"
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+DATA_NUM = [3158, 1831, 4649, 978, 2447, 657, 97, 331, 85, 1740, 2077, 1192]
+
+UNTUNED_MODEL_RESULTS = '''LLM & Flan-T5               & Flan-T5-XL &23.0 &29.0 &32.8 &31.8 &20.5 &31.8 &33.0 &18.2 &19.4 &23.2 &34.9 &25.4 \\
+         LLM                   & Vicuna                & Vicuna-7B  &23.4 &30.7 &29.7 &30.9 &30.8 &28.6 &29.8 &18.5 &13.4 &27.3 &34.5 &23.8 \\
+         LLM                   & LLaMA                 & LLaMA-7B   &26.3 &27.4 &26.2 &28.3 &25.1 &28.8 &19.2 &37.0 & 9.0 &33.0 &23.1 &26.2  \\
+         ImageLLM              & BLIP-2                & Flan-T5-XL &59.1 &53.9 &49.2 &42.3 &43.2 &36.7 &55.7 &45.6 &25.9 &32.6 &47.5 &24.0 \\
+         ImageLLM              & InstructBLIP          & Flan-T5-XL &60.3 &58.5 &63.4 &40.6 &58.4 &38.7 &51.6 &45.9 &25.9 &33.1 &49.1 &27.1 \\
+         ImageLLM              & InstructBLIP-Vicuna   & Vicuna-7B  &60.2 &58.9 &65.6 &43.6 &57.2 &40.3 &52.6 &47.7 &43.5 &34.5 &49.6 &23.1 \\
+         ImageLLM              & LLaVA                 & LLaMA-7B   &42.7 &34.9 &33.5 &28.4 &41.9 &30.8 &27.8 &46.8 &27.7 &29.7 &21.4 &19.1 \\
+         ImageLLM              & MiniGPT-4             & Flan-T5-XL &56.3 &49.2 &45.8 &37.9 &45.3 &32.6 &47.4 &57.1 &11.8 &38.2 &24.5 &27.1 \\
+         ImageLLM              & VPGTrans              & LLaMA-7B   &51.9 &44.1 &39.9 &36.1 &33.7 &36.4 &32.0 &53.2 &30.6 &39.5 &24.3 &31.9 \\
+         ImageLLM              & MultiModal-GPT        & LLaMA-7B   &43.6 &37.9 &31.5 &30.8 &27.3 &30.1 &29.9 &51.4 &18.8 &36.9 &25.8 &24.0 \\
+         ImageLLM              & Otter                 & LLaMA-7B   &44.9 &38.6 &32.2 &30.9 &26.3 &31.8 &32.0 &51.4 &31.8 &37.9 &27.2 &24.8 \\
+         ImageLLM              & OpenFlamingo          & LLaMA-7B   &43.9 &38.1 &31.3 &30.1 &27.3 &30.6 &29.9 &50.2 &20.0 &37.2 &25.4 &24.2 \\
+         ImageLLM              & LLaMA-Adapter V2      & LLaMA-7B   &45.2 &38.5 &29.3 &33.0 &29.7 &35.5 &39.2 &52.0 &24.7 &38.6 &18.5 &19.6 \\
+         ImageLLM              & GVT                   & Vicuna-7B  &41.7 &35.5 &31.8 &29.5 &36.2 &32.0 &32.0 &51.1 &27.1 &33.9 &25.4 &23.0 \\
+         ImageLLM              & mPLUG-Owl             & LLaMA-7B   &49.7 &45.3 &32.5 &36.7 &27.3 &32.7 &44.3 &54.7 &28.8 &26.7 &17.9 &26.5 \\
+         VideoLLM              & VideoChat             & Vicuna-7B  &47.1 &43.8 &34.9 &40.0 &32.8 &34.6 &42.3 &50.5 &17.7 &34.9 &36.4 &27.3 \\
+         VideoLLM              & Video-ChatGPT         & LLaMA-7B   &37.2 &31.4 &33.2 &28.4 &35.5 &29.5 &23.7 &42.3 &25.9 &27.6 &21.3 &21.1 \\
+         VideoLLM              & Valley                & LLaMA-13B  &39.3 &32.9 &31.6 &27.9 &24.2 &30.1 &27.8 &43.8 &11.8 &31.3 &23.2 &20.7 \\'''
+
+
+LEADERBORAD_INTRODUCTION = """# SEED-Bench Leaderboard
+    
+    Welcome to the leaderboard of the SEED-Bench! 🏆 
+    This is a community where participants create multimodal language models and action generation algorithms to generate API function calls based goals described in natural lanugage!
+    Please refer to [our paper](https://arxiv.org/abs/2307.16125) for more details.
+    """
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+        We use accurancy(%) as the primary evaluation metric for most tasks. 
+    """
+
+LEADERBORAD_INFO = """
+      Based on powerful Large Language Models (LLMs), recent generative Multimodal Large Language Models (MLLMs) have gained prominence as a pivotal research area, exhibiting remarkable capability for both comprehension and generation. 
+      In this work, we address the evaluation of generative comprehension in MLLMs as a preliminary step towards a comprehensive assessment of generative models, by introducing a benchmark named SEED-Bench. 
+      SEED-Bench consists of 19K multiple choice questions with accurate human annotations ($\times$6 larger than existing benchmarks), which spans 12 evaluation dimensions including the comprehension of both the image and video modality. 
+      We develop an advanced pipeline for generating multiple-choice questions that target specific evaluation dimensions, integrating both automatic filtering and manual verification processes. 
+      Multiple-choice questions with groundtruth options derived from human annotation enables an objective and efficient assessment of model performance, eliminating the need for human or GPT intervention during evaluation. 
+      We further evaluate the performance of 18 models across all 12 dimensions, covering both the spatial and temporal understanding. 
+      By revealing the limitations of existing MLLMs through evaluation results, we aim for SEED-Bench to provide insights for motivating future research.
+"""
+
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{li2023seed,
+  title={SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension},
+  author={Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying},
+  journal={arXiv preprint arXiv:2307.16125},
+  year={2023}
+}"""

file/result.csv

@@ -0,0 +1,21 @@
+Model Type,Model,Language Model,Scene Understanding,Instance Identity,Instance Attributes,Instance Localization,Instance Counting,Spatial Relation,Instance Interaction,Visual Reasoning,Text Recognition,Avg. Img,Action Recognition,Action Prediction,Procedure Understanding,Avg. Video,Avg. All
+LLM,Flan-T5,Flan-T5-XL,23.0,29.0,32.8,31.8,20.5,31.8,33.0,18.2,19.4,27.32,23.2,34.9,25.4,28.57,27.65
+LLM,Vicuna,Vicuna-7B,23.4,30.7,29.7,30.9,30.8,28.6,29.8,18.5,13.4,28.16,27.3,34.5,23.8,29.47,28.5
+LLM,LLaMA,LLaMA-7B,26.3,27.4,26.2,28.3,25.1,28.8,19.2,37.0,9.0,26.56,33.0,23.1,26.2,27.27,26.75
+ImageLLM,BLIP-2,Flan-T5-XL,59.1,53.9,49.2,42.3,43.2,36.7,55.7,45.6,25.9,49.74,32.6,47.5,24.0,36.71,46.35
+ImageLLM,InstructBLIP,Flan-T5-XL,60.3,58.5,63.4,40.6,58.4,38.7,51.6,45.9,25.9,57.8,33.1,49.1,27.1,38.31,52.73
+ImageLLM,InstructBLIP-Vicuna,Vicuna-7B,60.2,58.9,65.6,43.6,57.2,40.3,52.6,47.7,43.5,58.76,34.5,49.6,23.1,38.05,53.37
+ImageLLM,LLaVA,LLaMA-7B,42.7,34.9,33.5,28.4,41.9,30.8,27.8,46.8,27.7,36.96,29.7,21.4,19.1,23.76,33.52
+ImageLLM,MiniGPT-4,Flan-T5-XL,56.3,49.2,45.8,37.9,45.3,32.6,47.4,57.1,11.8,47.4,38.2,24.5,27.1,29.89,42.84
+ImageLLM,VPGTrans,LLaMA-7B,51.9,44.1,39.9,36.1,33.7,36.4,32.0,53.2,30.6,41.81,39.5,24.3,31.9,31.4,39.1
+ImageLLM,MultiModal-GPT,LLaMA-7B,43.6,37.9,31.5,30.8,27.3,30.1,29.9,51.4,18.8,34.54,36.9,25.8,24.0,29.21,33.15
+ImageLLM,Otter,LLaMA-7B,44.9,38.6,32.2,30.9,26.3,31.8,32.0,51.4,31.8,35.16,37.9,27.2,24.8,30.35,33.91
+ImageLLM,OpenFlamingo,LLaMA-7B,43.9,38.1,31.3,30.1,27.3,30.6,29.9,50.2,20.0,34.51,37.2,25.4,24.2,29.25,33.14
+ImageLLM,LLaMA-AdapterV2,LLaMA-7B,45.2,38.5,29.3,33.0,29.7,35.5,39.2,52.0,24.7,35.19,38.6,18.5,19.6,25.75,32.73
+ImageLLM,GVT,Vicuna-7B,41.7,35.5,31.8,29.5,36.2,32.0,32.0,51.1,27.1,35.49,33.9,25.4,23.0,27.77,33.48
+ImageLLM,mPLUG-Owl,LLaMA-7B,49.7,45.3,32.5,36.7,27.3,32.7,44.3,54.7,28.8,37.88,26.7,17.9,26.5,23.02,34.01
+VideoLLM,VideoChat,Vicuna-7B,47.1,43.8,34.9,40.0,32.8,34.6,42.3,50.5,17.7,39.02,34.9,36.4,27.3,33.68,37.63
+VideoLLM,Video-ChatGPT,LLaMA-7B,37.2,31.4,33.2,28.4,35.5,29.5,23.7,42.3,25.9,33.88,27.6,21.3,21.1,23.46,31.17
+VideoLLM,Valley,LLaMA-13B,39.3,32.9,31.6,27.9,24.2,30.1,27.8,43.8,11.8,32.04,31.3,23.2,20.7,25.41,30.32
+LLaMA-7B,test,LLaMA-7B,53.2,45.3,40.0,31.2,39.3,32.6,36.1,51.4,25.6,42.7,42.9,34.7,26.9,35.7,40.9
+LLaMA-7B,test2,LLaMA-7B,53.2,45.3,40.0,31.2,39.3,32.6,36.1,51.4,25.6,42.7,42.9,34.7,26.9,35.7,40.9

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+import glob
+import json
+import os
+from typing import Dict, List
+
+from ..utils_display import AutoEvalColumn
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator = " "):
+        return f"{self.value.symbol}{separator}{self.value.name}" 
+

src/utils_display.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"