app.py

__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']

import gradio as gr
import pandas as pd
import json
import traceback

from constants import *
from huggingface_hub import Repository
from eval_final_results import eval_final


HF_TOKEN = os.environ.get("HF_TOKEN")

global data_component, filter_component


def download_csv():
    # pull the results and return this file!
    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
                                 repo_type="dataset")
    submission_repo.git_pull()
    return CSV_DIR, gr.update(visible=True)


def upload_file(files):
    file_paths = [file.name for file in files]
    return file_paths


def add_new_eval(
        input_file,
        model_name_textbox: str,
        revision_name_textbox: str,
        model_link: str,
        model_date:str,
        LLM_type: str,
        LLM_name_textbox: str,
):
    if input_file is None:
        return "Error! Empty file!"

    # upload_data = json.loads(input_file)

    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
                                 repo_type="dataset",git_user="auto-uploader",git_email="uploader@163.com")
    submission_repo.git_pull()
    csv_data = pd.read_csv(CSV_DIR)
    try:
        upload_data = eval_final(test_answer_file,dev_answer_file, input_file)
    except:
        error_message = traceback.format_exc()
        print("Error:", error_message)
        return


    if LLM_type == 'Other':
        LLM_name = LLM_name_textbox
    else:
        LLM_name = LLM_type

    if revision_name_textbox == '':
        col = csv_data.shape[0]
        model_name = model_name_textbox
    else:
        model_name = revision_name_textbox
        model_name_list = csv_data['Model']
        name_list = [name.split(']')[0][1:] for name in model_name_list]
        if revision_name_textbox not in name_list:
            col = csv_data.shape[0]
        else:
            col = name_list.index(revision_name_textbox)

    if model_link == '' or "](" in model_name:
        model_name = model_name  # no url
    else:
        model_name = '[' + model_name + '](' + model_link + ')'

    # add new data
    new_data = [
        model_name,
        LLM_name,
        model_date,
        model_link
    ]
    try:
        for key in TASK_INFO:
            if key in key_map:
                new_data.append(round(100*upload_data[key_map[key]],1))
            else:
                new_data.append(0)
    except:
        error_message = traceback.format_exc()
        print("Error:", error_message)
        return
    # print(new_data)
    # print(csv_data.loc[col-1])
    csv_data.loc[col] = new_data
    csv_data = csv_data.to_csv(CSV_DIR, index=False)
    submission_repo.push_to_hub()
    return 0


def get_baseline_df():
    print("SUBMISSION_URL:", SUBMISSION_URL)
    print("HF_TOKEN:", HF_TOKEN)
    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
                                 repo_type="dataset")
    submission_repo.git_pull()
    df = pd.read_csv(CSV_DIR)
    df = df.sort_values(by="Dev Avg", ascending=False)
    present_columns = MODEL_INFO + checkbox_group.value
    df = df[present_columns]
    return df


def get_all_df():
    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
                                 repo_type="dataset")
    submission_repo.git_pull()
    df = pd.read_csv(CSV_DIR)
    df = df.sort_values(by="Dev Avg", ascending=False)
    return df


def on_filter_model_size_method_change(selected_columns):
    updated_data = get_all_df()

    # columns:
    selected_columns = [item for item in TASK_INFO if item in selected_columns]
    present_columns = MODEL_INFO + selected_columns
    # print("selected_columns",'|'.join(selected_columns))
    updated_data = updated_data[present_columns]
    updated_data = updated_data.sort_values(by=selected_columns[0], ascending=False)
    updated_headers = present_columns
    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
    # print(updated_data,present_columns,update_datatype)
    filter_component = gr.components.Dataframe(
        value=updated_data,
        headers=updated_headers,
        type="pandas",
        datatype=update_datatype,
        interactive=False,
        visible=True,
    )

    return filter_component  # .value


block = gr.Blocks()
with block:
    gr.Markdown(
        LEADERBORAD_INTRODUCTION
    )
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 MotionBench", elem_id="lvbench-tab-table", id=1):
            with gr.Row():
                with gr.Accordion("Citation", open=False):
                    citation_button = gr.Textbox(
                        value=CITATION_BUTTON_TEXT,
                        label=CITATION_BUTTON_LABEL,
                        elem_id="citation-button",
                        lines=10,
                    )

            gr.Markdown(
                TABLE_INTRODUCTION
            )

            # selection for column part:
            checkbox_group = gr.CheckboxGroup(
                choices=TASK_INFO,
                value=AVG_INFO,
                label="Evaluation Dimension",
                interactive=True,
            )

            data_component = gr.components.Dataframe(
                value=get_baseline_df,
                headers=COLUMN_NAMES,
                type="pandas",
                datatype=DATA_TITILE_TYPE,
                interactive=False,
                visible=True,
            )

            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group],
                                  outputs=data_component)

        # table 2
        with gr.TabItem("📝 About", elem_id="lvbench-tab-table", id=2):
            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")

        # table 3 
        with gr.TabItem("🚀 Submit here! ", elem_id="lvbench-tab-table", id=3):
            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")

            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(
                        label="Model name", placeholder="CogVLM2-Video"
                    )
                    revision_name_textbox = gr.Textbox(
                        label="Revision Model Name", placeholder=""
                    )

                with gr.Column():
                    LLM_type = gr.Dropdown(
                        choices=["LLaMA-3-8B", "Vicuna-7B", "Flan-T5-XL", "LLaMA-7B", "InternLM-7B", "Other"],
                        label="LLM type",
                        multiselect=False,
                        value="LLaMA-3-8B",
                        interactive=True,
                    )
                    LLM_name_textbox = gr.Textbox(
                        label="LLM model (for Other)",
                        placeholder="LLaMA-3-8B"
                    )
                    model_link = gr.Textbox(
                        label="Model Link", placeholder="https://cogvlm2-video.github.io/"
                    )
                    model_date = gr.Textbox(
                        label="Model Date", placeholder="2024/8/22"
                    )


            with gr.Column():
                input_file = gr.components.File(label="Click to Upload a json File", file_count="single", type='binary')
                submit_button = gr.Button("Submit Eval")

                submission_result = gr.Markdown()
                submit_button.click(
                    add_new_eval,
                    inputs=[
                        input_file,
                        model_name_textbox,
                        revision_name_textbox,
                        model_link,
                        model_date,
                        LLM_type,
                        LLM_name_textbox,
                    ],
                )


    def refresh_data():
        value1 = get_baseline_df()
        return value1


    with gr.Row():
        data_run = gr.Button("Refresh")
    with gr.Row():
        result_download = gr.Button("Download Leaderboard")
        file_download = gr.File(label="download the csv of leaderborad.", visible=False)
        data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
        result_download.click(download_csv, inputs=None, outputs=[file_download, file_download])

block.launch()