app_text_classification.py

import gradio as gr
import datasets
import os
import time
import subprocess
import logging
import collections

import json

from transformers.pipelines import TextClassificationPipeline

from text_classification import get_labels_and_features_from_dataset, check_model, get_example_prediction
from io_utils import read_scanners, write_scanners, read_inference_type, read_column_mapping, write_column_mapping, write_inference_type
from wordings import INTRODUCTION_MD, CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW

HF_REPO_ID = 'HF_REPO_ID'
HF_SPACE_ID = 'SPACE_ID'
HF_WRITE_TOKEN = 'HF_WRITE_TOKEN'

MAX_LABELS = 20
MAX_FEATURES = 20

EXAMPLE_MODEL_ID = 'cardiffnlp/twitter-roberta-base-sentiment-latest'
EXAMPLE_DATA_ID = 'tweet_eval'
CONFIG_PATH='./config.yaml'

def try_submit(m_id, d_id, config, split, local):
    all_mappings = read_column_mapping(CONFIG_PATH)

    if "labels" not in all_mappings.keys():
        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
        return gr.update(interactive=True)
    label_mapping = all_mappings["labels"]
    
    if "features" not in all_mappings.keys():
        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
        return gr.update(interactive=True)
    feature_mapping = all_mappings["features"]

    # TODO: Set column mapping for some dataset such as `amazon_polarity`
    if local:
        command = [
            "python",
            "cli.py",
            "--loader", "huggingface",
            "--model", m_id,
            "--dataset", d_id,
            "--dataset_config", config,
            "--dataset_split", split,
            "--hf_token", os.environ.get(HF_WRITE_TOKEN),
            "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
            "--output_format", "markdown",
            "--output_portal", "huggingface",
            "--feature_mapping", json.dumps(feature_mapping),
            "--label_mapping", json.dumps(label_mapping),
            "--scan_config", "../config.yaml",
        ]

        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
        start = time.time()
        logging.info(f"Start local evaluation on {eval_str}")

        evaluator = subprocess.Popen(
            command,
            cwd=os.path.join(os.path.dirname(os.path.realpath(__file__)), "cicd"),
            stderr=subprocess.STDOUT,
        )
        result = evaluator.wait()

        logging.info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")

        gr.Info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
    else:
        gr.Info("TODO: Submit task to an endpoint")
    
    return gr.update(interactive=True)  # Submit button


def check_dataset_and_get_config(dataset_id):
    try:
        configs = datasets.get_dataset_config_names(dataset_id)
        return gr.Dropdown(configs, value=configs[0], visible=True)
    except Exception:
        # Dataset may not exist
        pass

def check_dataset_and_get_split(dataset_id, dataset_config):
    try:
        splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
        return gr.Dropdown(splits, value=splits[0], visible=True)
    except Exception:
        # Dataset may not exist
        # gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
        pass

def get_demo():
    with gr.Row():
        gr.Markdown(INTRODUCTION_MD)
    with gr.Row():
        model_id_input = gr.Textbox(
            label="Hugging Face model id",
            placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
        )

        dataset_id_input = gr.Textbox(
            label="Hugging Face Dataset id",
            placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
        )
    
    with gr.Row():
        dataset_config_input = gr.Dropdown(label='Dataset Config', visible=False)
        dataset_split_input = gr.Dropdown(label='Dataset Split', visible=False)
    
    with gr.Row():
        example_input = gr.Markdown('Example Input', visible=False)
    with gr.Row():
        example_prediction = gr.Label(label='Model Prediction Sample', visible=False)
    
    with gr.Row():
        with gr.Accordion(label='Label and Feature Mapping', visible=False, open=False) as column_mapping_accordion:
            with gr.Row():
                gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
            column_mappings = []
            with gr.Row():
                with gr.Column():
                    for _ in range(MAX_LABELS):
                        column_mappings.append(gr.Dropdown(visible=False))
                with gr.Column():    
                    for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                        column_mappings.append(gr.Dropdown(visible=False))
    
    with gr.Accordion(label='Model Wrap Advance Config (optional)', open=False):
        run_local = gr.Checkbox(value=True, label="Run in this Space")
        use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
        run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
    
    with gr.Accordion(label='Scanner Advance Config (optional)', open=False):
        selected = read_scanners('./config.yaml')
        scan_config = selected + ['data_leakage']
        scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)

    with gr.Row():
        run_btn = gr.Button(
            "Get Evaluation Result",
            variant="primary",
            interactive=True,
            size="lg",
        )
        
    @gr.on(triggers=[label.change for label in column_mappings], 
           inputs=[dataset_id_input, dataset_config_input, dataset_split_input, *column_mappings])
    def write_column_mapping_to_config(dataset_id, dataset_config, dataset_split, *labels):
        ds_labels, ds_features = get_labels_and_features_from_dataset(dataset_id, dataset_config, dataset_split)
        if labels is None:
            return
        labels = [*labels]
        all_mappings = read_column_mapping(CONFIG_PATH)

        if "labels" not in all_mappings.keys():
            all_mappings["labels"] = dict()
        for i, label in enumerate(labels[:MAX_LABELS]):
            if label:
                all_mappings["labels"][label] = ds_labels[i]

        if "features" not in all_mappings.keys():
            all_mappings["features"] = dict()
        for i, feat in enumerate(labels[MAX_LABELS:(MAX_LABELS + MAX_FEATURES)]):
            if feat:
                all_mappings["features"][feat] = ds_features[i]
        write_column_mapping(all_mappings)

    def list_labels_and_features_from_dataset(ds_labels, ds_features, model_id2label):
        model_labels = list(model_id2label.values())
        lables = [gr.Dropdown(label=f"{label}", choices=model_labels, value=model_id2label[i], interactive=True, visible=True) for i, label in enumerate(ds_labels[:MAX_LABELS])]
        lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
        # TODO: Substitute 'text' with more features for zero-shot
        features = [gr.Dropdown(label=f"{feature}", choices=ds_features, value=ds_features[0], interactive=True, visible=True) for feature in ['text']]
        features += [gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))]
        return lables + features
    
    @gr.on(triggers=[model_id_input.change, dataset_config_input.change])
    def clear_column_mapping_config():
        write_column_mapping(None)
    
    @gr.on(triggers=[model_id_input.change, dataset_config_input.change, dataset_split_input.change],
        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
        outputs=[example_input, example_prediction, column_mapping_accordion, *column_mappings])
    def check_model_and_show_prediction(model_id, dataset_id, dataset_config, dataset_split):
        ppl = check_model(model_id)
        if ppl is None or not isinstance(ppl, TextClassificationPipeline):
            gr.Warning("Please check your model.")
            return (
                gr.update(visible=False),
                gr.update(visible=False),
                *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)]
            )
        
        dropdown_placement = [gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)]
            
        if ppl is None: # pipeline not found
            gr.Warning("Model not found")
            return (
                gr.update(visible=False),
                gr.update(visible=False),
                gr.update(visible=False, open=False),
                *dropdown_placement
            )
        model_id2label = ppl.model.config.id2label
        ds_labels, ds_features = get_labels_and_features_from_dataset(dataset_id, dataset_config, dataset_split)
        
        # when dataset does not have labels or features
        if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
            gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
            return (
                gr.update(visible=False),
                gr.update(visible=False),
                gr.update(visible=False, open=False),
                *dropdown_placement
            )
        
        column_mappings = list_labels_and_features_from_dataset(
            ds_labels,
            ds_features,
            model_id2label,
        )

        # when labels or features are not aligned
        # show manually column mapping
        if collections.Counter(model_id2label.items()) != collections.Counter(ds_labels) or ds_features[0] != 'text':
            gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
            return (
                gr.update(visible=False),
                gr.update(visible=False),
                gr.update(visible=True, open=True),
                *column_mappings
            )

        prediction_input, prediction_output = get_example_prediction(ppl, dataset_id, dataset_config, dataset_split)
        return (
            gr.update(value=prediction_input, visible=True),
            gr.update(value=prediction_output, visible=True),
            gr.update(visible=True, open=False),
            *column_mappings
        )

    dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)

    dataset_config_input.change(
        check_dataset_and_get_split, 
        inputs=[dataset_id_input, dataset_config_input], 
        outputs=[dataset_split_input])

    scanners.change(
        write_scanners,
        inputs=scanners
    )

    run_inference.change(
        write_inference_type,
        inputs=[run_inference]
    )

    gr.on(
        triggers=[
            run_btn.click,
            ],
        fn=try_submit,
        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, run_local],
        outputs=[run_btn])