fix debug section with files loading

README.md

@@ -1,10 +1,10 @@
 ---
 title: Giskard Evaluator
-emoji: 🐢🔍
+emoji: 🦀
 colorFrom: blue
 colorTo: indigo
 sdk: gradio
-sdk_version: 4.44.1
+sdk_version: 4.7.1
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,5 +1,4 @@
 import atexit
-import traceback
 
 import gradio as gr
 
@@ -7,18 +6,15 @@ from app_debug import get_demo as get_demo_debug
 from app_leaderboard import get_demo as get_demo_leaderboard
 from app_text_classification import get_demo as get_demo_text_classification
 from run_jobs import start_process_run_job, stop_thread
-from wordings import TITLE_MD, FOOTER_HTML
 
 try:
     with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:
-        gr.Markdown(TITLE_MD)
         with gr.Tab("Text Classification"):
-            get_demo_text_classification()
-        with gr.Tab("Leaderboard") as leaderboard_tab:
-            get_demo_leaderboard(leaderboard_tab)
+            get_demo_text_classification(demo)
+        with gr.Tab("Leaderboard"):
+            get_demo_leaderboard()
         with gr.Tab("Logs(Debug)"):
-            get_demo_debug()
-        gr.HTML(FOOTER_HTML)
+            get_demo_debug(demo)
 
     start_process_run_job()
 
@@ -27,7 +23,5 @@ try:
     atexit.register(stop_thread)
 
 except Exception as e:
-    # Print stack trace back
-    print(traceback.format_exc())
     print("stop background thread: ", e)
     stop_thread()

app_debug.py

@@ -1,20 +1,17 @@
 from os import listdir
 from os.path import isfile, join
-import html
 
 import gradio as gr
-import os
 import pipe
 from io_utils import get_logs_file
-from app_env import HF_WRITE_TOKEN
 
 LOG_PATH = "./tmp"
-CONFIG_PATH = "./cicd/configs/submitted/"
+CONFIG_PATH = "./cicd/configs/"
 MAX_FILES_NUM = 20
 
 
 def get_accordions_of_files(path, files):
-    components = [None for _ in range(0, MAX_FILES_NUM)]
+    components = [None for _ in range (0, MAX_FILES_NUM)]
     for i in range(0, len(files)):
         if i >= MAX_FILES_NUM:
             break
@@ -38,94 +35,17 @@ def get_accordions_of_config_files():
     ]
     return get_accordions_of_files(CONFIG_PATH, config_files)
 
-
 def get_config_files():
-    config_files = [
-        join(CONFIG_PATH, f)
-        for f in listdir(CONFIG_PATH)
-        if isfile(join(CONFIG_PATH, f)) and f.endswith(".yaml")
-    ]
+    config_files = [join(CONFIG_PATH, f) for f in listdir(CONFIG_PATH) if isfile(join(CONFIG_PATH, f)) and f.endswith(".yaml")]
     return config_files
 
-
-def get_log_files():
-    return [
-        join(LOG_PATH, f)
-        for f in listdir(LOG_PATH)
-        if isfile(join(LOG_PATH, f)) and f.endswith("log")
-    ]
-
-
-def get_jobs_info_in_queue():
-    return [
-        f"⌛️job id {html.escape(job[0])}: {html.escape(job[2])}<br/>"
-        for job in pipe.jobs
-    ]
-
-
-def get_queue_status():
-    if len(pipe.jobs) > 0 or pipe.current is not None:
-        current = pipe.current
-        if current is None:
-            current = "None"
-        return f'<div style="padding-top: 5%">Current job: {html.escape(current)} <br/> Job queue: <br/> {"".join(get_jobs_info_in_queue())}</div>'
-    else:
-        return '<div style="padding-top: 5%">No jobs waiting, please submit an evaluation task from Text-Classification tab.</div>'
-
-
-def can_write_this_space(hf_token):
-    # Only the user owning `HF_WRITE_TOKEN` is able to manage this space
-    if hf_token == os.getenv(HF_WRITE_TOKEN, ""):
-        return True
-    return False
-
-
-def stop_current_job(hf_token):
-    if not can_write_this_space(hf_token):
-        gr.Warning(
-            "You cannot stop the current job, "
-            "because your token does not match `HF_WRITE_TOKEN` in this space."
-        )
-        return
-
-    task_uuid = pipe.current
-    if not task_uuid:
-        gr.Warning("No job in progress")
-        return
-
-    # Interrupt and stop the task
-    pipe.current = None
-    gr.Info(f"Job {task_uuid} interrupted by admin.")
-
-
-def get_demo():
-    if not os.path.exists(CONFIG_PATH):
-        os.makedirs(CONFIG_PATH)
+def get_demo(demo):
     with gr.Row():
-        gr.HTML(
-            value=get_queue_status,
-            every=5,
-        )
-    with gr.Accordion(label="Admin", open=False):
-        with gr.Row():
-            hf_write_token_input = gr.Textbox(
-                label="HF write token",
-                type="password",
-                placeholder="Please input HF_WRITE_TOKEN configured in the current space",
-            )
-        with gr.Row():
-            stop_job_btn = gr.Button(value="Stop current job", variant="stop")
-            stop_job_btn.click(stop_current_job, inputs=hf_write_token_input)
-    with gr.Accordion(label="Log Files", open=True):
-        with gr.Row():
-            gr.Textbox(
-                value=get_logs_file,
-                every=0.5,
-                lines=10,
-                visible=True,
-                label="Current Log File",
-            )
-        with gr.Row():
-            gr.Files(value=get_log_files, label="Log Files", every=10)
+        # check if jobs is an attribute of pipe
+        if hasattr(pipe, "jobs"):
+            gr.Markdown(f"current jobs in queue: {len(pipe.jobs)}")
+    with gr.Accordion(label="Log Files", open=False):
+        logs = gr.Textbox(lines=10, visible=True, label="Log File")
+        demo.load(get_logs_file, None, logs, every=0.5)
     with gr.Accordion(label="Config Files", open=False):
         gr.Files(value=get_config_files, label="Config Files", every=10)

app_env.py

@@ -1,9 +0,0 @@
-
-HF_REPO_ID = "HF_REPO_ID"
-HF_SPACE_ID = "SPACE_ID"
-HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
-HF_GSK_HUB_URL = "GSK_HUB_URL"
-HF_GSK_HUB_PROJECT_KEY = "GSK_HUB_PROJECT_KEY"
-HF_GSK_HUB_KEY = "GSK_API_KEY"
-HF_GSK_HUB_HF_TOKEN = "GSK_HF_TOKEN"
-HF_GSK_HUB_UNLOCK_TOKEN = "GSK_HUB_UNLOCK_TOKEN"

app_leaderboard.py

@@ -2,35 +2,29 @@ import logging
 
 import datasets
 import gradio as gr
-import pandas as pd
-import datetime
 
 from fetch_utils import (check_dataset_and_get_config,
                          check_dataset_and_get_split)
 
-import leaderboard
-logger = logging.getLogger(__name__)
-global update_time 
-update_time = datetime.datetime.fromtimestamp(0)
 
 def get_records_from_dataset_repo(dataset_id):
     dataset_config = check_dataset_and_get_config(dataset_id)
 
-    logger.info(f"Dataset {dataset_id} has configs {dataset_config}")
+    logging.info(f"Dataset {dataset_id} has configs {dataset_config}")
     dataset_split = check_dataset_and_get_split(dataset_id, dataset_config[0])
-    logger.info(f"Dataset {dataset_id} has splits {dataset_split}")
+    logging.info(f"Dataset {dataset_id} has splits {dataset_split}")
 
     try:
-        ds = datasets.load_dataset(dataset_id, dataset_config[0], split=dataset_split[0])
+        ds = datasets.load_dataset(dataset_id, dataset_config[0])[dataset_split[0]]
         df = ds.to_pandas()
         return df
     except Exception as e:
-        logger.warning(
+        logging.warning(
             f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
         )
-        return pd.DataFrame()
+        return None
+
 
-    
 def get_model_ids(ds):
     logging.info(f"Dataset {ds} column names: {ds['model_id']}")
     models = ds["model_id"].tolist()
@@ -63,58 +57,37 @@ def get_display_df(df):
     columns = display_df.columns.tolist()
     if "model_id" in columns:
         display_df["model_id"] = display_df["model_id"].apply(
-            lambda x: f'<a href="https://huggingface.co/{x}" target="_blank" style="color:blue">🔗{x}</a>'
+            lambda x: f'<p href="https://huggingface.co/{x}" style="color:blue">🔗{x}</p>'
         )
     # style all elements in the dataset_id column
     if "dataset_id" in columns:
         display_df["dataset_id"] = display_df["dataset_id"].apply(
-            lambda x: f'<a href="https://huggingface.co/datasets/{x}" target="_blank" style="color:blue">🔗{x}</a>'
+            lambda x: f'<p href="https://huggingface.co/datasets/{x}" style="color:blue">🔗{x}</p>'
         )
     # style all elements in the report_link column
     if "report_link" in columns:
         display_df["report_link"] = display_df["report_link"].apply(
-            lambda x: f'<a href="{x}" target="_blank" style="color:blue">🔗{x}</a>'
+            lambda x: f'<p href="{x}" style="color:blue">🔗{x}</p>'
         )
     return display_df
 
-def get_demo(leaderboard_tab):
-    global update_time
-    update_time = datetime.datetime.now()
-    logger.info("Loading leaderboard records")
-    leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
-    records = leaderboard.records
+
+def get_demo():
+    records = get_records_from_dataset_repo("ZeroCommand/test-giskard-report")
 
     model_ids = get_model_ids(records)
     dataset_ids = get_dataset_ids(records)
 
     column_names = records.columns.tolist()
-    issue_columns = column_names[:11]
-    info_columns = column_names[15:]
     default_columns = ["model_id", "dataset_id", "total_issues", "report_link"]
     default_df = records[default_columns]  # extract columns selected
     types = get_types(default_df)
     display_df = get_display_df(default_df)  # the styled dataframe to display
 
-    with gr.Row():
-        with gr.Column():
-          info_columns_select = gr.CheckboxGroup(
-              label="Info Columns",
-              choices=info_columns,
-              value=default_columns,
-              interactive=True,
-        )
-        with gr.Column():
-          issue_columns_select = gr.CheckboxGroup(
-              label="Issue Columns",
-              choices=issue_columns,
-              value=[],
-              interactive=True,
-          )
-    
     with gr.Row():
         task_select = gr.Dropdown(
             label="Task",
-            choices=["text_classification"],
+            choices=["text_classification", "tabular"],
             value="text_classification",
             interactive=True,
         )
@@ -129,36 +102,27 @@ def get_demo(leaderboard_tab):
         )
 
     with gr.Row():
-        leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
-
-    def update_leaderboard_records(model_id, dataset_id, issue_columns, info_columns, task):
-        global update_time
-        if datetime.datetime.now() - update_time < datetime.timedelta(minutes=10):
-            return gr.update()
-        update_time = datetime.datetime.now()
-        logger.info("Updating leaderboard records")
-        leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
-        return filter_table(model_id, dataset_id, issue_columns, info_columns, task)
+        columns_select = gr.CheckboxGroup(
+            label="Show columns",
+            choices=column_names,
+            value=default_columns,
+            interactive=True,
+        )
 
-    leaderboard_tab.select(
-        fn=update_leaderboard_records, 
-        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select], 
-        outputs=[leaderboard_df])
+    with gr.Row():
+        leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
 
     @gr.on(
         triggers=[
             model_select.change,
             dataset_select.change,
-            issue_columns_select.change,
-            info_columns_select.change,
+            columns_select.change,
             task_select.change,
         ],
-        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select],
+        inputs=[model_select, dataset_select, columns_select, task_select],
         outputs=[leaderboard_df],
     )
-    def filter_table(model_id, dataset_id, issue_columns, info_columns, task):
-        logger.info("Filtering leaderboard records")
-        records = leaderboard.records
+    def filter_table(model_id, dataset_id, columns, task):
         # filter the table based on task
         df = records[(records["task"] == task)]
         # filter the table based on the model_id and dataset_id
@@ -167,9 +131,8 @@ def get_demo(leaderboard_tab):
         if dataset_id and dataset_id != "Any":
             df = df[(df["dataset_id"] == dataset_id)]
 
-        # filter the table based on the columns 
-        issue_columns.sort()
-        df = df[info_columns + issue_columns]
+        # filter the table based on the columns
+        df = df[columns]
         types = get_types(df)
         display_df = get_display_df(df)
-        return gr.update(value=display_df, datatype=types, interactive=False)
+        return gr.update(value=display_df, datatype=types, interactive=False)

app_text_classification.py

@@ -2,90 +2,49 @@ import uuid
 
 import gradio as gr
 
-from io_utils import read_scanners, write_scanners
-from text_classification_ui_helpers import (
-    get_related_datasets_from_leaderboard,
-    align_columns_and_show_prediction,
-    get_dataset_splits,
-    check_dataset,
-    show_hf_token_info,
-    precheck_model_ds_enable_example_btn,
-    try_submit,
-    empty_column_mapping,
-    write_column_mapping_to_config,
-    enable_run_btn,
-)
-
-import logging
-from wordings import (
-    EXAMPLE_MODEL_ID,
-    CONFIRM_MAPPING_DETAILS_MD,
-    INTRODUCTION_MD,
-    USE_INFERENCE_API_TIP,
-    USE_INFERENCE_API_NOTICE,
-    CHECK_LOG_SECTION_RAW,
-    HF_TOKEN_INVALID_STYLED,
-)
-
-MAX_LABELS = 40
+from io_utils import (get_logs_file, read_inference_type, read_scanners,
+                      write_inference_type, write_scanners)
+from text_classification_ui_helpers import (check_dataset_and_get_config,
+                                            check_dataset_and_get_split,
+                                            check_model_and_show_prediction,
+                                            deselect_run_inference,
+                                            select_run_mode, try_submit,
+                                            write_column_mapping_to_config)
+from wordings import CONFIRM_MAPPING_DETAILS_MD, INTRODUCTION_MD
+
+MAX_LABELS = 20
 MAX_FEATURES = 20
 
+EXAMPLE_MODEL_ID = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+EXAMPLE_DATA_ID = "tweet_eval"
 CONFIG_PATH = "./config.yaml"
-logger = logging.getLogger(__name__)
 
 
-def get_demo():
+def get_demo(demo):
     with gr.Row():
         gr.Markdown(INTRODUCTION_MD)
         uid_label = gr.Textbox(
             label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False
         )
     with gr.Row():
-        with gr.Column():
-            with gr.Row():
-                model_id_input = gr.Textbox(
-                    label="Hugging Face Model id",
-                    placeholder=f"e.g. {EXAMPLE_MODEL_ID}",
-                )
-            with gr.Row():
-                model_id_error_info = gr.HTML(visible=False)
-
-        with gr.Column():
-            dataset_id_input = gr.Dropdown(
-                choices=[],
-                value="",
-                allow_custom_value=True,
-                label="Hugging Face Dataset id",
-            )
-
-    with gr.Row():
-        dataset_config_input = gr.Dropdown(
-            label="Dataset Config", visible=False, allow_custom_value=True
-        )
-        dataset_split_input = gr.Dropdown(
-            label="Dataset Split", visible=False, allow_custom_value=True
+        model_id_input = gr.Textbox(
+            label="Hugging Face model id",
+            placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
         )
 
-    with gr.Row():
-        first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False)
-    with gr.Row():
-        loading_dataset_info = gr.HTML(visible=True)
-    with gr.Row():
-        example_btn = gr.Button(
-            "Validate Model & Dataset",
-            visible=True,
-            variant="primary",
-            interactive=False,
+        dataset_id_input = gr.Textbox(
+            label="Hugging Face Dataset id",
+            placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
         )
+
     with gr.Row():
-        loading_validation = gr.HTML(visible=True)
+        dataset_config_input = gr.Dropdown(label="Dataset Config", visible=False)
+        dataset_split_input = gr.Dropdown(label="Dataset Split", visible=False)
+
     with gr.Row():
-        validation_result = gr.HTML(visible=False)
+        example_input = gr.HTML(visible=False)
     with gr.Row():
-        example_input = gr.Textbox(
-            label="Example Input", visible=False, interactive=False
-        )
-        example_prediction = gr.Label(label="Model Sample Prediction", visible=False)
+        example_prediction = gr.Label(label="Model Prediction Sample", visible=False)
 
     with gr.Row():
         with gr.Accordion(
@@ -96,54 +55,40 @@ def get_demo():
             column_mappings = []
             with gr.Row():
                 with gr.Column():
-                    gr.Markdown("# Label Mapping")
                     for _ in range(MAX_LABELS):
                         column_mappings.append(gr.Dropdown(visible=False))
                 with gr.Column():
-                    gr.Markdown("# Feature Mapping")
                     for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                         column_mappings.append(gr.Dropdown(visible=False))
 
-    with gr.Accordion(label="Model Wrap Advanced Config", open=True):
-        gr.HTML(USE_INFERENCE_API_TIP)
+    with gr.Accordion(label="Model Wrap Advance Config (optional)", open=False):
+        run_local = gr.Checkbox(value=True, label="Run in this Space")
+        run_inference = gr.Checkbox(value="False", label="Run with Inference API")
+
+        @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[run_inference])
+        def get_run_mode(uid):
+            return gr.update(
+                value=read_inference_type(uid) == "hf_inference_api"
+                and not run_local.value
+            )
+
         inference_token = gr.Textbox(
-            placeholder="hf_xxxxxxxxxxxxxxxxxxxx",
             value="",
             label="HF Token for Inference API",
-            visible=True,
+            visible=False,
             interactive=True,
         )
-        inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False)
-        gr.HTML(USE_INFERENCE_API_NOTICE)
-
-        inference_token.change(
-            fn=show_hf_token_info,
-            inputs=[inference_token],
-            outputs=[inference_token_info],
-        )
-
-    with gr.Accordion(label="Scanner Advanced Config (optional)", open=False):
-        with gr.Group():
-            verbose_mode_checkbox = gr.Checkbox(label="Verbose mode")
 
-        scanners = gr.CheckboxGroup(visible=True)
+    with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
+        scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
 
         @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners])
         def get_scanners(uid):
             selected = read_scanners(uid)
-            # we remove data_leakage from the default scanners
+            # currently we remove data_leakage from the default scanners
             # Reason: data_leakage barely raises any issues and takes too many requests
             # when using inference API, causing rate limit error
-            scan_config = [
-                "ethical_bias",
-                "text_perturbation",
-                "robustness",
-                "performance",
-                "underconfidence",
-                "overconfidence",
-                "spurious_correlation",
-                "data_leakage",
-            ]
+            scan_config = selected + ["data_leakage"]
             return gr.update(
                 choices=scan_config, value=selected, label="Scan Settings", visible=True
             )
@@ -152,58 +97,51 @@ def get_demo():
         run_btn = gr.Button(
             "Get Evaluation Result",
             variant="primary",
-            interactive=False,
+            interactive=True,
             size="lg",
         )
 
     with gr.Row():
-        logs = gr.Textbox(
-            value=CHECK_LOG_SECTION_RAW,
-            label="Log",
-            visible=False,
-            every=0.5,
-        )
-
-    scanners.change(write_scanners, inputs=[scanners, uid_label])
-
-    gr.on(
-        triggers=[model_id_input.change],
-        fn=get_related_datasets_from_leaderboard,
-        inputs=[model_id_input, dataset_id_input],
-        outputs=[dataset_id_input],
-    ).then(
-        fn=check_dataset,
-        inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info],
-    )
+        logs = gr.Textbox(label="Giskard Bot Evaluation Log:", visible=False)
+        demo.load(get_logs_file, None, logs, every=0.5)
 
-    gr.on(
-        triggers=[dataset_id_input.input, dataset_id_input.select],
-        fn=check_dataset,
+    dataset_id_input.change(
+        check_dataset_and_get_config,
         inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info],
+        outputs=[dataset_config_input],
     )
 
     dataset_config_input.change(
-        fn=get_dataset_splits,
+        check_dataset_and_get_split,
         inputs=[dataset_id_input, dataset_config_input],
         outputs=[dataset_split_input],
     )
 
-    gr.on(
-        triggers=[
-            model_id_input.change,
-            dataset_id_input.change,
-            dataset_config_input.change,
-        ],
-        fn=empty_column_mapping,
-        inputs=[uid_label],
+    scanners.change(write_scanners, inputs=[scanners, uid_label])
+
+    run_inference.change(
+        select_run_mode,
+        inputs=[run_inference, inference_token, uid_label],
+        outputs=[inference_token, run_local],
+    )
+
+    run_local.change(
+        deselect_run_inference,
+        inputs=[run_local],
+        outputs=[inference_token, run_inference],
+    )
+
+    inference_token.change(
+        write_inference_type, inputs=[run_inference, inference_token, uid_label]
     )
 
     gr.on(
         triggers=[label.change for label in column_mappings],
         fn=write_column_mapping_to_config,
         inputs=[
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
             uid_label,
             *column_mappings,
         ],
@@ -214,6 +152,9 @@ def get_demo():
         triggers=[label.input for label in column_mappings],
         fn=write_column_mapping_to_config,
         inputs=[
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
             uid_label,
             *column_mappings,
         ],
@@ -226,44 +167,17 @@ def get_demo():
             dataset_config_input.change,
             dataset_split_input.change,
         ],
-        fn=precheck_model_ds_enable_example_btn,
-        inputs=[
-            model_id_input,
-            dataset_id_input,
-            dataset_config_input,
-            dataset_split_input,
-        ],
-        outputs=[
-            example_btn,
-            first_line_ds,
-            validation_result,
-            example_input,
-            example_prediction,
-            column_mapping_accordion,
-            model_id_error_info,
-        ],
-    )
-
-    gr.on(
-        triggers=[
-            example_btn.click,
-        ],
-        fn=align_columns_and_show_prediction,
+        fn=check_model_and_show_prediction,
         inputs=[
             model_id_input,
             dataset_id_input,
             dataset_config_input,
             dataset_split_input,
-            uid_label,
-            inference_token,
         ],
         outputs=[
-            validation_result,
             example_input,
             example_prediction,
             column_mapping_accordion,
-            run_btn,
-            loading_validation,
             *column_mappings,
         ],
     )
@@ -278,48 +192,32 @@ def get_demo():
             dataset_id_input,
             dataset_config_input,
             dataset_split_input,
-            inference_token,
+            run_local,
             uid_label,
-            verbose_mode_checkbox,
-        ],
-        outputs=[
-            run_btn,
-            logs,
-            uid_label,
-            validation_result,
-            example_input,
-            example_prediction,
-            column_mapping_accordion,
         ],
+        outputs=[run_btn, logs],
     )
 
+    def enable_run_btn():
+        return gr.update(interactive=True)
+
     gr.on(
         triggers=[
-            inference_token.input,
-            scanners.input,
+            model_id_input.change,
+            dataset_config_input.change,
+            dataset_split_input.change,
+            run_inference.change,
+            run_local.change,
+            scanners.change,
         ],
         fn=enable_run_btn,
-        inputs=[
-            uid_label,
-            inference_token,
-            model_id_input,
-            dataset_id_input,
-            dataset_config_input,
-            dataset_split_input,
-        ],
+        inputs=None,
         outputs=[run_btn],
     )
 
     gr.on(
-        triggers=[label.input for label in column_mappings],
+        triggers=[label.change for label in column_mappings],
         fn=enable_run_btn,
-        inputs=[
-            uid_label,
-            inference_token,
-            model_id_input,
-            dataset_id_input,
-            dataset_config_input,
-            dataset_split_input,
-        ],  # FIXME
+        inputs=None,
         outputs=[run_btn],
     )

config.yaml

@@ -1,8 +1,6 @@
 configuration:
   ethical_bias:
-    threshold: 0.05
-  performance:
-    alpha: 0.05
+    threshold: 0.01
 detectors:
   - ethical_bias
   - text_perturbation
@@ -11,3 +9,4 @@ detectors:
   - underconfidence
   - overconfidence
   - spurious_correlation
+inference_type: hf_pipeline

fetch_utils.py

@@ -5,7 +5,7 @@ import datasets
 
 def check_dataset_and_get_config(dataset_id):
     try:
-        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
+        configs = datasets.get_dataset_config_names(dataset_id)
         return configs
     except Exception:
         # Dataset may not exist
@@ -14,7 +14,7 @@ def check_dataset_and_get_config(dataset_id):
 
 def check_dataset_and_get_split(dataset_id, dataset_config):
     try:
-        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
+        ds = datasets.load_dataset(dataset_id, dataset_config)
     except Exception as e:
         # Dataset may not exist
         logging.warning(

io_utils.py

@@ -1,25 +1,17 @@
 import os
-import logging
+import subprocess
+
 import yaml
 
+import pipe
+
 YAML_PATH = "./cicd/configs"
-LOG_FILE = "temp_log"
 
-logger = logging.getLogger(__name__)
 
 class Dumper(yaml.Dumper):
     def increase_indent(self, flow=False, *args, **kwargs):
         return super().increase_indent(flow=flow, indentless=False)
 
-def get_submitted_yaml_path(uid):
-    if not os.path.exists(f"{YAML_PATH}/submitted"):
-        os.makedirs(f"{YAML_PATH}/submitted")
-    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
-        logger.error(f"config.yaml does not exist for {uid}")
-        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
-    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
-        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
-    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
 
 def get_yaml_path(uid):
     if not os.path.exists(YAML_PATH):
@@ -36,6 +28,7 @@ def read_scanners(uid):
     with open(get_yaml_path(uid), "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
         scanners = config.get("detectors", [])
+    f.close()
     return scanners
 
 
@@ -45,9 +38,11 @@ def write_scanners(scanners, uid):
         config = yaml.load(f, Loader=yaml.FullLoader)
         if config:
             config["detectors"] = scanners
+    f.close()
     # save scanners to detectors in yaml
     with open(get_yaml_path(uid), "w") as f:
         yaml.dump(config, f, Dumper=Dumper)
+    f.close()
 
 
 # read model_type from yaml file
@@ -56,6 +51,7 @@ def read_inference_type(uid):
     with open(get_yaml_path(uid), "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
         inference_type = config.get("inference_type", "")
+    f.close()
     return inference_type
 
 
@@ -70,9 +66,11 @@ def write_inference_type(use_inference, inference_token, uid):
             config["inference_type"] = "hf_pipeline"
             # FIXME: A quick and temp fix for missing token
             config["inference_token"] = ""
+    f.close()
     # save inference_type to inference_type in yaml
     with open(get_yaml_path(uid), "w") as f:
         yaml.dump(config, f, Dumper=Dumper)
+    f.close()
 
 
 # read column mapping from yaml file
@@ -82,8 +80,7 @@ def read_column_mapping(uid):
         config = yaml.load(f, Loader=yaml.FullLoader)
         if config:
             column_mapping = config.get("column_mapping", dict())
-    if column_mapping is None:
-        column_mapping = {}
+    f.close()
     return column_mapping
 
 
@@ -91,6 +88,7 @@ def read_column_mapping(uid):
 def write_column_mapping(mapping, uid):
     with open(get_yaml_path(uid), "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
+    f.close()
 
     if config is None:
         return
@@ -98,9 +96,10 @@ def write_column_mapping(mapping, uid):
         del config["column_mapping"]
     else:
         config["column_mapping"] = mapping
+
     with open(get_yaml_path(uid), "w") as f:
-        # yaml Dumper will by default sort the keys
-        yaml.dump(config, f, Dumper=Dumper, sort_keys=False)
+        yaml.dump(config, f, Dumper=Dumper)
+    f.close()
 
 
 # convert column mapping dataframe to json
@@ -112,23 +111,35 @@ def convert_column_mapping_to_json(df, label=""):
     return column_mapping
 
 
-def get_log_file_with_uid(uid):
-    try:
-        print(f"Loading {uid}.log")
-        with open(f"./tmp/{uid}.log", "a") as file:
-            return file.read()
-    except Exception:
-        return "Log file does not exist"
-
-
 def get_logs_file():
     try:
-        with open(LOG_FILE, "r") as file:
-            return file.read()
+        file = open(f"./tmp/temp_log", "r")
+        return file.read()
     except Exception:
         return "Log file does not exist"
 
 
-def write_log_to_user_file(task_id, log):
-    with open(f"./tmp/{task_id}.log", "a") as f:
+def write_log_to_user_file(id, log):
+    with open(f"./tmp/temp_log", "a") as f:
         f.write(log)
+    f.close()
+
+
+def save_job_to_pipe(id, job, lock):
+    with lock:
+        pipe.jobs.append((id, job))
+
+
+def pop_job_from_pipe():
+    if len(pipe.jobs) == 0:
+        return
+    job_info = pipe.jobs.pop()
+    write_log_to_user_file(job_info[0], f"Running job id {job_info[0]}\n")
+    command = job_info[1]
+
+    log_file = open(f"./tmp/temp_log", "a")
+    subprocess.Popen(
+        command,
+        stdout=log_file,
+        stderr=log_file,
+    )

isolated_env.py

@@ -1,34 +0,0 @@
-import os
-import subprocess
-
-from io_utils import write_log_to_user_file
-
-
-def prepare_venv(execution_id, deps):
-    python_executable = "python"
-    venv_base = f"tmp/venvs/{execution_id}"
-
-    pip_executable = os.path.join(venv_base, "bin", "pip")
-    # Check pyver
-    write_log_to_user_file(execution_id, "Checking Python version\n")
-    p = subprocess.run([python_executable, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    write_log_to_user_file(execution_id, p.stdout.decode())
-    if p.returncode != 0:
-        raise RuntimeError(f"{p.args} ended with {p.returncode}")
-    # Create venv
-    write_log_to_user_file(execution_id, "Creating virtual environment\n")
-    p = subprocess.run([python_executable, "-m", "venv", venv_base, "--clear"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    write_log_to_user_file(execution_id, p.stdout.decode())
-    if p.returncode != 0:
-        raise RuntimeError(f"{p.args} ended with {p.returncode}")
-    # Output requirements.txt
-    requirement_file = os.path.join(venv_base, "requirements.txt")
-    with open(requirement_file, "w") as f:
-        f.writelines(deps)
-    # Install deps
-    write_log_to_user_file(execution_id, "Installing dependencies\n")
-    p = subprocess.run([pip_executable, "install", "-r", requirement_file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    write_log_to_user_file(execution_id, p.stdout.decode())
-    if p.returncode != 0:
-        raise RuntimeError(f"{p.args} ended with {p.returncode}")
-    return os.path.join(venv_base, "bin", "giskard_scanner")

leaderboard.py

@@ -1,5 +0,0 @@
-import pandas as pd
-
-records = pd.DataFrame()
-
-LEADERBOARD = "giskard-bot/evaluator-leaderboard"

pipe.py

@@ -1,3 +1,3 @@
-
-jobs = list()
-current = None
+def init():
+    global jobs
+    jobs = list()

requirements.txt

@@ -1,11 +1,6 @@
-giskard<2.15.0
+giskard >= 2.1.0, < 2.3.0
 huggingface_hub
-hf-transfer
 torch==2.0.1
 transformers
 datasets
-tabulate
-avidtools
 -e git+https://github.com/Giskard-AI/cicd.git#egg=giskard-cicd
-numpy<2.0
-python-multipart==0.0.12

run_jobs.py

@@ -1,30 +1,12 @@
-import json
 import logging
-import os
-import subprocess
 import threading
 import time
-from pathlib import Path
 
 import pipe
-from app_env import (
-    HF_GSK_HUB_HF_TOKEN,
-    HF_GSK_HUB_KEY,
-    HF_GSK_HUB_PROJECT_KEY,
-    HF_GSK_HUB_UNLOCK_TOKEN,
-    HF_GSK_HUB_URL,
-    HF_REPO_ID,
-    HF_SPACE_ID,
-    HF_WRITE_TOKEN,
-)
-from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
-from isolated_env import prepare_venv
-from leaderboard import LEADERBOARD
+from io_utils import pop_job_from_pipe
 
 is_running = False
 
-logger = logging.getLogger(__file__)
-
 
 def start_process_run_job():
     try:
@@ -33,6 +15,7 @@ def start_process_run_job():
         thread = threading.Thread(target=run_job)
         thread.daemon = True
         is_running = True
+        pipe.init()
         thread.start()
 
     except Exception as e:
@@ -45,145 +28,6 @@ def stop_thread():
     is_running = False
 
 
-def prepare_env_and_get_command(
-    m_id,
-    d_id,
-    config,
-    split,
-    inference_token,
-    uid,
-    label_mapping,
-    feature_mapping,
-    verbose,
-):
-    leaderboard_dataset = None
-    if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
-        leaderboard_dataset = LEADERBOARD
-
-    executable = "giskard_scanner"
-    try:
-        # Copy the current requirements (might be changed)
-        with open("requirements.txt", "r") as f:
-            executable = prepare_venv(
-                uid,
-                "\n".join(f.readlines()),
-            )
-        logger.info(f"Using {executable} as executable")
-    except Exception as e:
-        logger.warning(f"Create env failed due to {e}, using the current env as fallback.")
-        executable = "giskard_scanner"
-
-    command = [
-        executable,
-        "--loader",
-        "huggingface",
-        "--model",
-        m_id,
-        "--dataset",
-        d_id,
-        "--dataset_config",
-        config,
-        "--dataset_split",
-        split,
-        "--output_format",
-        "markdown",
-        "--output_portal",
-        "huggingface",
-        "--feature_mapping",
-        json.dumps(feature_mapping),
-        "--label_mapping",
-        json.dumps(label_mapping),
-        "--scan_config",
-        get_submitted_yaml_path(uid),
-        "--inference_type",
-        "hf_inference_api",
-        "--inference_api_token",
-        inference_token,
-        "--persist_scan",
-    ]
-    # The token to publish post
-    if os.environ.get(HF_WRITE_TOKEN):
-        command.append("--hf_token")
-        command.append(os.environ.get(HF_WRITE_TOKEN))
-
-    # The repo to publish for ranking
-    if leaderboard_dataset:
-        command.append("--leaderboard_dataset")
-        command.append(leaderboard_dataset)
-
-    # The info to upload to Giskard hub
-    if os.environ.get(HF_GSK_HUB_KEY):
-        command.append("--giskard_hub_api_key")
-        command.append(os.environ.get(HF_GSK_HUB_KEY))
-        if os.environ.get(HF_GSK_HUB_URL):
-            command.append("--giskard_hub_url")
-            command.append(os.environ.get(HF_GSK_HUB_URL))
-        if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
-            command.append("--giskard_hub_project_key")
-            command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
-        if os.environ.get(HF_GSK_HUB_HF_TOKEN):
-            command.append("--giskard_hub_hf_token")
-            command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
-        if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
-            command.append("--giskard_hub_unlock_token")
-            command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
-
-    if verbose:
-        command.append("--verbose")
-
-    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-
-    write_log_to_user_file(
-        uid,
-        f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
-    )
-
-    return command
-
-
-def save_job_to_pipe(task_id, job, description, lock):
-    with lock:
-        pipe.jobs.append((task_id, job, description))
-
-
-def pop_job_from_pipe():
-    if len(pipe.jobs) == 0:
-        return
-    job_info = pipe.jobs.pop()
-    pipe.current = job_info[2]
-    task_id = job_info[0]
-
-    # Link to LOG_FILE
-    log_file_path = Path(LOG_FILE)
-    if log_file_path.exists():
-        log_file_path.unlink()
-    os.symlink(f"./tmp/{task_id}.log", LOG_FILE)
-
-    write_log_to_user_file(task_id, f"Running job id {task_id}\n")
-    command = prepare_env_and_get_command(*job_info[1])
-
-    with open(f"./tmp/{task_id}.log", "a") as log_file:
-        return_code = None
-        p = subprocess.Popen(command, stdout=log_file, stderr=subprocess.STDOUT)
-        while pipe.current and return_code is None:
-            # Wait for finishing
-            try:
-                return_code = p.wait(timeout=1)
-            except subprocess.TimeoutExpired:
-                return_code = None
-
-        if not pipe.current:
-            # Job interrupted before finishing
-            p.kill()
-
-            log_file.write(f"\nJob interrupted by admin at {time.asctime()}\n")
-
-        if return_code:
-            log_file.write(f"\nJob finished with {return_code} at {time.asctime()}\n")
-
-    pipe.current = None
-
-
 def run_job():
     global is_running
     while is_running:

text_classification.py

@@ -5,119 +5,27 @@ import datasets
 import huggingface_hub
 import pandas as pd
 from transformers import pipeline
-import requests
-import os
-from app_env import HF_WRITE_TOKEN
 
-logger = logging.getLogger(__name__)
-AUTH_CHECK_URL = "https://huggingface.co/api/whoami-v2"
 
-logger = logging.getLogger(__file__)
-
-
-class HuggingFaceInferenceAPIResponse:
-    def __init__(self, message):
-        self.message = message
-
-
-def get_labels_and_features_from_dataset(ds):
+def get_labels_and_features_from_dataset(dataset_id, dataset_config, split):
+    if not dataset_config:
+        dataset_config = "default"
+    if not split:
+        split = "train"
     try:
+        ds = datasets.load_dataset(dataset_id, dataset_config)[split]
         dataset_features = ds.features
-        label_keys = [i for i in dataset_features.keys() if i.startswith("label")]
-        features = [f for f in dataset_features.keys() if not f.startswith("label")]
-
-        if len(label_keys) == 0:  # no labels found
-            # return everything for post processing
-            return list(dataset_features.keys()), list(dataset_features.keys()), None
-
-        labels = None
-        if not isinstance(dataset_features[label_keys[0]], datasets.ClassLabel):
-            if hasattr(dataset_features[label_keys[0]], "feature"):
-                label_feat = dataset_features[label_keys[0]].feature
-                labels = label_feat.names
-            else:
-                labels = ds.unique(label_keys[0])
-        else:
-            labels = dataset_features[label_keys[0]].names
-        return labels, features, label_keys
+        labels = dataset_features["label"].names
+        features = [f for f in dataset_features.keys() if f != "label"]
+        return labels, features
     except Exception as e:
-        logging.warning(f"Get Labels/Features Failed for dataset: {e}")
-        return None, None, None
-
-
-def check_model_task(model_id):
-    # check if model is valid on huggingface
-    try:
-        task = huggingface_hub.model_info(model_id).pipeline_tag
-        if task is None:
-            return None
-        return task
-    except Exception:
-        return None
-
-
-def get_model_labels(model_id, example_input):
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-    payload = {"inputs": example_input, "options": {"use_cache": True}}
-    response = hf_inference_api(model_id, hf_token, payload)
-    if "error" in response:
-        return None
-    return extract_from_response(response, "label")
-
-
-def extract_from_response(data, key):
-    results = []
-
-    if isinstance(data, dict):
-        res = data.get(key)
-        if res is not None:
-            results.append(res)
-
-        for value in data.values():
-            results.extend(extract_from_response(value, key))
-
-    elif isinstance(data, list):
-        for element in data:
-            results.extend(extract_from_response(element, key))
-
-    return results
-
-
-def hf_inference_api(model_id, hf_token, payload):
-    hf_inference_api_endpoint = os.environ.get(
-        "HF_INFERENCE_ENDPOINT", default="https://api-inference.huggingface.co"
-    )
-    url = f"{hf_inference_api_endpoint}/models/{model_id}"
-    headers = {"Authorization": f"Bearer {hf_token}"}
-    response = requests.post(url, headers=headers, json=payload)
-
-    if not hasattr(response, "status_code") or response.status_code != 200:
-        logger.warning(f"Request to inference API returns {response}")
-
-    try:
-        output = response.json()
-        if "error" in output and "Input is too long" in output["error"]:
-            payload.update({"parameters": {"truncation": True, "max_length": 512}})
-            response = requests.post(url, headers=headers, json=payload)
-            if not hasattr(response, "status_code") or response.status_code != 200:
-                logger.warning(f"Request to inference API returns {response}")
-        return response.json()
-    except Exception:
-        return {"error": response.content}
-
-
-def preload_hf_inference_api(model_id):
-    payload = {
-        "inputs": "This is a test",
-        "options": {
-            "use_cache": True,
-        },
-    }
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-    hf_inference_api(model_id, hf_token, payload)
+        logging.warning(
+            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
+        )
+        return None, None
 
 
-def check_model_pipeline(model_id):
+def check_model(model_id):
     try:
         task = huggingface_hub.model_info(model_id).pipeline_tag
     except Exception:
@@ -278,7 +186,7 @@ def infer_output_label_column(
 
 def check_dataset_features_validity(d_id, config, split):
     # We assume dataset is ok here
-    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
+    ds = datasets.load_dataset(d_id, config)[split]
     try:
         dataset_features = ds.features
     except AttributeError:
@@ -290,52 +198,27 @@ def check_dataset_features_validity(d_id, config, split):
     return df, dataset_features
 
 
-def select_the_first_string_column(ds):
-    for feature in ds.features.keys():
-        if isinstance(ds[0][feature], str):
-            return feature
-    return None
-
-
-def get_example_prediction(
-    model_id, dataset_id, dataset_config, dataset_split, hf_token
-):
+def get_example_prediction(ppl, dataset_id, dataset_config, dataset_split):
     # get a sample prediction from the model on the dataset
     prediction_input = None
     prediction_result = None
     try:
         # Use the first item to test prediction
-        ds = datasets.load_dataset(
-            dataset_id, dataset_config, split=dataset_split, trust_remote_code=True
-        )
+        ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
         if "text" not in ds.features.keys():
             # Dataset does not have text column
-            prediction_input = ds[0][select_the_first_string_column(ds)]
+            prediction_input = ds[0][ds.features.keys()[0]]
         else:
             prediction_input = ds[0]["text"]
 
-        payload = {"inputs": prediction_input, "options": {"use_cache": True}}
-        results = hf_inference_api(model_id, hf_token, payload)
-
-        if isinstance(results, dict) and "error" in results.keys():
-            if "estimated_time" in results.keys():
-                return prediction_input, HuggingFaceInferenceAPIResponse(
-                    f"Estimated time: {int(results['estimated_time'])}s. Please try again later."
-                )
-            return prediction_input, HuggingFaceInferenceAPIResponse(
-                f"Inference Error: {results['error']}."
-            )
-
-        while isinstance(results, list):
-            if isinstance(results[0], dict):
-                break
-            results = results[0]
+        print("prediction_input", prediction_input)
+        results = ppl(prediction_input, top_k=None)
+        # Display results in original label and mapped label
         prediction_result = {
             f'{result["label"]}': result["score"] for result in results
         }
-    except Exception as e:
-        # inference api prediction failed, show the error message
-        logger.error(f"Get example prediction failed {e}")
+    except Exception:
+        # Pipeline prediction failed, need to provide labels
         return prediction_input, None
 
     return prediction_input, prediction_result
@@ -407,16 +290,3 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
         id2label_df,
         feature_map_df,
     )
-
-
-def check_hf_token_validity(hf_token):
-    if hf_token == "":
-        return False
-    if not isinstance(hf_token, str):
-        return False
-    # use huggingface api to check the token
-    headers = {"Authorization": f"Bearer {hf_token}"}
-    response = requests.get(AUTH_CHECK_URL, headers=headers)
-    if response.status_code != 200:
-        return False
-    return True

text_classification_ui_helpers.py

@@ -1,181 +1,108 @@
 import collections
+import json
 import logging
+import os
 import threading
-import uuid
 
 import datasets
 import gradio as gr
-import pandas as pd
-
-import leaderboard
-from io_utils import (
-    read_column_mapping,
-    write_column_mapping,
-    read_scanners,
-    write_scanners,
-)
-from run_jobs import save_job_to_pipe
-from text_classification import (
-    check_model_task,
-    preload_hf_inference_api,
-    get_example_prediction,
-    get_labels_and_features_from_dataset,
-    check_hf_token_validity,
-    HuggingFaceInferenceAPIResponse,
-)
-from wordings import (
-    EXAMPLE_MODEL_ID,
-    CHECK_CONFIG_OR_SPLIT_RAW,
-    CONFIRM_MAPPING_DETAILS_FAIL_RAW,
-    MAPPING_STYLED_ERROR_WARNING,
-    NOT_FOUND_DATASET_RAW,
-    NOT_FOUND_MODEL_RAW,
-    NOT_TEXT_CLASSIFICATION_MODEL_RAW,
-    UNMATCHED_MODEL_DATASET_STYLED_ERROR,
-    CHECK_LOG_SECTION_RAW,
-    VALIDATED_MODEL_DATASET_STYLED,
-    get_dataset_fetch_error_raw,
-)
-import os
-from app_env import HF_WRITE_TOKEN
-
-MAX_LABELS = 40
+from transformers.pipelines import TextClassificationPipeline
+from wordings import get_styled_input
+
+from io_utils import (get_yaml_path, read_column_mapping, save_job_to_pipe,
+                      write_column_mapping, write_inference_type,
+                      write_log_to_user_file)
+from text_classification import (check_model, get_example_prediction,
+                                 get_labels_and_features_from_dataset)
+from wordings import (CHECK_CONFIG_OR_SPLIT_RAW,
+                      CONFIRM_MAPPING_DETAILS_FAIL_RAW,
+                      MAPPING_STYLED_ERROR_WARNING)
+
+MAX_LABELS = 20
 MAX_FEATURES = 20
 
-ds_dict = None
-ds_config = None
-
-
-def get_related_datasets_from_leaderboard(model_id, dataset_id_input):
-    records = leaderboard.records
-    model_records = records[records["model_id"] == model_id]
-    datasets_unique = list(model_records["dataset_id"].unique())
-
-    if len(datasets_unique) == 0:
-        return gr.update(choices=[])
+HF_REPO_ID = "HF_REPO_ID"
+HF_SPACE_ID = "SPACE_ID"
+HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
 
-    if dataset_id_input in datasets_unique:
-        return gr.update(choices=datasets_unique)
 
-    return gr.update(choices=datasets_unique, value="")
-
-
-logger = logging.getLogger(__file__)
-
-
-def get_dataset_splits(dataset_id, dataset_config):
+def check_dataset_and_get_config(dataset_id):
     try:
-        splits = datasets.get_dataset_split_names(
-            dataset_id, dataset_config, trust_remote_code=True
-        )
-        return gr.update(choices=splits, value=splits[0], visible=True)
-    except Exception as e:
-        logger.warning(
-            f"Check your dataset {dataset_id} and config {dataset_config}: {e}"
-        )
-    return gr.update(visible=False)
-
+        # write_column_mapping(None, uid)  # reset column mapping
+        configs = datasets.get_dataset_config_names(dataset_id)
+        return gr.Dropdown(configs, value=configs[0], visible=True)
+    except Exception:
+        # Dataset may not exist
+        pass
 
-def check_dataset(dataset_id):
-    logger.info(f"Loading {dataset_id}")
-    if not dataset_id or len(dataset_id) == 0:
-        return (gr.update(visible=False), gr.update(visible=False), "")
 
+def check_dataset_and_get_split(dataset_id, dataset_config):
     try:
-        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
-        if len(configs) == 0:
-            return (gr.update(visible=False), gr.update(visible=False), "")
-        splits = datasets.get_dataset_split_names(
-            dataset_id, configs[0], trust_remote_code=True
-        )
-        return (
-            gr.update(choices=configs, value=configs[0], visible=True),
-            gr.update(choices=splits, value=splits[0], visible=True),
-            "",
-        )
-    except Exception as e:
-        logger.warning(f"Check your dataset {dataset_id}: {e}")
-        if "doesn't exist on the Hub or cannot be accessed" in str(e):
-            gr.Warning(NOT_FOUND_DATASET_RAW)
-        elif "forbidden" in str(e).lower():
-            # GSK-2770: illegal name
-            gr.Warning(get_dataset_fetch_error_raw(e))
-        else:
-            # Unknown error
-            gr.Warning(get_dataset_fetch_error_raw(e))
+        splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
+        return gr.Dropdown(splits, value=splits[0], visible=True)
+    except Exception:
+        # Dataset may not exist
+        # gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+        pass
 
-    return (gr.update(visible=False), gr.update(visible=False), "")
 
+def select_run_mode(run_inf, inf_token, uid):
+    if run_inf:
+        if len(inf_token) > 0:
+            write_inference_type(run_inf, inf_token, uid)
+        return (gr.update(visible=True), gr.update(value=False))
+    else:
+        return (gr.update(visible=False), gr.update(value=True))
 
-def empty_column_mapping(uid):
-    write_column_mapping(None, uid)
 
+def deselect_run_inference(run_local):
+    if run_local:
+        return (gr.update(visible=False), gr.update(value=False))
+    else:
+        return (gr.update(visible=True), gr.update(value=True))
 
-def write_column_mapping_to_config(uid, *labels):
+
+def write_column_mapping_to_config(
+    dataset_id, dataset_config, dataset_split, uid, *labels
+):
     # TODO: Substitute 'text' with more features for zero-shot
     # we are not using ds features because we only support "text" for now
-    all_mappings = read_column_mapping(uid)
-
+    ds_labels, _ = get_labels_and_features_from_dataset(
+        dataset_id, dataset_config, dataset_split
+    )
     if labels is None:
         return
-    all_mappings = export_mappings(all_mappings, "labels", None, labels[:MAX_LABELS])
-    all_mappings = export_mappings(
-        all_mappings,
-        "features",
-        ["text"],
-        labels[MAX_LABELS : (MAX_LABELS + MAX_FEATURES)],
-    )
-
-    write_column_mapping(all_mappings, uid)
-
-
-def export_mappings(all_mappings, key, subkeys, values):
-    if key not in all_mappings.keys():
-        all_mappings[key] = dict()
-    if subkeys is None:
-        subkeys = list(all_mappings[key].keys())
 
-    if not subkeys:
-        logging.debug(f"subkeys is empty for {key}")
-        return all_mappings
+    all_mappings = dict()
 
-    for i, subkey in enumerate(subkeys):
-        if subkey:
-            all_mappings[key][subkey] = values[i % len(values)]
-    return all_mappings
-
-
-def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels, uid):
-    all_mappings = read_column_mapping(uid)
-    # For flattened raw datasets with no labels
-    # check if there are shared labels between model and dataset
-    shared_labels = set(model_labels).intersection(set(ds_labels))
-    if shared_labels:
-        ds_labels = list(shared_labels)
-    if len(ds_labels) > MAX_LABELS:
-        ds_labels = ds_labels[:MAX_LABELS]
-        gr.Warning(
-            f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd."
-        )
+    if "labels" not in all_mappings.keys():
+        all_mappings["labels"] = dict()
+    for i, label in enumerate(labels[:MAX_LABELS]):
+        if label:
+            all_mappings["labels"][label] = ds_labels[i % len(ds_labels)]
+    if "features" not in all_mappings.keys():
+        all_mappings["features"] = dict()
+    for _, feat in enumerate(labels[MAX_LABELS : (MAX_LABELS + MAX_FEATURES)]):
+        if feat:
+            # TODO: Substitute 'text' with more features for zero-shot
+            all_mappings["features"]["text"] = feat
+    write_column_mapping(all_mappings, uid)
 
-    # sort labels to make sure the order is consistent
-    # prediction gives the order based on probability
-    ds_labels.sort()
-    model_labels.sort()
 
+def list_labels_and_features_from_dataset(ds_labels, ds_features, model_id2label):
+    model_labels = list(model_id2label.values())
+    len_model_labels = len(model_labels)
     lables = [
         gr.Dropdown(
             label=f"{label}",
             choices=model_labels,
-            value=model_labels[i % len(model_labels)],
+            value=model_id2label[i % len_model_labels],
             interactive=True,
             visible=True,
         )
-        for i, label in enumerate(ds_labels)
+        for i, label in enumerate(ds_labels[:MAX_LABELS])
     ]
     lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
-    all_mappings = export_mappings(all_mappings, "labels", ds_labels, model_labels)
-
     # TODO: Substitute 'text' with more features for zero-shot
     features = [
         gr.Dropdown(
@@ -190,122 +117,18 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
     features += [
         gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))
     ]
-    all_mappings = export_mappings(all_mappings, "features", ["text"], ds_features)
-    write_column_mapping(all_mappings, uid)
-
     return lables + features
 
 
-def precheck_model_ds_enable_example_btn(
+def check_model_and_show_prediction(
     model_id, dataset_id, dataset_config, dataset_split
 ):
-    model_task = check_model_task(model_id)
-    if not model_task:
-        # Model might be not found
-        error_msg_html = f"<p style='color: red;'>{NOT_FOUND_MODEL_RAW}</p>"
-        if model_id.startswith("http://") or model_id.startswith("https://"):
-            error_msg = f"Please input your model id, such as {EXAMPLE_MODEL_ID}, instead of URL"
-            error_msg_html = f"<p style='color: red;'>{error_msg}</p>"
-
+    ppl = check_model(model_id)
+    if ppl is None or not isinstance(ppl, TextClassificationPipeline):
+        gr.Warning("Please check your model.")
         return (
-            gr.update(interactive=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(value=error_msg_html, visible=True),
-        )
-
-    if model_task != "text-classification":
-        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-        return (
-            gr.update(interactive=False),
-            gr.update(value=df, visible=True),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(
-                value=f"<p style='color: red;'>{NOT_TEXT_CLASSIFICATION_MODEL_RAW}",
-                visible=True,
-            ),
-        )
-
-    preload_hf_inference_api(model_id)
-
-    if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
-        return (
-            gr.update(interactive=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
-
-    try:
-        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
-        df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
-        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(
-            ds[dataset_split]
-        )
-
-        if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
-            gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-            return (
-                gr.update(interactive=False),
-                gr.update(value=df, visible=True),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-            )
-
-        return (
-            gr.update(interactive=True),
-            gr.update(value=df, visible=True),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
-    except Exception as e:
-        # Config or split wrong
-        logger.warning(
-            f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}"
-        )
-        return (
-            gr.update(interactive=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
-            gr.update(visible=False),
-        )
-
-
-def align_columns_and_show_prediction(
-    model_id,
-    dataset_id,
-    dataset_config,
-    dataset_split,
-    uid,
-    inference_token,
-):
-    model_task = check_model_task(model_id)
-    if model_task is None or model_task != "text-classification":
-        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
             *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)],
         )
 
@@ -313,229 +136,131 @@ def align_columns_and_show_prediction(
         gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
     ]
 
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-
-    prediction_input, prediction_response = get_example_prediction(
-        model_id, dataset_id, dataset_config, dataset_split, hf_token
-    )
-
-    if prediction_input is None or prediction_response is None:
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *dropdown_placement,
-        )
-
-    if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
+    if ppl is None:  # pipeline not found
+        gr.Warning("Model not found")
         return (
-            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            f"Hugging Face Inference API is loading your model. {prediction_response.message}",
             *dropdown_placement,
         )
-
-    model_labels = list(prediction_response.keys())
-
-    ds = datasets.load_dataset(
-        dataset_id, dataset_config, split=dataset_split, trust_remote_code=True
+    model_id2label = ppl.model.config.id2label
+    ds_labels, ds_features = get_labels_and_features_from_dataset(
+        dataset_id, dataset_config, dataset_split
     )
-    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
 
     # when dataset does not have labels or features
     if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
         gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
         return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *dropdown_placement,
-        )
-
-    if len(ds_labels) != len(model_labels):
-        return (
-            gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
             *dropdown_placement,
         )
 
     column_mappings = list_labels_and_features_from_dataset(
         ds_labels,
         ds_features,
-        model_labels,
-        uid,
+        model_id2label,
     )
 
     # when labels or features are not aligned
     # show manually column mapping
     if (
-        collections.Counter(model_labels) != collections.Counter(ds_labels)
+        collections.Counter(model_id2label.values()) != collections.Counter(ds_labels)
         or ds_features[0] != "text"
     ):
         return (
             gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
-            gr.update(
-                value=prediction_input,
-                lines=min(len(prediction_input) // 225 + 1, 5),
-                visible=True,
-            ),
-            gr.update(value=prediction_response, visible=True),
+            gr.update(visible=False),
             gr.update(visible=True, open=True),
-            gr.update(interactive=(inference_token != "")),
-            "",
             *column_mappings,
         )
 
+    prediction_input, prediction_output = get_example_prediction(
+        ppl, dataset_id, dataset_config, dataset_split
+    )
     return (
-        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
-        gr.update(
-            value=prediction_input,
-            lines=min(len(prediction_input) // 225 + 1, 5),
-            visible=True,
-        ),
-        gr.update(value=prediction_response, visible=True),
+        gr.update(value=get_styled_input(prediction_input), visible=True),
+        gr.update(value=prediction_output, visible=True),
         gr.update(visible=True, open=False),
-        gr.update(interactive=(inference_token != "")),
-        "",
         *column_mappings,
     )
 
 
-def check_column_mapping_keys_validity(all_mappings):
+def try_submit(m_id, d_id, config, split, local, uid):
+    all_mappings = read_column_mapping(uid)
+
     if all_mappings is None:
-        logger.warning("all_mapping is None")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
-        return False
+        return (gr.update(interactive=True), gr.update(visible=False))
 
     if "labels" not in all_mappings.keys():
-        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
-        return False
-
-    return True
-
-
-def enable_run_btn(
-    uid, inference_token, model_id, dataset_id, dataset_config, dataset_split
-):
-    if inference_token == "":
-        logger.warning("Inference API is not enabled")
-        return gr.update(interactive=False)
-    if (
-        model_id == ""
-        or dataset_id == ""
-        or dataset_config == ""
-        or dataset_split == ""
-    ):
-        logger.warning("Model id or dataset id is not selected")
-        return gr.update(interactive=False)
-
-    all_mappings = read_column_mapping(uid)
-    if not check_column_mapping_keys_validity(all_mappings):
-        logger.warning("Column mapping is not valid")
-        return gr.update(interactive=False)
-
-    if not check_hf_token_validity(inference_token):
-        logger.warning("HF token is not valid")
-        return gr.update(interactive=False)
-    return gr.update(interactive=True)
-
-
-def construct_label_and_feature_mapping(
-    all_mappings, ds_labels, ds_features, label_keys=None
-):
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return (gr.update(interactive=True), gr.update(visible=False))
     label_mapping = {}
-    if len(all_mappings["labels"].keys()) != len(ds_labels):
-        logger.warning(
-            f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}"""
-        )
-
-    if len(all_mappings["features"].keys()) != len(ds_features):
-        logger.warning(
-            f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_features: {ds_features}"""
-        )
-
-    for i, label in zip(range(len(ds_labels)), ds_labels):
-        # align the saved labels with dataset labels order
-        label_mapping.update({str(i): all_mappings["labels"][label]})
+    for i, label in zip(
+        range(len(all_mappings["labels"].keys())), all_mappings["labels"].keys()
+    ):
+        label_mapping.update({str(i): label})
 
     if "features" not in all_mappings.keys():
-        logger.warning("features not in all_mappings")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
-
-    feature_mapping = all_mappings["features"]
-    if len(label_keys) > 0:
-        feature_mapping.update({"label": label_keys[0]})
-    return label_mapping, feature_mapping
-
-
-def show_hf_token_info(token):
-    valid = check_hf_token_validity(token)
-    if not valid:
-        return gr.update(visible=True)
-    return gr.update(visible=False)
-
-
-def try_submit(m_id, d_id, config, split, inference_token, uid, verbose):
-    all_mappings = read_column_mapping(uid)
-    if not check_column_mapping_keys_validity(all_mappings):
         return (gr.update(interactive=True), gr.update(visible=False))
+    feature_mapping = all_mappings["features"]
 
-    # get ds labels and features again for alignment
-    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
-    label_mapping, feature_mapping = construct_label_and_feature_mapping(
-        all_mappings, ds_labels, ds_features, label_keys
-    )
-
-    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-    save_job_to_pipe(
-        uid,
-        (
+    leaderboard_dataset = None
+    if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
+        leaderboard_dataset = "ZeroCommand/test-giskard-report"
+
+    # TODO: Set column mapping for some dataset such as `amazon_polarity`
+    if local:
+        command = [
+            "giskard_scanner",
+            "--loader",
+            "huggingface",
+            "--model",
             m_id,
+            "--dataset",
             d_id,
+            "--dataset_config",
             config,
+            "--dataset_split",
             split,
-            inference_token,
+            "--hf_token",
+            os.environ.get(HF_WRITE_TOKEN),
+            "--discussion_repo",
+            os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+            "--output_format",
+            "markdown",
+            "--output_portal",
+            "huggingface",
+            "--feature_mapping",
+            json.dumps(feature_mapping),
+            "--label_mapping",
+            json.dumps(label_mapping),
+            "--scan_config",
+            get_yaml_path(uid),
+            "--leaderboard_dataset",
+            leaderboard_dataset,
+        ]
+
+        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+        logging.info(f"Start local evaluation on {eval_str}")
+        save_job_to_pipe(uid, command, threading.Lock())
+        write_log_to_user_file(
             uid,
-            label_mapping,
-            feature_mapping,
-            verbose,
-        ),
-        eval_str,
-        threading.Lock(),
-    )
-    gr.Info("Your evaluation has been submitted")
+            f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
+        )
+        gr.Info(f"Start local evaluation on {eval_str}")
+
+        return (
+            gr.update(interactive=False),
+            gr.update(lines=5, visible=True, interactive=False),
+        )
 
-    new_uid = uuid.uuid4()
-    scanners = read_scanners(uid)
-    write_scanners(scanners, new_uid)
+    else:
+        gr.Info("TODO: Submit task to an endpoint")
 
-    return (
-        gr.update(interactive=False),  # Submit button
-        gr.update(
-            value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ",
-            lines=5,
-            visible=True,
-            interactive=False,
-        ),
-        new_uid,  # Allocate a new uuid
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
-    )
+    return (gr.update(interactive=True), gr.update(visible=False))  # Submit button

wordings.py

@@ -1,119 +1,44 @@
-EXAMPLE_MODEL_ID = "cardiffnlp/twitter-roberta-base-sentiment-latest"
-
-TITLE_MD = """
-    <div style="display: flex; justify-content: center;">
-    <h1 style="text-align: center;">
-    🐢Giskard Evaluator
-    </h1>
-    </div>
-    Welcome to the Giskard Evaluator Space!
-    Get a model vulnerability report immediately and freely by simply providing the model and dataset id on Hugging Face.
-    You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
-    <!-- Don’t hesitate to give the us a <a href="https://github.com/Giskard-AI/giskard">star on GitHub</a> ⭐️ if you find this work useful! -->
-"""
-
-FOOTER_HTML = """
-    <div style="display: flex; justify-content: center;">
-    <p style="text-align: center;">
-    This evaluator is built on top of&nbsp;<a href="https://github.com/Giskard-AI/giskard">Giskard Open Source library</a>
-    and <a href="https://github.com/Giskard-AI/cicd">Giskard CI/CD</a>.
-    <br/>
-    If you like our work, please give the projects ⭐️ star on&nbsp;<a href="https://github.com/Giskard-AI/giskard">GitHub</a>.
-    </p>
-    </div>
-"""
-
 INTRODUCTION_MD = """
-                <div style="display: flex; justify-content: center;">
-                <h2 style="text-align: center;">
-                Text Classification
-                </h2>
-                </div>
+                <h1 style="text-align: center;">
+                🐢Giskard Evaluator
+                </h1>
+                Welcome to Giskard Evaluator Space! Get your report immediately by simply input your model id and dataset id below. Follow our leads and improve your model in no time.
                 """
 CONFIRM_MAPPING_DETAILS_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
+                            Please confirm the pre-processing details below. Align the column names of your model in the <b>dropdown</b> menu to your dataset's. If you are not sure, please double check your model and dataset.
                             """
 CONFIRM_MAPPING_DETAILS_FAIL_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            Sorry, we cannot align the input/output of your dataset with the model. <b>Pleaser double check your model and dataset.</b>
                             """
 
 CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            Sorry, we cannot align the input/output of your dataset with the model. Pleaser double check your model and dataset.
                             """
 
 CHECK_CONFIG_OR_SPLIT_RAW = """
                             Please check your dataset config or split.
                             """
 
-CHECK_LOG_SECTION_RAW = """Your have successfully submitted a Giskard evaluation job. Further details are available in the Logs tab. You can find your report posted in your model's community discussion section."""
-
 PREDICTION_SAMPLE_MD = """
                         <h1 style="text-align: center;">
                         Model Prediction Sample
                         </h1>
-                        Here's a sample of your model's prediction on an example from the dataset.
+                        Here is a sample prediction from your model based on your dataset.
                         """
 
 MAPPING_STYLED_ERROR_WARNING = """
-                        <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
-                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                        <h3 style="text-align: center;color: coral; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
+                        Sorry, we cannot auto-align the labels/features of your dataset and model. Please double check.
                         </h3>
                         """
 
-UNMATCHED_MODEL_DATASET_STYLED_ERROR = """
-                        <h3 style="text-align: center;color: #fa5f5f; background-color: #fbe2e2; border-radius: 8px; padding: 10px; ">
-                        ❌ Your model and dataset have different numbers of labels. Please double check your model and dataset.
-                        </h3>
-                        """
-
-NOT_FOUND_MODEL_RAW = """
-    We cannot find your model on Hugging Face. Please check that the model id is correct and publicly accessible.
-"""
-
-NOT_TEXT_CLASSIFICATION_MODEL_RAW = """
-                        Your model does not fall under the category of text classification. This page is specifically designated for the evaluation of text classification models.  
-                      """
-
-USE_INFERENCE_API_TIP = """
-                We use the <b>free</b>
-                <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
-                    Hugging Face Inference API
-                </a>
-                to evaluate the models with a <a href="https://huggingface.co/docs/hub/security-tokens#user-access-tokens">Hugging Face acces token</a>.
-                <br/>
-                Please <a href="https://huggingface.co/settings/tokens">get your token</a> and input here. This would help us to avoid the API calling rate limit and make the evaluation faster.
-                <br/>
-            """
-
-USE_INFERENCE_API_NOTICE = """
-                <b>Notice:</b>
-                <ul>
-                    <li><b>Your token will only be used for your own evaluation, and will not be saved.</b></li>
-                    <li><b>You will not be charged for the free Hugging Face Inference API.</b></li>
-                </ul>
-"""
-
-HF_TOKEN_INVALID_STYLED = """
-                <p style="text-align: left;color: red; ">
-                Your Hugging Face token is invalid. Please double check your token.
-                </p>
-                """
-
-VALIDATED_MODEL_DATASET_STYLED = """
-            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
-            Your model and dataset have been validated!
+def get_styled_input(input):
+    return f"""<h3 style="text-align: center;color: #5ec26a; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Sample input: {input}
             </h3>"""
-
-
-NOT_FOUND_DATASET_RAW = """
-    We cannot find your dataset on Hugging Face. Please check that the dataset id is correct and publicly accessible.
-"""
-
-def get_dataset_fetch_error_raw(error):
-    return f"""Sorry you cannot use this dataset because of an unknwon error: "{error}"."""