fix bugs and add logs for leaderboard tab

app_leaderboard.py

@@ -5,9 +5,11 @@ from fetch_utils import check_dataset_and_get_config, check_dataset_and_get_spli
 
 def get_records_from_dataset_repo(dataset_id):
     dataset_config = check_dataset_and_get_config(dataset_id)
+
     logging.info(f"Dataset {dataset_id} has configs {dataset_config}")
     dataset_split = check_dataset_and_get_split(dataset_id, dataset_config[0])
     logging.info(f"Dataset {dataset_id} has splits {dataset_split}")
+
     try:
         ds = datasets.load_dataset(dataset_id, dataset_config[0])[dataset_split[0]]
         df = ds.to_pandas()
@@ -40,13 +42,14 @@ def get_types(ds):
 def get_display_df(df):
     # style all elements in the model_id column
     display_df = df.copy()
-    if display_df['model_id'].any():
+    columns = display_df.columns.tolist()
+    if 'model_id' in columns:
         display_df['model_id'] = display_df['model_id'].apply(lambda x: f'<p href="https://huggingface.co/{x}" style="color:blue">🔗{x}</p>')
     # style all elements in the dataset_id column
-    if display_df['dataset_id'].any():
+    if 'dataset_id' in columns:
         display_df['dataset_id'] = display_df['dataset_id'].apply(lambda x: f'<p href="https://huggingface.co/datasets/{x}" style="color:blue">🔗{x}</p>')
     # style all elements in the report_link column
-    if display_df['report_link'].any():
+    if 'report_link' in columns:
         display_df['report_link'] = display_df['report_link'].apply(lambda x: f'<p href="{x}" style="color:blue">🔗{x}</p>')
     return display_df
 
@@ -57,7 +60,7 @@ def get_demo():
     dataset_ids = get_dataset_ids(records)
 
     column_names = records.columns.tolist()
-    default_columns = ['model_id', 'dataset_id', 'total_issue', 'report_link']
+    default_columns = ['model_id', 'dataset_id', 'total_issues', 'report_link']
     # set the default columns to show
     default_df = records[default_columns]
     types = get_types(default_df)
@@ -79,7 +82,7 @@ def get_demo():
            outputs=[leaderboard_df])
     def filter_table(model_id, dataset_id, columns, task):
         # filter the table based on task
-        df = records[(records['hf_pipeline_type'] == task)]
+        df = records[(records['task'] == task)]
         # filter the table based on the model_id and dataset_id
         if model_id:
             df = records[(records['model_id'] == model_id)]

app_text_classification.py

@@ -11,7 +11,7 @@ from transformers.pipelines import TextClassificationPipeline
 
 from text_classification import get_labels_and_features_from_dataset, check_model, get_example_prediction
 from io_utils import read_scanners, write_scanners, read_inference_type, read_column_mapping, write_column_mapping, write_inference_type
-from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW
+from wordings import INTRODUCTION_MD, CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW
 
 HF_REPO_ID = 'HF_REPO_ID'
 HF_SPACE_ID = 'SPACE_ID'
@@ -95,7 +95,7 @@ def check_dataset_and_get_split(dataset_id, dataset_config):
 
 def get_demo():
     with gr.Row():
-        gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+        gr.Markdown(INTRODUCTION_MD)
     with gr.Row():
         model_id_input = gr.Textbox(
             label="Hugging Face model id",
@@ -117,13 +117,17 @@ def get_demo():
         example_prediction = gr.Label(label='Model Prediction Sample', visible=False)
     
     with gr.Row():
-        column_mappings = []
-        with gr.Column():
-            for _ in range(MAX_LABELS):
-                column_mappings.append(gr.Dropdown(visible=False))
-        with gr.Column():    
-            for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
-                column_mappings.append(gr.Dropdown(visible=False))
+        with gr.Accordion(label='Label and Feature Mapping', visible=False, open=False) as column_mapping_accordion:
+            with gr.Row():
+                gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+            column_mappings = []
+            with gr.Row():
+                with gr.Column():
+                    for _ in range(MAX_LABELS):
+                        column_mappings.append(gr.Dropdown(visible=False))
+                with gr.Column():    
+                    for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
+                        column_mappings.append(gr.Dropdown(visible=False))
     
     with gr.Accordion(label='Model Wrap Advance Config (optional)', open=False):
         run_local = gr.Checkbox(value=True, label="Run in this Space")
@@ -182,7 +186,7 @@ def get_demo():
     
     @gr.on(triggers=[model_id_input.change, dataset_config_input.change, dataset_split_input.change],
         inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-        outputs=[example_input, example_prediction, *column_mappings])
+        outputs=[example_input, example_prediction, column_mapping_accordion, *column_mappings])
     def check_model_and_show_prediction(model_id, dataset_id, dataset_config, dataset_split):
         ppl = check_model(model_id)
         if ppl is None or not isinstance(ppl, TextClassificationPipeline):
@@ -207,12 +211,14 @@ def get_demo():
             return (
                 gr.update(visible=False),
                 gr.update(visible=False),
+                gr.update(visible=False, open=False),
                 *column_mappings
             )
         prediction_input, prediction_output = get_example_prediction(ppl, dataset_id, dataset_config, dataset_split)
         return (
             gr.update(value=prediction_input, visible=True),
             gr.update(value=prediction_output, visible=True),
+            gr.update(open=False),
             *column_mappings
         )
 
@@ -223,10 +229,16 @@ def get_demo():
         inputs=[dataset_id_input, dataset_config_input], 
         outputs=[dataset_split_input])
 
+    scanners.change(
+        write_scanners,
+        inputs=scanners
+    )
+
     run_inference.change(
         write_inference_type,
         inputs=[run_inference]
     )
+
     gr.on(
         triggers=[
             run_btn.click,

fetch_utils.py

@@ -1,5 +1,6 @@
 import huggingface_hub
 import datasets
+import logging
 
 def check_dataset_and_get_config(dataset_id):
     try:
@@ -12,12 +13,14 @@ def check_dataset_and_get_config(dataset_id):
 def check_dataset_and_get_split(dataset_id, dataset_config):
     try:
         ds = datasets.load_dataset(dataset_id, dataset_config)
-    except Exception:
+    except Exception as e:
         # Dataset may not exist
+        logging.warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
         return None
     try:
         splits = list(ds.keys())
         return splits
-    except Exception:
+    except Exception as e:
         # Dataset has no splits
+        logging.warning(f"Dataset {dataset_id} with config {dataset_config} has no splits: {e}")
         return None

wordings.py

@@ -1,10 +1,15 @@
-CONFIRM_MAPPING_DETAILS_MD = '''
+INTRODUCTION_MD = '''
                 <h1 style="text-align: center;">
-                Giskard Evaluator
+                🐢Giskard Evaluator
                 </h1>
                 Welcome to Giskard Evaluator Space! Get your report immediately by simply input your model id and dataset id below. Follow our leads and improve your model in no time.
                 '''
-
+CONFIRM_MAPPING_DETAILS_MD = '''
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            Please confirm the pre-processing details below. If you are not sure, please double check your model and dataset.
+                            '''
 CONFIRM_MAPPING_DETAILS_FAIL_MD = '''
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details