GSK-2396-allow-edit-feature-mappings (#12)

- GSK-2396 allow edit feature mapping and scan config (63bdb5b0f808d5f54ef4f7ec38eab4296aa55e64)
- fix markdown format (5de0e8aa7f04632fa6fc1dd9702aa6c9fa4af6ce)
- change comfirm label title to comfirm preprocess (a48ba214ef410e0df91d89a43e5ebe47ea10e1f6)
- update inference api arg in yaml; update error handling (09f3a52e92201eeeb56dfc0481687f35f1f65f9f)
- change model_type to inference_type (263c1739af61a30f017cba1c7a9b28cd5f8f307f)
- change inf type to k-v pair (384deb64de767f55e948d83fb210a8cfa7ec7d83)


Co-authored-by: zcy <ZeroCommand@users.noreply.huggingface.co>

app.py

@@ -11,13 +11,12 @@ import json
 from transformers.pipelines import TextClassificationPipeline
 
 from text_classification import check_column_mapping_keys_validity, text_classification_fix_column_mapping
-
+from utils import read_scanners, write_scanners, read_inference_type, write_inference_type, convert_column_mapping_to_json
 
 HF_REPO_ID = 'HF_REPO_ID'
 HF_SPACE_ID = 'SPACE_ID'
 HF_WRITE_TOKEN = 'HF_WRITE_TOKEN'
 
-
 theme = gr.themes.Soft(
     primary_hue="green",
 )
@@ -70,6 +69,7 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
             gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
+            gr.update(visible=False),       # feature mapping preview
         )
     if isinstance(ppl, Exception):
         gr.Warning(f'Failed to load model": {ppl}')
@@ -80,6 +80,7 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
             gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
+            gr.update(visible=False),       # feature mapping preview
         )
 
     # Validate dataset
@@ -105,7 +106,7 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
             gr.update(visible=False),       # Model prediction input
             gr.update(visible=False),       # Model prediction preview
             gr.update(visible=False),       # Label mapping preview
-            # gr.update(visible=True),        # Column mapping
+            gr.update(visible=False),       # feature mapping preview
         )
 
     # TODO: Validate column mapping by running once
@@ -118,21 +119,21 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
         except Exception:
             column_mapping = {}
 
-        column_mapping, prediction_input, prediction_result, id2label_df = \
+        column_mapping, prediction_input, prediction_result, id2label_df, feature_df = \
             text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split)
 
         column_mapping = json.dumps(column_mapping, indent=2)
 
-    if prediction_result is None:
+    if prediction_result is None and id2label_df is not None:
         gr.Warning('The model failed to predict with the first row in the dataset. Please provide column mappings in "Advance" settings.')
         return (
             gr.update(interactive=False),   # Submit button
-            gr.update(visible=True),       # Loading row
-            gr.update(visible=False),        # Preview row
-            gr.update(visible=False),       # Model prediction input
+            gr.update(visible=False),       # Loading row
+            gr.update(visible=True),        # Preview row
+            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
             gr.update(visible=False),   # Model prediction preview
-            gr.update(visible=False),   # Label mapping preview
-            # gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
+            gr.update(value=id2label_df, visible=True, interactive=True),   # Label mapping preview
+            gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
         )
     elif id2label_df is None:
         gr.Warning('The prediction result does not conform the labels in the dataset. Please provide label mappings in "Advance" settings.')
@@ -142,8 +143,8 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
             gr.update(visible=True),        # Preview row
             gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
             gr.update(value=prediction_result, visible=True),   # Model prediction preview
-            gr.update(visible=False),   # Label mapping preview
-            # gr.update(value=column_mapping, visible=True, interactive=True),    # Column mapping
+            gr.update(visible=True, interactive=True),   # Label mapping preview
+            gr.update(visible=True, interactive=True),   # feature mapping preview
         )
 
     gr.Info("Model and dataset validations passed. Your can submit the evaluation task.")
@@ -155,13 +156,18 @@ def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_ma
         gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
         gr.update(value=prediction_result, visible=True),   # Model prediction preview
         gr.update(value=id2label_df, visible=True, interactive=True), # Label mapping preview
+        gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
     )
 
 
-def try_submit(m_id, d_id, config, split, column_mappings, local):
+def try_submit(m_id, d_id, config, split, id2label_mapping_dataframe, feature_mapping_dataframe, local):
     label_mapping = {}
-    for i, label in column_mappings["Model Prediction Labels"].items():
+    for i, label in id2label_mapping_dataframe["Model Prediction Labels"].items():
         label_mapping.update({str(i): label})
+    
+    feature_mapping = {}
+    for i, feature in feature_mapping_dataframe["Dataset Features"].items():
+        feature_mapping.update({feature_mapping_dataframe["Model Input Features"][i]: feature})
 
     # TODO: Set column mapping for some dataset such as `amazon_polarity`
 
@@ -178,8 +184,9 @@ def try_submit(m_id, d_id, config, split, column_mappings, local):
             "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
             "--output_format", "markdown",
             "--output_portal", "huggingface",
-            # TODO: "--feature_mapping", json.dumps(column_mapping),
+            "--feature_mapping", json.dumps(feature_mapping),
             "--label_mapping", json.dumps(label_mapping),
+            "--scan_config", "./config.yaml",
         ]
 
         eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
@@ -221,12 +228,15 @@ with gr.Blocks(theme=theme) as iface:
                 gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
                 pass
         
-        def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split, id2label_mapping_dataframe=None):
+        def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split, id2label_mapping_dataframe=None, feature_mapping_dataframe=None):
             column_mapping = '{}'
-            m_id, ppl = check_model(model_id=model_id)
+            _, ppl = check_model(model_id=model_id)
 
             if id2label_mapping_dataframe is not None:
-                column_mapping = id2label_mapping_dataframe.to_json(orient="split")
+                labels = convert_column_mapping_to_json(id2label_mapping_dataframe.value, label="data")
+                features = convert_column_mapping_to_json(feature_mapping_dataframe.value, label="text")
+                column_mapping = json.dumps({**labels, **features}, indent=2)
+
             if check_column_mapping_keys_validity(column_mapping, ppl) is False:
                 gr.Warning('Label mapping table has invalid contents. Please check again.')
                 return (gr.update(interactive=False), 
@@ -234,18 +244,18 @@ with gr.Blocks(theme=theme) as iface:
                         gr.update(),
                         gr.update(),
                         gr.update(),
+                        gr.update(),
                         gr.update())
             else:
                 if model_id and dataset_id and dataset_config and dataset_split:
-                    return try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping)
+                    return try_validate(model_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping)
                 else:
-                    del ppl
-
                     return (gr.update(interactive=False), 
                             gr.update(visible=True),
                             gr.update(visible=False),
                             gr.update(visible=False),
                             gr.update(visible=False),
+                            gr.update(visible=False),
                             gr.update(visible=False))
         with gr.Row():
             gr.Markdown('''
@@ -256,6 +266,13 @@ with gr.Blocks(theme=theme) as iface:
                 ''')
         with gr.Row():
             run_local = gr.Checkbox(value=True, label="Run in this Space")
+            use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
+            run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
+        
+        with gr.Row() as advanced_row:
+            selected = read_scanners('./config.yaml')
+            scan_config = selected + ['data_leakage']
+            scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
 
         with gr.Row():
             model_id_input = gr.Textbox(
@@ -271,30 +288,32 @@ with gr.Blocks(theme=theme) as iface:
             dataset_config_input = gr.Dropdown(['default'], value='default', label='Dataset Config', visible=False)
             dataset_split_input = gr.Dropdown(['default'], value='default', label='Dataset Split', visible=False)
 
-            dataset_id_input.change(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
-            dataset_config_input.change(
+            dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+            dataset_id_input.submit(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+
+            dataset_config_input.blur(
                 check_dataset_and_get_split, 
                 inputs=[dataset_config_input, dataset_id_input], 
                 outputs=[dataset_split_input])
         
         with gr.Row(visible=True) as loading_row:
             gr.Markdown('''
-                        <h1 style="text-align: center;">
-                        Please validate your model and dataset first...
-                        </h1>
+                        <p style="text-align: center;">
+                        🚀🐢Please validate your model and dataset first...
+                        </p>
                         ''')
-
+            
         with gr.Row(visible=False) as preview_row:
             gr.Markdown('''
                 <h1 style="text-align: center;">
-                Confirm Label Details
+                Confirm Pre-processing Details
                 </h1>
-                Base on your model and dataset, we inferred this label mapping. **If the mapping is incorrect, please modify it in the table below.**
+                Base on your model and dataset, we inferred this label mapping and feature mapping. <b>If the mapping is incorrect, please modify it in the table below.</b>
                 ''')
         
         with gr.Row():
             id2label_mapping_dataframe = gr.DataFrame(label="Preview of label mapping", interactive=True, visible=False)
-
+            feature_mapping_dataframe = gr.DataFrame(label="Preview of feature mapping", interactive=True, visible=False)
         with gr.Row():
             example_input = gr.Markdown('Sample Input: ', visible=False)
         
@@ -308,22 +327,30 @@ with gr.Blocks(theme=theme) as iface:
             size="lg",
         )
 
-        model_id_input.change(gate_validate_btn, 
+        model_id_input.blur(gate_validate_btn, 
                                 inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
-        dataset_id_input.change(gate_validate_btn, 
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+        dataset_id_input.blur(gate_validate_btn, 
                                 inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe])
-        dataset_config_input.change(gate_validate_btn,
+                                outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+        dataset_config_input.input(gate_validate_btn,
                                 inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
-        dataset_split_input.change(gate_validate_btn,
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+        dataset_split_input.input(gate_validate_btn,
                                 inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
         id2label_mapping_dataframe.input(gate_validate_btn,
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe])
-        
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+        feature_mapping_dataframe.input(gate_validate_btn,
+                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
+                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+        scanners.change(write_scanners, inputs=scanners)
+        run_inference.change(
+            write_inference_type,
+            inputs=[run_inference]
+        )
+
         run_btn.click(
             try_submit,
             inputs=[
@@ -332,6 +359,7 @@ with gr.Blocks(theme=theme) as iface:
                 dataset_config_input,
                 dataset_split_input,
                 id2label_mapping_dataframe,
+                feature_mapping_dataframe,
                 run_local,
             ],
             outputs=[

config.yaml

@@ -0,0 +1,9 @@
+detectors:
+  - ethical_bias
+  - text_perturbation
+  - robustness
+  - performance
+  - underconfidence
+  - overconfidence
+  - spurious_correlation
+inference_type: hf_pipeline

text_classification.py

@@ -19,9 +19,8 @@ def text_classification_map_model_and_dataset_labels(id2label, dataset_features)
             continue
         if len(feature.names) != len(id2label_mapping.keys()):
             continue
-
+        
         dataset_labels = feature.names
-
         # Try to match labels
         for label in feature.names:
             if label in id2label_mapping.keys():
@@ -31,10 +30,23 @@ def text_classification_map_model_and_dataset_labels(id2label, dataset_features)
                 model_label, label = text_classificaiton_match_label_case_unsensative(id2label_mapping, label)
             if model_label is not None:
                 id2label_mapping[model_label] = label
+            else:
+                print(f"Label {label} is not found in model labels")
 
     return id2label_mapping, dataset_labels
 
-
+'''
+params:
+    column_mapping: dict
+    example: {
+        "text": "sentences",
+        "label": {
+            "label0": "LABEL_0",
+            "label1": "LABEL_1"
+        }
+    }
+    ppl: pipeline
+'''
 def check_column_mapping_keys_validity(column_mapping, ppl):
     # get the element in all the list elements
     column_mapping = json.loads(column_mapping)
@@ -48,19 +60,10 @@ def check_column_mapping_keys_validity(column_mapping, ppl):
     
     return user_labels == model_labels == original_labels
 
-
-def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
-    # We assume dataset is ok here
-    ds = datasets.load_dataset(d_id, config)[split]
-
-    try:
-        dataset_features = ds.features
-    except AttributeError:
-        # Dataset does not have features, need to provide everything
-        return None, None, None
-
+def infer_text_input_column(column_mapping, dataset_features):
     # Check whether we need to infer the text input column
     infer_text_input_column = True
+    feature_map_df = None
     if "text" in column_mapping.keys():
         dataset_text_column = column_mapping["text"]
         if dataset_text_column in dataset_features.keys():
@@ -71,12 +74,26 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
     if infer_text_input_column:
         # Try to retrieve one
         candidates = [f for f in dataset_features if dataset_features[f].dtype == "string"]
+        feature_map_df = pd.DataFrame({
+            "Dataset Features": [candidates[0]],
+            "Model Input Features": ["text"]
+        })
         if len(candidates) > 0:
             logging.debug(f"Candidates are {candidates}")
             column_mapping["text"] = candidates[0]
-        else:
-            # Not found a text feature
-            return column_mapping, None, None
+    
+    return column_mapping, feature_map_df 
+
+def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
+    # We assume dataset is ok here
+    ds = datasets.load_dataset(d_id, config)[split]
+    try:
+        dataset_features = ds.features
+    except AttributeError:
+        # Dataset does not have features, need to provide everything
+        return None, None, None, None, None
+
+    column_mapping, feature_map_df = infer_text_input_column(column_mapping, dataset_features)
 
     # Load dataset as DataFrame
     df = ds.to_pandas()
@@ -85,24 +102,13 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
     id2label_mapping = {}
     id2label = ppl.model.config.id2label
     label2id = {v: k for k, v in id2label.items()}
-    prediction_input = None
-    prediction_result = None
-    try:
-        # Use the first item to test prediction
-        prediction_input = df.head(1).at[0, column_mapping["text"]]
-        results = ppl({"text": prediction_input}, top_k=None)
-        prediction_result = {
-            f'{result["label"]}({label2id[result["label"]]})': result["score"] for result in results
-        }
-    except Exception:
-        # Pipeline prediction failed, need to provide labels
-        return column_mapping, None, None
 
     # Infer labels
     id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(id2label, dataset_features)
     id2label_mapping_dataset_model = {
         v: k for k, v in id2label_mapping.items()
     }
+
     if "data" in column_mapping.keys():
         if isinstance(column_mapping["data"], list):
             # Use the column mapping passed by user
@@ -112,15 +118,31 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
         column_mapping["label"] = {
             i: None for i in id2label.keys()
         }
-        return column_mapping, prediction_result, None
+        return column_mapping, None, None, None, feature_map_df
 
-    prediction_result = {
-        f'[{label2id[result["label"]]}]{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result["score"] for result in results
-    }
     id2label_df = pd.DataFrame({
         "Dataset Labels": dataset_labels,
         "Model Prediction Labels": [id2label_mapping_dataset_model[label] for label in dataset_labels],
     })
+    
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        prediction_input = df.head(1).at[0, column_mapping["text"]]
+        results = ppl({"text": prediction_input}, top_k=None)
+        prediction_result = {
+            f'{result["label"]}({label2id[result["label"]]})': result["score"] for result in results
+        }
+    except Exception as e:
+        # Pipeline prediction failed, need to provide labels
+        print(e, '>>>> error')
+        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+    
+    prediction_result = {
+        f'[{label2id[result["label"]]}]{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result["score"] for result in results
+    }
 
     if "data" not in column_mapping.keys():
         # Column mapping should contain original model labels
@@ -128,4 +150,4 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
             str(i): id2label_mapping_dataset_model[label] for i, label in zip(id2label.keys(), dataset_labels)
         }
 
-    return column_mapping, prediction_input, prediction_result, id2label_df
+    return column_mapping, prediction_input, prediction_result, id2label_df, feature_map_df

utils.py

@@ -0,0 +1,54 @@
+import yaml
+
+YAML_PATH = "./config.yaml"
+
+class Dumper(yaml.Dumper):
+    def increase_indent(self, flow=False, *args, **kwargs):
+        return super().increase_indent(flow=flow, indentless=False)
+    
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(path):
+    scanners = []
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", None)
+    return scanners
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners):
+    with open(YAML_PATH, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+
+    config["detectors"] = scanners
+    with open(YAML_PATH, "w") as f:
+        # save scanners to detectors in yaml
+        yaml.dump(config, f, Dumper=Dumper)
+
+# read model_type from yaml file
+def read_inference_type(path):
+    inference_type = ""
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        inference_type = config.get("inference_type", None)
+    return inference_type
+
+# write model_type to yaml file
+def write_inference_type(use_inference):
+    with open(YAML_PATH, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+    if use_inference:
+        config["inference_type"] = 'hf_inference_api'
+    else:
+        config["inference_type"] = 'hf_pipeline'
+    with open(YAML_PATH, "w") as f:
+        # save inference_type to inference_type in yaml
+        yaml.dump(config, f, Dumper=Dumper)
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping