Pulling out year-text tokenization into sep function. Improving docs. Changing prefix name order.

app.py

@@ -82,7 +82,7 @@ assert label_list[0] == LABEL_DICT["female"], "LABEL_DICT not an ordered dict"
 
 label2id = {label: idx for idx, label in enumerate(label_list)}
 
-# Prepare text
+
 def tokenize_and_append_metadata(text, tokenizer):
     tokenized = tokenizer(
         text,
@@ -90,6 +90,7 @@ def tokenize_and_append_metadata(text, tokenizer):
         padding=True,
         max_length=MAX_TOKEN_LENGTH,
     )
+    """Tokenize text and mask/flag 'gendered_tokens_ids' in token_ids and labels."""
 
     # Finding the gender pronouns in the tokens
     token_ids = tokenized["input_ids"]
@@ -133,35 +134,46 @@ def tokenize_and_append_metadata(text, tokenizer):
     return tokenized
 
 
-# Run inference
-def predict_gender_pronouns(
-    num_points, conditioning_variables, f_weights, bert_like_models, input_text
-):
-
+def get_tokenized_text_with_years(years, input_text):
+    """Construct dict of tokenized texts with each year injected into the text."""
     text_portions = input_text.split(SPLIT_KEY)
 
-    years = np.linspace(START_YEAR, STOP_YEAR, int(num_points)).astype(int)
-    num_preds = None
-
-    dfs = []
-    dfs.append(pd.DataFrame({"year": years}))
-
-    tokenized = {'ids':[], 'atten_mask':[], 'toks':[], 'labels':[]}
+    tokenized_w_year = {'ids':[], 'atten_mask':[], 'toks':[], 'labels':[]}
     for b_date in years:
+
         target_text = f"{b_date}".join(text_portions)
         tokenized_sample = tokenize_and_append_metadata(
             target_text,
             tokenizer=tokenizer,    
         )
 
-        tokenized['ids'].append(tokenized_sample["input_ids"])
-        tokenized['atten_mask'].append(torch.tensor(tokenized_sample["attention_mask"]))
-        tokenized['toks'].append(tokenizer.convert_ids_to_tokens(tokenized_sample["input_ids"]))
-        tokenized['labels'].append(tokenized_sample["labels"])
+        tokenized_w_year['ids'].append(tokenized_sample["input_ids"])
+        tokenized_w_year['atten_mask'].append(torch.tensor(tokenized_sample["attention_mask"]))
+        tokenized_w_year['toks'].append(tokenizer.convert_ids_to_tokens(tokenized_sample["input_ids"]))
+        tokenized_w_year['labels'].append(tokenized_sample["labels"])
+
+    # Also returning last `target_text`` to display as example text
+    return tokenized_w_year, target_text  
 
+
+def predict_gender_pronouns(
+    num_points, conditioning_variables, f_weights, bert_like_models, input_text
+):
+    """Run inference on input_text for each model type, returning df and plots of precentage
+    of gender pronouns predicted as female and male in each target text.
+    """
+
+    years = np.linspace(START_YEAR, STOP_YEAR, int(num_points)).astype(int)
+
+    tokenized, target_text = get_tokenized_text_with_years(years, input_text)
+    is_masked = tokenized['ids'][0] == MASK_TOKEN_ID
+    num_preds = torch.sum(is_masked).item()
+
+    dfs = []
+    dfs.append(pd.DataFrame({"year": years}))
     for f_weight in f_weights:
         for var in conditioning_variables:
-            prefix = f"w{f_weight}_{var}"
+            prefix = f"{var}_w{f_weight}"
             model = models[(var, f_weight)]
 
             p_female = []
@@ -175,11 +187,8 @@ def predict_gender_pronouns(
                     outputs = model(ids.unsqueeze(dim=0), atten_mask.unsqueeze(dim=0))
                     preds = torch.argmax(outputs[0][0].cpu(), dim=1)
 
-                    was_masked = labels.cpu() != -100
-                    preds = torch.where(was_masked, preds, -100)
-
-                    if not num_preds:
-                        num_preds = torch.sum(was_masked).item()
+                    #was_masked = labels.cpu() != -100
+                    preds = torch.where(is_masked, preds, -100)
 
                     p_female.append(len(torch.where(preds == 0)[0]) / num_preds * 100)
                     p_male.append(len(torch.where(preds == 1)[0]) / num_preds * 100)
@@ -240,28 +249,28 @@ def predict_gender_pronouns(
 title = "Changing Gender Pronouns"
 description = """
 <h2> Intro </h2>
-This is a demo for a project exploring possible spurious correlations in training datasets that can be exploited and manipulated to achieve alternative outcomes. In this case, a user can demo what context changes will cause predicted gender pronouns to change, in a range of models. 
-
-In a user provided sentence, with at least one reference to a `DATE` and one gender pronoun, we will see how sweeping through a range of `DATE` values can change the predicted pronouns. 
+This is a demo for a project exploring possible spurious correlations that have been learned by our models. We can examine the training datasets and learning tasks to hypothesize what spurious correlations may exist, then condition on these variables to determine if we can achieve alternative outcomes. 
 
-We see this in both the BERT base model and a model fine-tuned with a specific pronoun predicting task on the [wiki-bio](https://huggingface.co/datasets/wiki_bio) dataset.
+Specially in this demo: In a user provided sentence, with at least one reference to a `DATE` and one gender pronoun, we will see how sweeping through a range of `DATE` values can change the predicted pronouns. This effect can be observed in BERT base models and in our fine-tuned models (with a specific pronoun predicting task on the [wiki-bio](https://huggingface.co/datasets/wiki_bio) dataset).
 
-One way to explain this phenomena is by looking at a likely  data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG. 
+One way to explain this phenomena is by looking at a likely data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG. 
  
 <h2> Causal DAG </h2> 
-In the DAG, we can see that `birth_place`, `birth_date` and `gender` are all independent elements that have no common cause with the other covariates in the DAG. However `birth_place`, `birth_date` and `gender` may all have a role in causing one's `access_to_resources`, with the general trend that `access_to_resources` has become less gender-dependent over time, but not in every `birth_place`, with recent events in Afghanistan providing a stark counterexample to this trend. `access_to_resources` further determines how or if at all, you may appear in the dataset’s `context_words`.
+In the DAG, we can see that `birth_place`, `birth_date` and `gender` are all independent elements that have no common cause with the other covariates in the DAG. However `birth_place`, `birth_date` and `gender` may all have a role in causing one's `access_to_resources`, with the general trend that `access_to_resources` has become less gender-dependent over time, but not in every `birth_place`, with recent events in Afghanistan providing a stark counterexample to this trend. Importantly, `access_to_resources` determines how, **if at all**, you may appear in the dataset's `context_words`.
  
-We also argue that although there are complex causal interactions between words in a segment, the `context_words` are more likely to cause the `gender_pronouns`, rather than vice versa. For example, if the subject is a famous doctor and the object is her wealthy father, these context words will determine which person is being referred to, and thus which gendered-pronoun to use.
+We argue that although there are complex causal interactions between each words in any given sentence, the `context_words` are more likely to cause the `gender_pronouns`, rather than vice versa. For example, if the subject is a famous doctor and the object is her wealthy father, these context words will determine which person is being referred to, and thus which gendered-pronoun to use.
  
  
-In this graph, any pink path between `context_words` and `gender_pronouns` will allow the flow of statistical correlation (regardless of direction of the causal arrow), inviting confounding and thus spurious correlations into the trained model.
+In this graph, arrow heads are intended to show the assumed direction of caustion. E.g. as descriped above, we are claiming `context_words` cause the `gender_pronouns`. While causation follow direction of the arrows, statistical correlation can flow in any direction (it is cause-agnostic). 
+
+In the case of this graph, any pink path between `context_words` and `gender_pronouns` will allow the flow of statistical correlation, inviting confounding and thus spurious correlations into the trained model.
  
 <center>
 <img src="https://www.dropbox.com/s/x60r43h7uwztnru/generic_ds_dag.png?raw=1" 
     alt="DAG of possible data generating process for datasets used in training.">
 </center>
   
-Those familiar with causal DAGs may note when can simply condition on `gender` to block any confounding between the `context_words` and the `gender_pronouns`.  However, this is not always possible, particularly in generative or mask-filling tasks, like those common in language models and in the demo below.  
+Those familiar with causal DAGs may note when can simply condition on `gender` to block any confounding between the `context_words` and the `gender_pronouns`.  However, this is not always possible, particularly in generative or mask-filling tasks where gender may be unknown, common in language modeling and in the demo below.  
  
  <h2> How to use this demo </h2> 
 In this demo, a user can add any sentence that contains at least one gender pronoun and the capitalized word `DATE`. We then sweep through a range of `date` values in the place of `DATE`, while masking (for prediction) the gender pronouns (included in the list below).
@@ -281,9 +290,7 @@ In addition to chosing the test sentence, we ask that you pick how the fine-tune
 - conditioning variable: which, if any, conditioning variable from the three noted above in the DAG, was included in the text at train time.
 - loss function weight: weight assigned to the minority class (female pronouns in this fine-tuning dataset) that was included in the text at train time.
 
-
-
-
+You can also optionally pick a bert-like model for comparison.
 
  <h2> What are the results</h2> 
 
@@ -293,9 +300,11 @@ In the resulting plots, we can look for a dose-response relationship between:
  
 Specifically we are seeing if making larger magnitude intervention: an older `DATE` in the text will result in a larger magnitude effect in the outcome: higher percentage of predicted female pronouns.
 
-One trend that appears is: conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to 'stratify' a learned relationship between gender pronouns and dates, when both are present in the text.
-While conditioning on either no metadata or `birth_place` data training, have similar middle-ground effects for this inference task. 
-Finally,  conditioning on `name` metadata in training, (while again conditioning on `date` in inference) has almost no dose-response relationship. It appears the learning of a `name —> gender pronouns` relationship was sufficiently successful to overwhelm any potential more nuanced learning, such as that driven by `birth_date` or `place`. 
+- One trend that appears is: conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to 'stratify' a learned relationship between gender pronouns and dates, when both are present in the text.
+- While conditioning on either no metadata or `birth_place` data training, have similar middle-ground effects for this inference task. 
+- Finally,  conditioning on `name` metadata in training, (while again conditioning on `date` in inference) has almost no dose-response relationship. It appears the learning of a `name —> gender pronouns` relationship was sufficiently successful to overwhelm any potential more nuanced learning, such as that driven by `birth_date` or `place`. 
+
+Please feel free to ping me on the Hugging Face discord (I'm 'emily_learner' there), with any feedback/comments/concerns or interesting findings!
 """
 
 
@@ -313,23 +322,23 @@ gr.Interface(
             CONDITIONING_VARIABLES,
             default=["none", "birth_date"],
             type="value",
-            label="Pick conditioning variable included in text during fine-tuning.",
+            label="(1) Pick conditioning variable included in text during fine-tuning.",
         ),
         gr.inputs.CheckboxGroup(
             FEMALE_WEIGHTS,
             default=[5],
             type="value",
-            label="Pick loss function weight placed on female predictions  during fine-tuning.",
+            label="(2) Pick loss function weight placed on female predictions  during fine-tuning.",
         ),
         gr.inputs.CheckboxGroup(
             BERT_LIKE_MODELS,
             default=["bert"],
             type="value",
-            label="Pick optional bert-like base uncased model for comparison.",
+            label="(Optional) Pick bert-like base uncased model for comparison.",
         ),
         gr.inputs.Textbox(
             lines=7,
-            label="Input Text. Include one of more instance of the word 'DATE' below, to be replace with a range of dates in demo.",
+            label="Input Text: Include one of more instance of the word 'DATE' below (to be replace with a range of dates in demo), and one of more gender pronoun (to be masked for prediction).",
             default="Born DATE, she was a computer scientist. Her work was greatly respected, and she was well-regarded in her field.",
         ),
     ],
@@ -356,3 +365,4 @@ gr.Interface(
     description=description,
     article=article,
 ).launch()
+