Push dashboard

PR_ENT.py

@@ -0,0 +1,165 @@
+import streamlit as st
+
+from PIL import Image
+
+from transformers import pipeline
+import nltk
+import spacy
+import en_core_web_lg
+
+from helpers import prompt_to_nli, display_nli_pr_results_as_list
+
+@st.cache()
+def download_punkt():
+    nltk.download('punkt')
+
+@st.cache(allow_output_mutation=True)
+def load_spacy_pipeline():
+    return en_core_web_lg.load()
+
+def choose_text_menu(text):
+    if 'text' not in st.session_state:
+        st.session_state.text = 'Several demonstrators were injured.'
+    text = st.text_area('Event description', st.session_state.text)
+
+    return text
+
+# Load Models in cache
+
+@st.cache(allow_output_mutation=True)
+def load_model_prompting():
+    return pipeline("fill-mask", model="distilbert-base-uncased")
+
+@st.cache(allow_output_mutation=True)
+def load_model_nli():
+    return pipeline(task="sentiment-analysis", model="roberta-large-mnli")
+
+download_punkt()
+nlp = load_spacy_pipeline()
+
+### App START
+st.markdown("""# Rethinking the Event Coding Pipeline with Prompt Entailment
+## Author: Anonymized for submission""")
+st.markdown(""" 
+### 1. PR-ENT summary
+""")
+
+@st.cache()
+def load_prent_image():
+    return Image.open('pipeline_flow.png')
+st.image(load_prent_image(), caption="""PR-ENT Flow. First, we concatenate the event description e and the template t. Then we feed them through a pretrained
+prompting model to obtain a list of answer candidates. Then, for each answer candidate we build a 
+hypothesis by filling the template and check for entailment with the premise (the event description). 
+Finally, by filtering on the entailment score, we obtain a list of entailed answer candidates related to the event description.
+""")
+
+
+
+model_nli = load_model_nli()
+model_prompting = load_model_prompting()
+
+st.markdown("""
+### 2. Write an event description:
+The first step is to write an event description that will be fed to the pipeline. This can be any text in English.
+""")
+text = choose_text_menu('')
+st.session_state.text = text
+
+st.markdown(""" 
+### 3. Template design:
+The second step is to design a template while keeping in mind the objective of the classification.
+- A good starting point is to use `This event involves [Z].`. This template will ideally be filled with a 1 word summary of the event.
+- Another good example is `People were [Z].`. With this one we mostly expect a verb that describes the action.
+  You can also use any template you design. Keep in mind that if the masked slot `[Z]` is at the end of the sentence, to not forget punctuation,
+  otherwise the model may fill the template with punctuation signs.
+""")
+
+if 'prompt' not in st.session_state:
+    st.session_state.prompt = 'This event involves [Z].'
+prompt = st.text_input('Template:',st.session_state.prompt)
+st.session_state.prompt = prompt
+
+
+st.markdown("""
+### 4. Select the two parameters:
+- The first parameter `top_k` is the maximum number of tokens that will be given by the prompting model. 
+  It's also the number of tokens that will be tried for entailment. Ideally, you want a high enough number of tokens, otherwise you may miss critical information.
+  However, each additional token will increase the computation time as it needs to go through the entailment model. 
+  From our experiments, a good choice is between `[10,50]`, lower and you miss information, higher and you start getting unrelated tokens and long computation time.
+- The second parameter is the minimum entailment score to confirm that the token is entailed with the event description. 
+  By default, we set it at `0.5` (more entailed than not) but it can be modified depending on needs.
+""")
+
+def select_top_k():
+    if 'top_k' not in st.session_state:
+        st.session_state.top_k = 10
+
+    return st.number_input('Number of max tokens to output (default: 10, min: 0, max: 50)? ',step = 100, min_value=0, max_value=50, value=int(st.session_state.top_k))
+
+
+def select_nli_limit():
+    if 'nli_limit' not in st.session_state:
+        st.session_state.nli_limit = 0.5
+
+    return st.number_input('Minimum score of entailment (default: 0.5, min: 0, max: 1)? ',step = 100.0, min_value=0.0, max_value=1.0, value=st.session_state.nli_limit)
+
+def update_session_state_callback(value, key):
+    st.session_state[value] = st.session_state[key]
+
+top_k = select_top_k()
+st.session_state.top_k = top_k
+
+nli_limit = select_nli_limit()
+st.session_state.nli_limit = nli_limit
+
+st.markdown("""
+### 5. Remove similar tokens from output:
+An additional option is to remove similar tokens (e.g. `protest, protests`) from the output. 
+This computes the lemma of each word (based on the template) and removes duplicate lemmas.
+""")
+if 'remove_lemma' not in st.session_state:
+    st.session_state.remove_lemma = False
+remove_lemma = st.checkbox('Remove similar lemma (e.g. protest, protests) from output?', value= st.session_state.remove_lemma)
+st.session_state.remove_lemma = remove_lemma
+
+
+# Save settings to display before the results
+if "old_prompt" not in st.session_state:
+    st.session_state.old_text =st.session_state.text
+    st.session_state.old_prompt =st.session_state.prompt
+    st.session_state.old_top_k = st.session_state.top_k
+    st.session_state.old_nli_limit = st.session_state.nli_limit  
+
+st.markdown("""
+### 6. Run the pipeline
+""")
+
+st.markdown("""The entailed tokens are given as a list of words associated with the probability of entailment.""")
+
+if st.button("Run PR-ENT"):
+    computation_state_prent = st.text("PR-ENT Computation Running.")
+    st.session_state.old_text =st.session_state.text
+    st.session_state.old_prompt =st.session_state.prompt
+    st.session_state.old_top_k = st.session_state.top_k
+    st.session_state.old_nli_limit = st.session_state.nli_limit
+
+    # Replace the mask
+    prompt = prompt.replace('[Z]', '{}')
+    prompt = prompt.replace('[MASK]', '{}')
+    results = prompt_to_nli(text, prompt, model_prompting, model_nli, nlp, top_k, nli_limit, remove_lemma)
+    list_results = [x[0][0] + ' ' + str(int(x[1][1]*100)) + '%' for x in results]
+    st.session_state.list_results = list_results
+
+    computation_state_prent.text("PR-ENT Computation Done.")
+
+if 'list_results' in st.session_state:
+    st.write('**Event Description**: {}'.format(st.session_state.old_text))
+    st.write('**Template**: "{}"; **Top K**: {}; **Entailment Threshold**: {}.'.format(st.session_state.old_prompt,st.session_state.old_top_k, st.session_state.old_nli_limit))
+    display_nli_pr_results_as_list('', st.session_state.list_results)
+
+st.markdown("""
+### 7. Actor-target coding (experimental)
+
+Available in actor-target tab (on the left)
+
+""")

README.md

@@ -1,11 +1,12 @@
 ---
-title: PRENT Demo
-emoji: 👁
-colorFrom: blue
-colorTo: purple
+title: PR-ENT Dashboard
+emoji: 💻
+colorFrom: green
+colorTo: yellow
 sdk: streamlit
+python_version: 3.8.9
 sdk_version: 1.10.0
-app_file: app.py
+app_file: PR_ENT.py
 pinned: false
 ---
 

helpers.py

@@ -0,0 +1,234 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+
+from nltk.tokenize import sent_tokenize
+
+# Split the text into sentences. Necessary for NLI models
+def split_sentences(text):
+    return sent_tokenize(text)
+
+###### Prompting
+def query_model_prompting(model, text, prompt_with_mask, top_k, targets):
+    """Query the prompting model
+
+    :param model: Prompting model object
+    :type model: Huggingface pipeline object
+    :param text: Event description (context)
+    :type text: str
+    :param prompt_with_mask: Prompt with a mask
+    :type prompt_with_mask: str
+    :param top_k: Number of tokens to output
+    :type top_k: integer
+    :param targets: Restrict the answer to these possible tokens
+    :type targets: list
+    :return: Results of the prompting model
+    :rtype: list of dict
+    """
+    sequence = text + prompt_with_mask
+    output_tokens = model(sequence, top_k=top_k, targets=targets)
+    
+    return output_tokens
+
+def do_sentence_entailment(sentence, hypothesis, model):
+    """Concatenate context and hypothesis then perform entailment
+
+    :param sentence: Event description (context), 1 sentence
+    :type sentence: str
+    :param hypothesis: Mask filled with a token
+    :type hypothesis: str
+    :param model: NLI Model
+    :type model: Huggingface pipeline
+    :return: DataFrame containing the result of the entailment
+    :rtype: pandas DataFrame
+    """
+    text = sentence + '</s></s>' + hypothesis
+    res = model(text, return_all_scores=True)
+    df_res = pd.DataFrame(res[0])
+    df_res['label'] = df_res['label'].apply(lambda x: x.lower())
+    df_res.columns = ["Label", "Score"]
+    return df_res
+
+def softmax(x):
+    """Compute softmax values for each sets of scores in x."""
+    return np.exp(x) / np.sum(np.exp(x), axis=0)
+
+
+
+######### NLI + PROMPTING
+def do_text_entailment(text, hypothesis, model):
+    """
+    Do entailment for each sentence of the event description as
+    model was trained on sentence pair
+
+    :param text: Event Description (context)
+    :type text: str
+    :param hypothesis: Mask filled with a token
+    :type hypothesis: str
+    :param model: Model NLI
+    :type model: Huggingface pipeline
+    :return: List of entailment results for each sentence of the text
+    :rtype: list
+    """
+    text_entailment_results = []
+    for i, sentence in enumerate(split_sentences(text)):
+        df_score = do_sentence_entailment(sentence, hypothesis, model)
+        text_entailment_results.append((sentence, hypothesis, df_score))
+    return text_entailment_results
+
+def get_true_entailment(text_entailment_results, nli_limit):
+    """
+    From the result of each sentence entailment, extract the maximum entailment score and 
+    check if it's higher than the entailment threshold. 
+    """
+    true_hypothesis_list = []
+    max_score = 0
+    for sentence_entailment in text_entailment_results:
+        df_score = sentence_entailment[2]
+        score = df_score[df_score["Label"] == 'entailment']["Score"].values.max()
+        if score > max_score:
+            max_score = score
+    if max_score > nli_limit:
+        true_hypothesis_list.append((sentence_entailment[1], np.round(max_score,2)))
+    return list(set(true_hypothesis_list))
+
+def prompt_to_nli(text, prompt, model_prompting, nli_model, nlp, top_k=10, nli_limit=0.5, remove_lemma=False):
+    """
+    Apply the PR-ENT pipeline
+    
+    :param text: Event description
+    :type text: str
+    :param prompt: Prompt with mask
+    :type prompt: str
+    :param model_prompting: Prompting Model
+    :type model_prompting: Huggingface pipeline
+    :param nli_model: NLI Model
+    :type nli_model: Huggingface pipeline
+    :param top_k: Number of words output by the prompting model
+    :type top_k: int
+    :param nli_limit: Entailment threshold
+    :type nli_limit: float
+
+    :return: Results of the pipeline
+    :rtype: list
+    """
+    prompt_masked = prompt.format(model_prompting.tokenizer.mask_token)
+    label = []
+    output_prompting = query_model_prompting(model_prompting, text, prompt_masked, top_k, targets=None)
+    if remove_lemma:
+        output_prompting = filter_prompt_output_by_lemma(prompt, output_prompting, nlp)
+    for token in output_prompting:
+        hypothesis = prompt.format(token['token_str'])
+        text_entailment_results = do_text_entailment(text, hypothesis, nli_model)
+        true_hypothesis_list = get_true_entailment(text_entailment_results, nli_limit)
+        if len(true_hypothesis_list) > 0:
+            label.append(((token['token_str'], token['score']), true_hypothesis_list[0]))
+    return label
+
+
+def display_nli_pr_results_as_list(title, list_results):
+    """
+        Display the list of entailment results as a streamlit choice list
+    """
+    st.markdown(
+        """
+    <style>
+    span[data-baseweb="tag"] {
+    background-color: red !important;
+    }
+    </style>
+    """,
+        unsafe_allow_html=True,
+    )
+    prompt_list = st.multiselect(
+        title,
+        list_results 
+        ,
+        list_results, key='results_mix')
+
+
+##### QA
+def question_answering(model, text, questions_list, to_print=True):
+    """
+    Apply question answering model
+
+    :param model: QA Model
+    :type model: Huggingface pipeline
+    :param text: Event description (context)
+    :type text: str
+    :param question: Question to answer
+    :type question: str
+    :return: Tuple containing the answer and the confidence score
+    :rtype: tuple
+    """
+    for question in questions_list:
+        QA_input = {
+        'question': question,
+        'context': text}
+        res = model(QA_input, handle_impossible_answer=False)
+
+        if to_print:
+            st.write("Question: {}".format(question))
+            st.write("Answer: {}".format(res["answer"]))
+
+    return res["answer"], res["score"]
+
+
+### Prompt + NLI + QA
+
+def get_who_what_whom_qa(text, tokens, model_qa):
+    who_what_whom = []
+    if not tokens:
+        res_dict = {"Actor":'', "Action":'', "Target": ''}
+        st.write("No entailed tokens.")
+
+    else:
+        for token in tokens:
+            # res_dict = {"who":'', "did_what":token, "to_whom": '', "qa_score": []}
+            res_dict = {"Actor":'', "Action":token, "Target": ''}
+
+            if token[-3:] == 'ing':
+                perp,score_p = question_answering(model_qa, text, ["Who was {}?".format(token)], to_print=False)
+            else:
+                perp,score_p = question_answering(model_qa, text, ["Who {} people?".format(token)], to_print=False)
+            if perp:
+                res_dict["Actor"] = perp + ' [' + str(np.round(score_p*100,1)) + '%]'
+            else:
+                res_dict["Actor"] = 'N/A' + ' [' + str(np.round(score_p*100,1)) + '%]'
+
+            victim,score_v = question_answering(model_qa, text, ["Who was {}?".format(token)], to_print=False)         
+
+            if victim:
+                res_dict["Target"] = victim + ' [' + str(np.round(score_v*100,1)) + '%]'
+            else:
+                res_dict["Target"] = 'N/A' + ' [' + str(np.round(score_v*100,1)) + '%]'
+            
+            who_what_whom.append(res_dict)
+
+    return who_what_whom
+
+def remove_similar_lemma_from_list(prompt, list_words, nlp):
+    ## Compute a dictionnary with the lemma for all tokens
+    ## If there is a duplicate lemma then the dictionnary value will be a list of the corresponding tokens
+    lemma_dict = {}
+    for each in list_words:
+        mask_filled = nlp(prompt.strip('.').format(each))
+        lemma_dict.setdefault([x.lemma_ for x in mask_filled][-1],[]).append(each)
+
+    ## Get back the list of tokens
+    ## If multiple tokens available then take the shortest one
+    new_token_list = []
+    for key in lemma_dict.keys():
+        if len(lemma_dict[key]) >= 1:
+            new_token_list.append(min(lemma_dict[key], key=len))
+        else:
+            raise ValueError("Lemma dict has 0 corresponding words")
+    return new_token_list
+
+def filter_prompt_output_by_lemma(prompt, output_prompting, nlp):
+    """
+        Remove all similar lemmas from the prompt output (e.g. "protest", "protests")
+    """
+    list_words = [x['token_str'] for x in output_prompting]
+    new_token_list = remove_similar_lemma_from_list(prompt, list_words, nlp)
+    return [x for x in output_prompting if x['token_str'] in new_token_list]

pages/Actor_Target.py

@@ -0,0 +1,107 @@
+import sys
+import os
+
+import streamlit as st
+
+from PIL import Image
+import pandas as pd
+
+from transformers import pipeline
+import spacy
+import en_core_web_lg
+
+current = os.path.dirname(os.path.realpath(__file__))
+parent = os.path.dirname(current)
+sys.path.append(parent)
+from helpers import display_nli_pr_results_as_list, prompt_to_nli, get_who_what_whom_qa
+
+@st.cache(allow_output_mutation=True)
+def load_spacy_pipeline():
+    return en_core_web_lg.load()
+
+def choose_text_menu(text):
+    if 'text' not in st.session_state:
+        st.session_state.text = 'Several demonstrators were injured.'
+    text = st.text_area('Event description', st.session_state.text)
+
+    return text
+
+# # Load Models in cache
+@st.cache(allow_output_mutation=True)
+def load_model_prompting():
+    return pipeline("fill-mask", model="distilbert-base-uncased")
+
+@st.cache(allow_output_mutation=True)
+def load_model_nli():
+    return pipeline(task="sentiment-analysis", model="roberta-large-mnli")
+
+@st.cache(allow_output_mutation=True)
+def load_model_qa():
+    model_name = "deepset/roberta-base-squad2"
+    model = pipeline(model=model_name, tokenizer=model_name, task="question-answering")
+    return model
+
+nlp = load_spacy_pipeline()
+
+### App START
+st.markdown("""# Rethinking the Event Coding Pipeline with Prompt Entailment
+## Author: Anonymized for submission""")
+
+st.markdown("### 1. Actor-target coding (experimental):")
+@st.cache()
+def load_qa_image():
+    return Image.open('pipeline_qa.png')
+st.image(load_qa_image(),caption="""Actor-target Coding Flow. First we get the entailed answer candidates through the PR-ENT pipeline. 
+Then we construct questions based on these tokens to extract actors and targets, 2 questions per verb. 
+Finally, we pass these questions and event description to a pre-trained extractive question answering model and fill a table of [Actor, Action, Target].""")
+
+st.markdown("""
+Here we use an extractive question answering model to find the actor and target of an event.
+As this is still in experimental phase, there are some limitations:
+- The only template possible is `People were [Z].`, this allows us to get a verb to construct the two questions:
+- `Who was [Z]?` to find the target.
+- `Who [Z] people?` to find the actor.
+- `top_k = 10` and `entailment_threshold = 0.5`.
+The results of the QA are given along the confidence score of the model in brackets `[xx.x%]`
+""")
+
+## Load Models
+model_nli = load_model_nli()
+model_prompting = load_model_prompting()
+model_qa = load_model_qa()
+
+
+
+st.markdown("""
+### 2. Write an event description:
+The first step is to write an event description that will be fed to the pipeline. This can be any text in English.
+""")
+text = choose_text_menu('')
+st.session_state.text = text
+
+
+st.markdown("""
+### 3. Run actor-target coding:
+""")
+
+if "old_text_qa" not in st.session_state:
+    st.session_state.old_text_qa =st.session_state.text
+
+qa_button = st.button("Run actor-target coding")
+if qa_button:
+    computation_state_qa = st.text("Computation Running.")
+    st.session_state.old_text_qa =st.session_state.text
+    prompt = "People were {}."
+    results = prompt_to_nli(text, prompt, model_prompting, model_nli, nlp, 10, 0.5, True)
+    list_results = [x[0][0] + ' ' + str(int(x[1][1]*100)) + '%' for x in results]
+    st.session_state.list_results_prompt_qa = list_results
+    list_tokens = [x[0][0] for x in results]
+    who_what_whom = get_who_what_whom_qa(text, list_tokens, model_qa)
+    st.session_state.who_what_whom = who_what_whom
+    computation_state_qa.text("Computation Done.")
+
+if 'who_what_whom' in st.session_state:
+    st.write('**Event Description**: {}'.format(st.session_state.old_text_qa))
+    st.write('**Template**: "{}"; **Top K**: {}; **Entailment Threshold**: {}.'.format("People were [Z]",10, 0.5))
+    display_nli_pr_results_as_list('', st.session_state.list_results_prompt_qa)
+    st.write(pd.DataFrame(st.session_state.who_what_whom))

requirements.txt

@@ -0,0 +1,8 @@
+# see environments.yml
+numpy==1.22.3
+pandas==1.4.2
+spacy==3.2.3
+https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+transformers[torch]==4.19.2
+nltk==3.7
+sentence_transformers==2.2.0