new version

demo.py

@@ -16,32 +16,34 @@ from utils.visualize import visualize_spans
 #     "packages/en_engagement_RoBERTa-0.0.2/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.2"
 # )
 
-#Load from local storage
-#MODEL_LIST = ['en_engagement_RoBERTa-ME-AtoE.tar.gz']
+# Load from local storage
+# MODEL_LIST = ['en_engagement_RoBERTa-ME-AtoE.tar.gz']
 
-#model = st.selectbox('Select model', MODEL_LIST, index=0)
-#nlp = spacy.load("packages/" + model)
+# model = st.selectbox('Select model', MODEL_LIST, index=0)
+# nlp = spacy.load("packages/" + model)
 
 # Load from huggingface
 # sm = spacy.load('en_core_web_sm', disable=['ner'])
 
-st.set_page_config(page_title="ENGAGEMENT analyzer (beta ver 0.3)",
-                   layout="wide",
-                   initial_sidebar_state="expanded")
+st.set_page_config(
+    page_title="ENGAGEMENT analyzer (beta ver 0.3)",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
 
 
-@st.cache(allow_output_mutation=True)
+@st.cache_resource()
 def load_model():
     # nlp = spacy.load("en_engagement_RoBERTa_context_flz")
     nlp = spacy.load("en_engagement_LSTM")
     # nlp = spacy.load("en_engagement_spl_RoBERTa_base_attention")
-    return (nlp)
+    return nlp
 
 
 nlp = load_model()
 
 doc = nlp(
-    'Welcome! Probably this is one of the few attempts to teach a machine how to read the discourse...! Although it is not perfect, you should be able to get a good place to start for your stance-taking analyses. The result will be presented here.'
+    "Welcome! Probably this is one of the few attempts to teach a machine how to read the discourse...! Although it is not perfect, you should be able to get a good place to start for your stance-taking analyses. The result will be presented here."
 )
 
 # TPL_ENT = """
@@ -93,44 +95,44 @@ TPL_SPAN_START = """
 DEFAULT_TEXT = """Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia."""
 
 TEXT_LIST = [
-    '''To a significant extent, individuals can be considered responsible for the rise of Hitler to power on the 31st of January, 1933. Hitler himself, the charismatic leader of the Nazi Party, as well as creator of Nazi policy, played a key role in his own rise to power. However, other individuals in government, such as Hindenburg and von Papen were influential in Hitler’s rise. To a small extent, other factors also enabled Hitler to rise to power such as the Depression and the weakness of the political system. Nevertheless to a significant extent, individuals can be held responsible for the rise of Adolf Hitler to power.''',
-    '''Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia.''',
-    '''Certainly, the argumentation is not without some faults. For example, the statement that “linking homosexuality to witches fulfills the same purpose” is not supported by references to the readings. It is not clear who was linking homosexuality to witches and in what context. Nevertheless, overall and in line with the general tendencies reported in the previous section, the author employs various contracting and expanding engagement resources successfully. However, a large part of the successful use of engagement resources seems to be related to how the author structures these strategies throughout the text, namely in a wave-like fashion: from acknowledging the opinions of others, to countering them by offering one’s own interpretation, to supporting it by acknowledging other sources.''',
-    '''As the centuries passed, accounts of witchcraft became more and more specific; details of witches’ ceremonies and oaths became more concrete and whatever the condemned humans confessed to was treated as fact. As discussants correctly pointed out, Bernardino of Siena, Martin Le Franc, and the anonymous author of the Errores Gazariorum all have an even more aggressive campaign against witches than did the authors of our previous readings. By depicting their rituals and customs, they look to paint the most grotesque picture of witches possible. Their frenzied accusations, were some of the main catalysts of the subsequent witch hunts.''',
-    '''The post labeled “Witchcraft as a Problem in Society” clearly explains the contribution that each text makes to the witch hunts. While two of the authors focused on describing, in full detail, the shocking and disturbing practices that witches partook of, the others tried to prove that the witch threat was real. These last texts sought to explain witchcraft so as to convince readers that witches actually existed. As all posts reiterate, the devil is definitely at the source of witchcraft.''',
-    '''The third part temporarily puts aside mediation analysis and shifts the discussion to moderation analysis. In Chapter 7, I show how a multiple regression model can be made more flexible by allowing one variable’s effect to depend linearly on another variable in the model. The resulting moderated multiple regression model allows an investigator to ascertain the extent to which X’s influence on outcome variable Y is contingent on or interacts with a moderator variable W.''',
-    '''For instance, research has shown that people have a tendency to justify close others’ unethical actions to protect them (Gino and Galinsky 2012). Research has also shown that parents who feel close to their children often adopt strict curfew practices (Elder et al., 1995). (EC-33)''',
-    '''Fitzpatrick and Pagani (2013) found that engagement skills in classroom behaviour at kindergarten were related with better math scores and academic success. (LC-0525-EN)''',
-    '''The COAG Reform Council (2013) indicated that when compared to other students, Australian Year 4 students who attended one year of ECEC services or programs gained 11 points higher in reading (LC-0471-MA). Preliminary evidence suggests that teaching children from low-income families using humanoid robots increases motivation, sense of community, and self-expression... (EC-64). These findings suggest that visual perception takes up only a small fraction of fixation durations. Specifically, Verdelhan (2010) proposes a two-country, one-good model in which each country has an exogenously specified i.i.d. consumption growth process. Waters & Baur (2003) suggest that children or adolescents who are overweight or obese suffer from social and psychological issues. (LC-0460-EN)''',
-    '''According to the Australian Bureau of Statistics (2008), the percentage of obese or overweight adults is a staggering 60%.
+    """To a significant extent, individuals can be considered responsible for the rise of Hitler to power on the 31st of January, 1933. Hitler himself, the charismatic leader of the Nazi Party, as well as creator of Nazi policy, played a key role in his own rise to power. However, other individuals in government, such as Hindenburg and von Papen were influential in Hitler’s rise. To a small extent, other factors also enabled Hitler to rise to power such as the Depression and the weakness of the political system. Nevertheless to a significant extent, individuals can be held responsible for the rise of Adolf Hitler to power.""",
+    """Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia.""",
+    """Certainly, the argumentation is not without some faults. For example, the statement that “linking homosexuality to witches fulfills the same purpose” is not supported by references to the readings. It is not clear who was linking homosexuality to witches and in what context. Nevertheless, overall and in line with the general tendencies reported in the previous section, the author employs various contracting and expanding engagement resources successfully. However, a large part of the successful use of engagement resources seems to be related to how the author structures these strategies throughout the text, namely in a wave-like fashion: from acknowledging the opinions of others, to countering them by offering one’s own interpretation, to supporting it by acknowledging other sources.""",
+    """As the centuries passed, accounts of witchcraft became more and more specific; details of witches’ ceremonies and oaths became more concrete and whatever the condemned humans confessed to was treated as fact. As discussants correctly pointed out, Bernardino of Siena, Martin Le Franc, and the anonymous author of the Errores Gazariorum all have an even more aggressive campaign against witches than did the authors of our previous readings. By depicting their rituals and customs, they look to paint the most grotesque picture of witches possible. Their frenzied accusations, were some of the main catalysts of the subsequent witch hunts.""",
+    """The post labeled “Witchcraft as a Problem in Society” clearly explains the contribution that each text makes to the witch hunts. While two of the authors focused on describing, in full detail, the shocking and disturbing practices that witches partook of, the others tried to prove that the witch threat was real. These last texts sought to explain witchcraft so as to convince readers that witches actually existed. As all posts reiterate, the devil is definitely at the source of witchcraft.""",
+    """The third part temporarily puts aside mediation analysis and shifts the discussion to moderation analysis. In Chapter 7, I show how a multiple regression model can be made more flexible by allowing one variable’s effect to depend linearly on another variable in the model. The resulting moderated multiple regression model allows an investigator to ascertain the extent to which X’s influence on outcome variable Y is contingent on or interacts with a moderator variable W.""",
+    """For instance, research has shown that people have a tendency to justify close others’ unethical actions to protect them (Gino and Galinsky 2012). Research has also shown that parents who feel close to their children often adopt strict curfew practices (Elder et al., 1995). (EC-33)""",
+    """Fitzpatrick and Pagani (2013) found that engagement skills in classroom behaviour at kindergarten were related with better math scores and academic success. (LC-0525-EN)""",
+    """The COAG Reform Council (2013) indicated that when compared to other students, Australian Year 4 students who attended one year of ECEC services or programs gained 11 points higher in reading (LC-0471-MA). Preliminary evidence suggests that teaching children from low-income families using humanoid robots increases motivation, sense of community, and self-expression... (EC-64). These findings suggest that visual perception takes up only a small fraction of fixation durations. Specifically, Verdelhan (2010) proposes a two-country, one-good model in which each country has an exogenously specified i.i.d. consumption growth process. Waters & Baur (2003) suggest that children or adolescents who are overweight or obese suffer from social and psychological issues. (LC-0460-EN)""",
+    """According to the Australian Bureau of Statistics (2008), the percentage of obese or overweight adults is a staggering 60%.
     According to George et al. (2011), in the UK immigration has improved the academic performance of the native children.
-    According to UNICEF (2011) a child that is breastfed within the first hour of life is fourteen times less likely to die from diarrhoea or pneumonia.''',
-    '''As far as I am concerned, I do think globalization is good chance for China’s developing. From my point of view, I prefer to think that advantages of globalization outweighs disadvantages. ''',
-    '''As we know, China has made great progress for these years. I think it is the result of globalization. We all know China is a fast-developing country. We can seethe great progress that China has made. ''',
-    '''His idea was that an important ninth century bishop called John Anglicus may indeed have given birth to a child in full view of everyone on the streets of Rome, but that this bishop was not and never had been the pope. Of course, there is no evidence whatever for this, as Leibnitz himself well knew.''',
-    '''On the whole, however, when evaluating meanings metaphorically, the Chinese EFL learners hedge and qualify their statements subjectively, tempering the certainty and authority of their assertions rather than using the resources of interpersonal metaphor to reinforce and substantiate their arguments. These tendencies reveal a key area for pedagogical intervention. Namely, instruction could focus on the value of construing metaphors objectively to obscure the author as the source of the evaluation. Similarly, raising students’ awareness of the space of negotiation and the value of offering assertions on a cline of certainty (e.g., IT IS EVIDENT) rather than through exclusive declarations of shared knowledge (e.g., AS WE ALL KNOW) is critical for academic writing refinement. Instructional interventions such as these are key areas for further investigation.''',
-    '''Of the defendants involved in Utah Pie Company’s case only one seems to have emerged as exceptionally successful. However this success was not a factor of overwhelming market power, as can be seen by the dominant position of Mrs. Smith’s during this time, which had maintained a 39-45 percent market share over the corresponding period.''',
-    '''Because of the evidence presented by Tremblay and Tremblay, it would appear that mergers in the brewing industry would have been procompetitive because of economies of scale. However, allowing a firm to acquire more than 20% of the market in Wisconsin would give it too much power to charge higher prices, even if the merger would help lower total average costs.''',
-    '''Taken in whole, the economic evidence for grocery retailers in the decades after the Von’s decision suggests that increased concentration is pro-competitive and good for consumers, running contrary to the fears proposed by the Court.''',
-    '''The remedies that Justice Lewis Powell prescribed did not gain the desired effect, and I feel that they were not very effective in promoting competition. (Elan, S86)''',
-    '''There is the possibility for abuse if the producer sets different maximum prices for different retailers, allowing some to reap higher profits.''',
-    '''Such a program, with appropriate limits, would provide a balanced structure that would ensure quality patient care.''',
-    '''A recent survey of physician satisfaction by Harvard Medical School found that physician autonomy and the ability to provide high-quality care, not income, are most strongly associated with changes in job satisfaction . Thus, it seems reasonable to assume that health care providers would take advantage of the greater bargaining power to improve the quality of care. (Ken, S78-79)''',
-    '''It appears, then, that maximum price fixing does the greatest harm when set below a competitive level [evidentialize]. In Case 4 it could potentially do harm to small retailers trying to enter the market [suggest], but does so for the benefit of consumers and the producer. Based purely on the models, it appears that, at the very least, maximum prices deserve a Rule of Reason approach to evaluate their cost and benefits.''',
-    '''It could be seen that for this 68% of the respondents, Tampines was characteristically a location that provided for them all their basic needs. It can be seen from chart [11] that many people quoted accessibility and proximity to home, and even shopping as one of the ideal factors that drew them there. Accessibility is quite a key factor because it is evident that the regional centre was built on the basis of good infrastructure. In comparison, 32% of the respondents felt that the conventional downtown was still a major attraction, even though the regional centre had gained quite a vast amount of popularity and did to large extent have an air of modernity.'''
+    According to UNICEF (2011) a child that is breastfed within the first hour of life is fourteen times less likely to die from diarrhoea or pneumonia.""",
+    """As far as I am concerned, I do think globalization is good chance for China’s developing. From my point of view, I prefer to think that advantages of globalization outweighs disadvantages. """,
+    """As we know, China has made great progress for these years. I think it is the result of globalization. We all know China is a fast-developing country. We can seethe great progress that China has made. """,
+    """His idea was that an important ninth century bishop called John Anglicus may indeed have given birth to a child in full view of everyone on the streets of Rome, but that this bishop was not and never had been the pope. Of course, there is no evidence whatever for this, as Leibnitz himself well knew.""",
+    """On the whole, however, when evaluating meanings metaphorically, the Chinese EFL learners hedge and qualify their statements subjectively, tempering the certainty and authority of their assertions rather than using the resources of interpersonal metaphor to reinforce and substantiate their arguments. These tendencies reveal a key area for pedagogical intervention. Namely, instruction could focus on the value of construing metaphors objectively to obscure the author as the source of the evaluation. Similarly, raising students’ awareness of the space of negotiation and the value of offering assertions on a cline of certainty (e.g., IT IS EVIDENT) rather than through exclusive declarations of shared knowledge (e.g., AS WE ALL KNOW) is critical for academic writing refinement. Instructional interventions such as these are key areas for further investigation.""",
+    """Of the defendants involved in Utah Pie Company’s case only one seems to have emerged as exceptionally successful. However this success was not a factor of overwhelming market power, as can be seen by the dominant position of Mrs. Smith’s during this time, which had maintained a 39-45 percent market share over the corresponding period.""",
+    """Because of the evidence presented by Tremblay and Tremblay, it would appear that mergers in the brewing industry would have been procompetitive because of economies of scale. However, allowing a firm to acquire more than 20% of the market in Wisconsin would give it too much power to charge higher prices, even if the merger would help lower total average costs.""",
+    """Taken in whole, the economic evidence for grocery retailers in the decades after the Von’s decision suggests that increased concentration is pro-competitive and good for consumers, running contrary to the fears proposed by the Court.""",
+    """The remedies that Justice Lewis Powell prescribed did not gain the desired effect, and I feel that they were not very effective in promoting competition. (Elan, S86)""",
+    """There is the possibility for abuse if the producer sets different maximum prices for different retailers, allowing some to reap higher profits.""",
+    """Such a program, with appropriate limits, would provide a balanced structure that would ensure quality patient care.""",
+    """A recent survey of physician satisfaction by Harvard Medical School found that physician autonomy and the ability to provide high-quality care, not income, are most strongly associated with changes in job satisfaction . Thus, it seems reasonable to assume that health care providers would take advantage of the greater bargaining power to improve the quality of care. (Ken, S78-79)""",
+    """It appears, then, that maximum price fixing does the greatest harm when set below a competitive level [evidentialize]. In Case 4 it could potentially do harm to small retailers trying to enter the market [suggest], but does so for the benefit of consumers and the producer. Based purely on the models, it appears that, at the very least, maximum prices deserve a Rule of Reason approach to evaluate their cost and benefits.""",
+    """It could be seen that for this 68% of the respondents, Tampines was characteristically a location that provided for them all their basic needs. It can be seen from chart [11] that many people quoted accessibility and proximity to home, and even shopping as one of the ideal factors that drew them there. Accessibility is quite a key factor because it is evident that the regional centre was built on the basis of good infrastructure. In comparison, 32% of the respondents felt that the conventional downtown was still a major attraction, even though the regional centre had gained quite a vast amount of popularity and did to large extent have an air of modernity.""",
 ]
 
 
-@st.cache(suppress_st_warning=True)
+@st.cache_resource()
 def preprocess(text):
-    text = re.sub("\n\n", ' &&&&&&&&#&#&#&#&', text)
-    text = re.sub('\n', ' ', text)
-    text = re.sub('\s+', " ", text)
-    text = re.sub('&&&&&&&&#&#&#&#&', '\n\n', text)
+    text = re.sub("\n\n", " &&&&&&&&#&#&#&#&", text)
+    text = re.sub("\n", " ", text)
+    text = re.sub("\s+", " ", text)
+    text = re.sub("&&&&&&&&#&#&#&#&", "\n\n", text)
     return text
 
 
-@st.cache(allow_output_mutation=True)
+@st.cache_resource()
 def delete_span(span_sc: dict):
     id_del = []
     for n, spn in enumerate(span_sc, start=1):
@@ -171,7 +173,7 @@ with st.sidebar:
 | `Monogloss`           | An utterance which does not employ any value of engagement. Such an utterance ignores the dialogic potential in an utterance.                                                                                                        |
 
     """)
-#For a more complete description of the category, visit [the annotation guideline](https://egumasa.github.io/engagement-annotation-project/3_Categories/)!!
+# For a more complete description of the category, visit [the annotation guideline](https://egumasa.github.io/engagement-annotation-project/3_Categories/)!!
 
 st.sidebar.markdown("""
     Engagement Analyzer is developed by [Masaki Eguchi](https://masakieguchi.weebly.com).
@@ -211,45 +213,45 @@ with st.expander("See more explanation"):
 
     """)
 
-st.info('''Updated on Jan.11th, 2023\n
+st.info("""Updated on Jan.11th, 2023\n
 The current version was trained on 2,519 sentences and tested on 443 sentences. It achieved the following benchmark:
 - Macro F1 = .75
 - Macro Precision = .78
 - Macro Recall = .74
 I expect that the model's performance improves as the annotated dataset gets larger.
-''')
+""")
 
 with st.form("my_form"):
-
     st.subheader("Option 1: selecting example text from list")
-    text = st.selectbox('', TEXT_LIST)
+    text = st.selectbox("", TEXT_LIST)
 
     st.subheader("Option 2: analyze your own text")
     input_text = st.text_area(
         label="",
-        value=
-        "I would strongly encourage you to put your texts here to analyze it for stance-taking expressions.",
-        height=120)
+        value="I would strongly encourage you to put your texts here to analyze it for stance-taking expressions.",
+        height=120,
+    )
     st.text(
-        'The text from the pull-down list and in the textbox cannot be analyzed at the same time. Please select the mode.'
+        "The text from the pull-down list and in the textbox cannot be analyzed at the same time. Please select the mode."
     )
 
     textmode = st.radio(
-        label='Choose the mode.',
-        options=['Option 1: Pull-down choice', 'Option 2: My own text'],
-        index=1)
+        label="Choose the mode.",
+        options=["Option 1: Pull-down choice", "Option 2: My own text"],
+        index=1,
+    )
 
     submitted = st.form_submit_button("Submit")
     if submitted:
-        if textmode == 'Option 2: My own text':
+        if textmode == "Option 2: My own text":
             text = input_text
-            with st.spinner('Analysis in progress...'):
+            with st.spinner("Analysis in progress..."):
                 doc = nlp(preprocess(text))
-            #st.markdown("> " + input_text)
+            # st.markdown("> " + input_text)
         else:
-            with st.spinner('Analysis in progress...'):
+            with st.spinner("Analysis in progress..."):
                 doc = nlp(preprocess(text))
-            #st.markdown("> " + text)
+            # st.markdown("> " + text)
 
 ## Dependency parsing
 
@@ -265,21 +267,21 @@ with st.form("my_form"):
 # st.write(text)
 # delete_span(doc.spans['sc'])
 
-cleanup_justify(doc, doc.spans['sc'])
-delete_overlapping_span(doc.spans['sc'])
+cleanup_justify(doc, doc.spans["sc"])
+delete_overlapping_span(doc.spans["sc"])
 
 visualize_spans(
     doc,
     spans_key="sc",
     displacy_options={
-        'template': {
+        "template": {
             "span": TPL_SPAN,
-            'slice': TPL_SPAN_SLICE,
-            'start': TPL_SPAN_START,
+            "slice": TPL_SPAN_SLICE,
+            "start": TPL_SPAN_START,
         },
         "colors": {
             "ENTERTAIN": "#82b74b",
-            "DENY": '#c94c4c',
+            "DENY": "#c94c4c",
             "COUNTER": "#eea29a",
             "PRONOUNCE": "#92a8d1",
             "ENDORSE": "#034f84",
@@ -291,10 +293,12 @@ visualize_spans(
             "CITATION": "#F8C471",
             "SOURCES": "#F7DC6F",
             "JUSTIFYING": "#2ECC71",
-            "ENDOPHORIC": "#FAD7A0"
+            "ENDOPHORIC": "#FAD7A0",
         },
     },
-    simple = True
+    simple=False,
+    show_diversity=True,
+    show_confidence=False,
 )
 
 st.subheader("Bibliography")
@@ -304,4 +308,9 @@ st.markdown("""
 * Ryshina-Pankova, M. (2014). Exploring academic argumentation in course-related blogs through ENGAGEMENT. In G. Thompson & L. Alba-Juez (Eds.), _Pragmatics & Beyond New Series (Vol. 242, pp. 281–302)_. John Benjamins Publishing Company. https://doi.org/10.1075/pbns.242.14rys
 * Wu, S. M. (2007). The use of engagement resources in high- and low-rated undergraduate geography essays. _Journal of English for Academic Purposes, 6_ (3), 254–271. https://doi.org/10.1016/j.jeap.2007.09.006
 
-""")
+""")
+
+st.subheader("Please cite the following papers:")
+st.markdown("""* Eguchi, M., & Kyle, K. (2023). Span Identification of Epistemic Stance-Taking in Academic Written English. Proceedings of the 18th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2023), 429–442. https://aclanthology.org/2023.bea-1.35
+* Eguchi, M., & Kyle, K. (2024). Building custom NLP tools to annotate discourse-functional features for second language writing research: A tutorial. *Research Methods in Applied Linguistics, 3*(3), 100153. https://doi.org/10.1016/j.rmal.2024.100153
+""")

pipeline/post_processors.py

@@ -1,4 +1,3 @@
-
 from typing import List, Sequence, Tuple, Optional, Dict, Union, Callable
 import pandas as pd
 import spacy
@@ -6,23 +5,38 @@ from spacy.language import Language
 from skbio import diversity as dv
 
 SPAN_ATTRS = ["text", "label_", "start", "end"]
-CATEGORIES = ['ATTRIBUTION', "CITATION", "COUNTER", "DENY", "ENDOPHORIC", "ENTERTAIN", "JUSTIFYING", "MONOGLOSS", "PROCLAIM", "SOURCES"]
-
-
-def simple_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
-                 spans_key: str = "sc",
-                 attrs: List[str] = SPAN_ATTRS):
+CATEGORIES = [
+    "ATTRIBUTION",
+    "CITATION",
+    "COUNTER",
+    "DENY",
+    "ENDOPHORIC",
+    "ENTERTAIN",
+    "JUSTIFYING",
+    "MONOGLOSS",
+    "PROCLAIM",
+    "SOURCES",
+]
+
+
+def simple_table(
+    doc: Union[spacy.tokens.Doc, Dict[str, str]],
+    spans_key: str = "sc",
+    attrs: List[str] = SPAN_ATTRS,
+):
     columns = attrs + ["Conf. score"]
     data = [
-        [str(getattr(span, attr))
-         for attr in attrs] + [score]  # [f'{score:.5f}']
-        for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores'])
+        [str(getattr(span, attr)) for attr in attrs] + [score]  # [f'{score:.5f}']
+        for span, score in zip(
+            doc.spans[spans_key], doc.spans[spans_key].attrs["scores"]
+        )
     ]
     return data, columns
 
 
 # def span_info_aggregator()
 
+
 def construction_classifier(doc, span):
     category = None
     spanroot = span.root
@@ -33,7 +47,6 @@ def construction_classifier(doc, span):
     span_token = [t.norm_ for t in span]
     span_tag = [t.tag_ for t in span]
 
-
     c = [c for c in spanroot.children]
     c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]
 
@@ -44,30 +57,65 @@ def construction_classifier(doc, span):
 
     right_dep = [c.dep_ for c in spanroot.rights]
 
-    #conditionals
-    subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
-    argmentless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in spanroot.children)
-    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
+    # conditionals
+    subjless = all(
+        c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+        for c in spanroot.children
+    )
+    argmentless = all(
+        c.dep_
+        not in [
+            "nsubj",
+            "nsubjpass",
+            "csubj",
+            "csubjpass",
+            "dobj",
+            "ccomp",
+            "xcomp",
+            "dative",
+            "attr",
+            "oprd",
+            "acomp",
+        ]
+        for c in spanroot.children
+    )
+    argless_span = all(
+        c.dep_
+        not in [
+            "nsubj",
+            "nsubjpass",
+            "csubj",
+            "csubjpass",
+            "dobj",
+            "ccomp",
+            "xcomp",
+            "dative",
+            "attr",
+            "oprd",
+            "acomp",
+        ]
+        for c in span
+    )
 
     ## nesting classifiers
     if spanroot.dep_ == "conj":
-        while spanroot.dep_ == 'conj':
+        while spanroot.dep_ == "conj":
             spanroot = spanroot.head
     # if spanroot.dep_ == "poss":
     #     while spanroot.dep_ == 'poss':
     #         spanroot = spanroot.head
 
-    ## Conjunctions 
+    ## Conjunctions
     # Preconjunctions
-    if spanroot.dep_ in ['preconj', 'cc']:
+    if spanroot.dep_ in ["preconj", "cc"]:
         category = "Conjunction"
 
     ## NOUN PHRASES
     # adverbial phrases
-    if spanroot.dep_ in ['amod']:
+    if spanroot.dep_ in ["amod"]:
         category = "Adjectival modifier"
         # adverbial phrases
-    if spanroot.dep_ in ['compound']:
+    if spanroot.dep_ in ["compound"]:
         category = "Compound noun"
 
     ## Nominal category
@@ -85,21 +133,24 @@ def construction_classifier(doc, span):
 
     ## ADJUNCTS
     # prep phrases
-    if spanroot.dep_ in ['prep', 'agent']:
-        category = 'Prepositional phrase'
+    if spanroot.dep_ in ["prep", "agent"]:
+        category = "Prepositional phrase"
     # adverbial phrases
-    if spanroot.dep_ in ['advmod', "npadvmod", "nmod", "npmod", 'quantmod']:
+    if spanroot.dep_ in ["advmod", "npadvmod", "nmod", "npmod", "quantmod"]:
         category = "Adverbial phrase"
 
     ## Predication patterns
-    if spanroot.dep_ in ['acomp', 'oprd']:
+    if spanroot.dep_ in ["acomp", "oprd"]:
         if "xcomp" in c_dep:
             category = "Subject predicate to-cl"
         else:
             category = "Adjectival complement"
 
-    if spanroot.dep_ in ['attr']:
-        subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+    if spanroot.dep_ in ["attr"]:
+        subjless = all(
+            c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+            for c in spanroot.children
+        )
 
         c_head = [c.dep_ for c in spanroot.head.children]
         if "expl" in c_head and "no_det" in span_t_dep_:
@@ -108,86 +159,115 @@ def construction_classifier(doc, span):
             category = "There is/are + Noun complement"
         elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
             category = "There is/are + Noun complement"
-        
+
         elif spanroot.pos_ in ["NOUN", "PRON"]:
             if "acl" in c_dep:
                 category = "Noun + Complement (attr)"
             else:
                 category = "Nominal complement"
 
-        elif not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+        elif not subjless and spanroot.pos_ in ["VERB", "AUX"]:
             category = "Main verb 4"
 
-        elif spanroot.tag_ in ['NNP']:
+        elif spanroot.tag_ in ["NNP"]:
             category = "Nominal complement"
 
-
     ####################################
     ### clausal ####
     ####################################
-    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", 'acl', 'pcomp', 'relcl']:
-
-        _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
-        _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
-        root_before_ccomp = [c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"]
-
-        _check_for_to = ["_".join([c.norm_, c.dep_])  for c in spanroot.subtree if c.head.dep_ == "advcl" and (c.dep_=="mark" or c.dep_ == "aux")]
-        entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", "acl", "pcomp", "relcl"]:
+        _check_to = [
+            c.dep_
+            for c in spanroot.subtree
+            if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"])
+            and c.head.dep_ == "xcomp"
+        ]
+        _check_ing = [
+            c.dep_
+            for c in spanroot.subtree
+            if "Prog" in str(c.morph) and c.dep_ == "xcomp"
+        ]
+        root_before_ccomp = [
+            c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"
+        ]
+
+        _check_for_to = [
+            "_".join([c.norm_, c.dep_])
+            for c in spanroot.subtree
+            if c.head.dep_ == "advcl" and (c.dep_ == "mark" or c.dep_ == "aux")
+        ]
+        entire_cl = (
+            spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+        )
 
         ## Start with broad category, which is then re-evaluated for specific constructions.
-        if spanroot.dep_ in ['advcl', 'mark', 'acl', 'pcomp']:
+        if spanroot.dep_ in ["advcl", "mark", "acl", "pcomp"]:
             ## Adverbial clauses
             ### Finite-adverbial clauses
             ### Non-finite adverbial clauses
-            subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
-            entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
-
-            if "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+            subjless = all(
+                c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+                for c in spanroot.children
+            )
+            entire_cl = (
+                spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+            )
+
+            if "mark" in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                 category = "Finite adverbial clause"
-            elif "mark" in span_dep and "aux" in span_dep :
+            elif "mark" in span_dep and "aux" in span_dep:
                 category = "Finite adverbial clause"
 
-            elif "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"] and "expl" in c_dep:
+            elif (
+                "mark" in span_dep
+                and spanroot.pos_ in ["VERB", "AUX"]
+                and "expl" in c_dep
+            ):
                 category = "Finite adverbial clause"
 
             elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
-                if spanroot.pos_ in ['VERB', "AUX"]:
+                if spanroot.pos_ in ["VERB", "AUX"]:
                     category = "Finite adverbial clause"
 
-                elif spanroot.pos_ not in ['VERB', "AUX"] and subjless:
+                elif spanroot.pos_ not in ["VERB", "AUX"] and subjless:
                     category = "Non-finite adv clause 1"
 
                 elif entire_cl:
                     category = "Finite adverbial clause"
 
-            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+            elif (
+                str(spanroot.morph)
+                in [
+                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
+                    "Aspect=Perf|Tense=Past|VerbForm=Part",
+                ]
+                and "aux" not in c_dep
+            ):
                 # he doing his job
                 if argmentless:
-                    #e.g., frankly speaking, strictly speaking
+                    # e.g., frankly speaking, strictly speaking
                     category = "Adverbial Phrase"
                 else:
                     category = "Non-finite adv clause 2"
 
-            elif spanroot.pos_ not in ['VERB', "AUX"] and "mark" in span_dep and subjless:
-
+            elif (
+                spanroot.pos_ not in ["VERB", "AUX"] and "mark" in span_dep and subjless
+            ):
                 category = "Non-finite adv clause 3"
-            
+
             elif "aux" in c_dep and "TO" in c_tag:
                 category = "Adverbial Phrase"
 
-
-            elif "mark" not in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+            elif "mark" not in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                 category = "Dependent Verb phrase"
-            
-            elif not argmentless:
-                category =  "Adverbial clause"
-            
-            elif spanroot.dep_ == "advcl":
-                category =  "Adverbial phrase"
 
+            elif not argmentless:
+                category = "Adverbial clause"
 
-        if spanroot.dep_ in ['relcl', 'ccomp', 'acl']:
+            elif spanroot.dep_ == "advcl":
+                category = "Adverbial phrase"
 
+        if spanroot.dep_ in ["relcl", "ccomp", "acl"]:
             head = spanroot.head
             if ";" in [t.norm_ for t in head.children]:
                 category = "Main verb 3"
@@ -195,13 +275,20 @@ def construction_classifier(doc, span):
                 category = "Dependent verb 1"
             elif "mark" in span_dep:
                 category = "Complement clause"
-            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+            elif (
+                str(spanroot.morph)
+                in [
+                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
+                    "Aspect=Perf|Tense=Past|VerbForm=Part",
+                ]
+                and "aux" not in c_dep
+            ):
                 category = "Non-finite complement clause"
-            elif spanroot.dep_ in ['relcl']:
+            elif spanroot.dep_ in ["relcl"]:
                 category = "Relative clause"
-            elif spanroot.dep_ in ['ccomp']:
+            elif spanroot.dep_ in ["ccomp"]:
                 category = "Complement clause"
-            elif spanroot.dep_ in ['acl']:
+            elif spanroot.dep_ in ["acl"]:
                 category = "Noun Complement clause"
             else:
                 # print(_check_for_to)
@@ -209,55 +296,78 @@ def construction_classifier(doc, span):
 
         ## Specific constructions
         # Extraposed that-clause or to-infinitives
-        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in ["VERB", "AUX"]:
+        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in [
+            "VERB",
+            "AUX",
+        ]:
             print(c_dep)
             if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
-                #eg it seems odd (oprd) that X.
-                #eg it is certain (acomp) that X.
-                category = "Extraposed that-cl (adj-complement)" #e.g., it is certain that X.
+                # eg it seems odd (oprd) that X.
+                # eg it is certain (acomp) that X.
+                category = (
+                    "Extraposed that-cl (adj-complement)"  # e.g., it is certain that X.
+                )
 
             elif "xcomp" in c_dep or ("advcl" in c_dep):
                 if "for_mark" in _check_for_to:
-                    category = "Extraposed to-cl (explicit subj)" #eg It is possible to .
+                    category = (
+                        "Extraposed to-cl (explicit subj)"  # eg It is possible to .
+                    )
                 elif _check_to:
-                    category = "Extraposed to-cl 1" #eg It is possible to .
+                    category = "Extraposed to-cl 1"  # eg It is possible to .
                 elif _check_ing:
-                    category = "Extraposed -ing 1" #eg It is possible to .
-            elif ("prep" in right_dep or "npadvmod" in right_dep) and "ccomp" in right_dep and spanroot.lemma_ == "be":
-                category = "Cleft construction" 
+                    category = "Extraposed -ing 1"  # eg It is possible to .
+            elif (
+                ("prep" in right_dep or "npadvmod" in right_dep)
+                and "ccomp" in right_dep
+                and spanroot.lemma_ == "be"
+            ):
+                category = "Cleft construction"
 
             elif "attr" in c_dep:
-                category = "Extraposed that-cl (copula)" #eg It is a wonder that X.
+                category = "Extraposed that-cl (copula)"  # eg It is a wonder that X.
 
             else:
-                category = "Extraposed that-cl (VERB)" 
+                category = "Extraposed that-cl (VERB)"
 
         # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
         #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
-        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "acomp" in c_dep:
+        elif (
+            "it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_
+        ) and "acomp" in c_dep:
             if "xcomp" in c_dep:
                 if _check_to:
-                    category = "Extraposed to-cl 2" #eg it is difficult to decide.
+                    category = "Extraposed to-cl 2"  # eg it is difficult to decide.
                 elif _check_ing:
-                    category = "Extraposed -ing 2" #eg it is difficult to decide.
-            
+                    category = "Extraposed -ing 2"  # eg it is difficult to decide.
+
             else:
                 category = "Extraposed that-cl (adj-complement) 2"
 
         elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
+            category = (
+                "Extraposed that-cl (adj-complement) 3"  # eg it seems odd that X.
+            )
 
-            category = "Extraposed that-cl (adj-complement) 3" #eg it seems odd that X.
-
-        
         # something without dummy subject "it"
-        elif (("nsubj" in c_dep and spanroot.lemma_ in ['be']) or "nsubjpass" in c_dep) and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm:
-            
+        elif (
+            (("nsubj" in c_dep and spanroot.lemma_ in ["be"]) or "nsubjpass" in c_dep)
+            and spanroot.pos_ in ["AUX", "VERB"]
+            and "it" not in c_norm
+        ):
             # store xcomp, if the head of the xcomp is acomp
-            _check_xcomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"]
-            _check_ccomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"]
+            _check_xcomp = [
+                c.dep_
+                for c in spanroot.subtree
+                if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"
+            ]
+            _check_ccomp = [
+                c.dep_
+                for c in spanroot.subtree
+                if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"
+            ]
             # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
             # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
-            
 
             if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
                 if any(root_before_ccomp):
@@ -271,13 +381,13 @@ def construction_classifier(doc, span):
             elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
                 category = "Post-predicate to-cl"
 
-            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_to:
+            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_to:
                 category = "Subject predicate to-cl"
 
             elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
                 category = "Subject predicate to-cl (passive)"
 
-            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_ing:
+            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_ing:
                 category = "Subject predicate -ing"
             elif "ccomp" in c_dep:
                 category = "Subject predicate that-cl"
@@ -290,9 +400,27 @@ def construction_classifier(doc, span):
                 category = "Main verb 1"
 
         ## without dummy subject it, and lexical verbs
-        elif ("nsubj" in c_dep or "nsubjpass" in c_dep)  in c_dep and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm and spanroot.lemma_ not in ['be']:
-            _check_wh = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["attr", "advmod", 'dobj', 'nsubj'] and c.tag_ in ["WP", "WRB", "WDT", "WP$"]) and c.head.dep_ == "ccomp"]
-            _check_if = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"]) and c.head.dep_ == "ccomp"]
+        elif (
+            ("nsubj" in c_dep or "nsubjpass" in c_dep) in c_dep
+            and spanroot.pos_ in ["AUX", "VERB"]
+            and "it" not in c_norm
+            and spanroot.lemma_ not in ["be"]
+        ):
+            _check_wh = [
+                c.dep_
+                for c in spanroot.subtree
+                if (
+                    c.dep_ in ["attr", "advmod", "dobj", "nsubj"]
+                    and c.tag_ in ["WP", "WRB", "WDT", "WP$"]
+                )
+                and c.head.dep_ == "ccomp"
+            ]
+            _check_if = [
+                c.dep_
+                for c in spanroot.subtree
+                if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"])
+                and c.head.dep_ == "ccomp"
+            ]
 
             # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
             # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
@@ -315,27 +443,34 @@ def construction_classifier(doc, span):
         # Existential
         elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
             category = "There is/are NOUN"
-            
-        elif "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]:
-            category = "Cleft construction"
 
+        elif (
+            "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]
+        ):
+            category = "Cleft construction"
 
-    if spanroot.dep_ in ['parataxis']:
-        if "_".join(span_dep) in ["nsubj_parataxis", "aux_parataxis", "nsubj_aux_parataxis"]:
+    if spanroot.dep_ in ["parataxis"]:
+        if "_".join(span_dep) in [
+            "nsubj_parataxis",
+            "aux_parataxis",
+            "nsubj_aux_parataxis",
+        ]:
             category = "Comment clause"
         else:
             category = "parataxis (for now)"
-    
 
     ## External comp
-    if spanroot.dep_ in ['xcomp']:
-        if spanroot.head.pos_ == 'ADJ' and "to_aux" in c_t_dep_:
+    if spanroot.dep_ in ["xcomp"]:
+        if spanroot.head.pos_ == "ADJ" and "to_aux" in c_t_dep_:
             category = "Adjective complement to-cl"
-        if spanroot.head.pos_ == 'VERB' and "to_aux" in c_t_dep_:
+        if spanroot.head.pos_ == "VERB" and "to_aux" in c_t_dep_:
             category = "Verb complement to-cl"
-    
-    if spanroot.dep_ in ['pcomp']:
-        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"] and 'ccomp' in c_dep:
+
+    if spanroot.dep_ in ["pcomp"]:
+        if (
+            str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]
+            and "ccomp" in c_dep
+        ):
             category = "Participle + that-cl"
         elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
             category = "Participle"
@@ -345,25 +480,28 @@ def construction_classifier(doc, span):
     #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
     #         category = "Gerund"
 
-    if spanroot.dep_ in ['neg']:
+    if spanroot.dep_ in ["neg"]:
         category = "Negative particle"
-    if spanroot.dep_ in ['aux', 'auxpass']:
+    if spanroot.dep_ in ["aux", "auxpass"]:
         category = "Auxiliary"
 
     # Modal verbs
     if spanroot.tag_ == "MD":
         category = "Modal auxiliary"
 
-
-    if spanroot.dep_ in ['dep', "csubj", 'csubjpass']:
-        if spanroot.head.dep_ in ['ROOT', 'ccomp'] and spanroot.head.pos_ in ['AUX', 'VERB'] and spanroot.pos_ in ['AUX', 'VERB']:
+    if spanroot.dep_ in ["dep", "csubj", "csubjpass"]:
+        if (
+            spanroot.head.dep_ in ["ROOT", "ccomp"]
+            and spanroot.head.pos_ in ["AUX", "VERB"]
+            and spanroot.pos_ in ["AUX", "VERB"]
+        ):
             if spanroot.morph == spanroot.head.morph:
                 category = "Main verb 4"
             else:
                 category = "Dependent verb 2"
         elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
             category = "Gerund"
-        elif spanroot.head.dep_ in ['conj', 'acl','relcl']:
+        elif spanroot.head.dep_ in ["conj", "acl", "relcl"]:
             if spanroot.morph == spanroot.head.morph:
                 category = "Main verb 4"
             else:
@@ -372,7 +510,7 @@ def construction_classifier(doc, span):
             category = "Dependent verb 2"
 
     # Appositive phrases
-    if spanroot.dep_ in ['appos']:
+    if spanroot.dep_ in ["appos"]:
         if "nummod" in c_dep:
             category = "Apposition"
         elif spanroot.pos_ in ["PROPN"]:
@@ -380,21 +518,23 @@ def construction_classifier(doc, span):
         elif spanroot.pos_ in ["NOUN"]:
             category = "Appositive Noun Phrase"
         elif spanroot.pos_ in ["VERB", "AUX"]:
-            _check = any(c.dep_ in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            _check = any(
+                c.dep_ in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+                for c in spanroot.children
+            )
             if _check:
                 category = "Appositive Finite-clause"
-            
-    if spanroot.dep_ in ['appos', "dep", "attr"]:
-        if not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+
+    if spanroot.dep_ in ["appos", "dep", "attr"]:
+        if not subjless and spanroot.pos_ in ["VERB", "AUX"]:
             category = "Main verb 5"
 
     if spanroot.dep_ in ["dep", "mark"]:
         if spanroot.tag_ in ["RB", "IN", "CC"]:
             category = "Conjunction"
 
-
-    #sometimes the extra-clausal links are not accurate
-    if spanroot.dep_ in ['aux', "auxpass", 'oprd', 'appos', "xcomp"]:
+    # sometimes the extra-clausal links are not accurate
+    if spanroot.dep_ in ["aux", "auxpass", "oprd", "appos", "xcomp"]:
         if spanroot.head.dep_ == "ROOT":
             category = "Main verb"
         else:
@@ -402,7 +542,7 @@ def construction_classifier(doc, span):
 
     if span.label_ == "CITATION":
         if "NNP" in span_tag or "NNPS" in span_tag:
-            if span_dep[0] == 'punct' and span_dep[-1] == 'punct':
+            if span_dep[0] == "punct" and span_dep[-1] == "punct":
                 category = "Parenthetical Citation"
             elif span_tag[0] in ["NNP", "NNPS"]:
                 category = "Narrative Citation"
@@ -425,7 +565,6 @@ def construction_classifier2(doc, span):
     span_token = [t.norm_ for t in span]
     span_tag = [t.tag_ for t in span]
 
-
     c = [c for c in spanroot.children]
     c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]
 
@@ -436,43 +575,92 @@ def construction_classifier2(doc, span):
 
     right_dep = [c.dep_ for c in spanroot.rights]
 
-    #conditionals
-    subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
-    argmentless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in spanroot.children)
-    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
-    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
-
+    # conditionals
+    subjless = all(
+        c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+        for c in spanroot.children
+    )
+    argmentless = all(
+        c.dep_
+        not in [
+            "nsubj",
+            "nsubjpass",
+            "csubj",
+            "csubjpass",
+            "dobj",
+            "ccomp",
+            "xcomp",
+            "dative",
+            "attr",
+            "oprd",
+            "acomp",
+        ]
+        for c in spanroot.children
+    )
+    argless_span = all(
+        c.dep_
+        not in [
+            "nsubj",
+            "nsubjpass",
+            "csubj",
+            "csubjpass",
+            "dobj",
+            "ccomp",
+            "xcomp",
+            "dative",
+            "attr",
+            "oprd",
+            "acomp",
+        ]
+        for c in span
+    )
+    argless_span = all(
+        c.dep_
+        not in [
+            "nsubj",
+            "nsubjpass",
+            "csubj",
+            "csubjpass",
+            "dobj",
+            "ccomp",
+            "xcomp",
+            "dative",
+            "attr",
+            "oprd",
+            "acomp",
+        ]
+        for c in span
+    )
 
     ## nesting classifiers
     if spanroot.dep_ == "conj":
-        while spanroot.dep_ == 'conj':
+        while spanroot.dep_ == "conj":
             spanroot = spanroot.head
 
     if spanroot.dep_ == "poss":
         head = spanroot.head
-        if head.dep_ in ["pobj", "dobj", "obj", "iobj" , "dative"]:
+        if head.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
             category = "Posessive Noun (Object)"
         elif head.dep_ in ["nsubj", "nsubjpass"]:
             category = "Posessive Noun (Subject)"
         else:
             category = "Posessive Noun (Other)"
 
-
-    ## Conjunctions 
+    ## Conjunctions
     # Preconjunctions
-    if spanroot.dep_ in ['preconj', 'cc']:
+    if spanroot.dep_ in ["preconj", "cc"]:
         category = "Conjunction"
 
     ## NOUN PHRASES
     # adverbial phrases
-    if spanroot.dep_ in ['amod']:
+    if spanroot.dep_ in ["amod"]:
         category = "Adjectival modifier"
         # adverbial phrases
-    if spanroot.dep_ in ['compound']:
+    if spanroot.dep_ in ["compound"]:
         category = "Compound noun"
 
     ## Nominal category
-    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj" , "dative"]:
+    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
         if "acl" in c_dep:
             category = "Noun + Complement (Object)"
         else:
@@ -486,22 +674,25 @@ def construction_classifier2(doc, span):
 
     ## ADJUNCTS
     # prep phrases
-    if spanroot.dep_ in ['prep', 'agent']:
-        category = 'Prepositional phrase'
+    if spanroot.dep_ in ["prep", "agent"]:
+        category = "Prepositional phrase"
 
     # adverbial phrases
-    if spanroot.dep_ in ['advmod', "npadvmod", "nmod", "npmod", 'quantmod',  'nummod']:
+    if spanroot.dep_ in ["advmod", "npadvmod", "nmod", "npmod", "quantmod", "nummod"]:
         category = "Adverbial phrase"
 
     ## Predication patterns
-    if spanroot.dep_ in ['acomp', 'oprd']:
+    if spanroot.dep_ in ["acomp", "oprd"]:
         if "xcomp" in c_dep:
             category = "Subject predicate to-cl"
         else:
             category = "Adjectival complement"
 
-    if spanroot.dep_ in ['attr']:
-        subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+    if spanroot.dep_ in ["attr"]:
+        subjless = all(
+            c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+            for c in spanroot.children
+        )
 
         c_head = [c.dep_ for c in spanroot.head.children]
         if "expl" in c_head and "no_det" in span_t_dep_:
@@ -510,28 +701,31 @@ def construction_classifier2(doc, span):
             category = "There is/are + Noun complement"
         elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
             category = "There is/are + Noun complement"
-        
+
         elif spanroot.pos_ in ["NOUN", "PRON"]:
             if "acl" in c_dep:
                 category = "Noun + Complement (attr)"
             else:
                 category = "Nominal complement"
 
-        elif not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+        elif not subjless and spanroot.pos_ in ["VERB", "AUX"]:
             category = "Main verb 4"
 
-        elif spanroot.tag_ in ['NNP']:
+        elif spanroot.tag_ in ["NNP"]:
             category = "Nominal complement"
 
     ## External comp
-    if spanroot.dep_ in ['xcomp']:
-        if spanroot.head.pos_ == 'ADJ' and "to_aux" in c_t_dep_:
+    if spanroot.dep_ in ["xcomp"]:
+        if spanroot.head.pos_ == "ADJ" and "to_aux" in c_t_dep_:
             category = "Adjective complement to-cl"
-        if spanroot.head.pos_ == 'VERB' and "to_aux" in c_t_dep_:
+        if spanroot.head.pos_ == "VERB" and "to_aux" in c_t_dep_:
             category = "Verb complement to-cl"
-    
-    if spanroot.dep_ in ['pcomp']:
-        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"] and 'ccomp' in c_dep:
+
+    if spanroot.dep_ in ["pcomp"]:
+        if (
+            str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]
+            and "ccomp" in c_dep
+        ):
             category = "Participle + that-cl"
         elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
             category = "Participle"
@@ -541,86 +735,117 @@ def construction_classifier2(doc, span):
     #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
     #         category = "Gerund"
 
-    if spanroot.dep_ in ['neg']:
+    if spanroot.dep_ in ["neg"]:
         category = "Negative particle"
-    if spanroot.dep_ in ['aux', 'auxpass']:
+    if spanroot.dep_ in ["aux", "auxpass"]:
         category = "Auxiliary"
 
     # Modal verbs
     if spanroot.tag_ == "MD":
         category = "Modal auxiliary"
 
-
     ####################################
     ### clausal ####
     ####################################
-    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", 'acl', 'pcomp', 'relcl', 'punct']:
-
-        _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
-        _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
-        root_before_ccomp = [c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"]
-
-        _check_for_to = ["_".join([c.norm_, c.dep_])  for c in spanroot.subtree if c.head.dep_ == "advcl" and (c.dep_=="mark" or c.dep_ == "aux")]
-        entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
-
+    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", "acl", "pcomp", "relcl", "punct"]:
+        _check_to = [
+            c.dep_
+            for c in spanroot.subtree
+            if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"])
+            and c.head.dep_ == "xcomp"
+        ]
+        _check_ing = [
+            c.dep_
+            for c in spanroot.subtree
+            if "Prog" in str(c.morph) and c.dep_ == "xcomp"
+        ]
+        root_before_ccomp = [
+            c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"
+        ]
+
+        _check_for_to = [
+            "_".join([c.norm_, c.dep_])
+            for c in spanroot.subtree
+            if c.head.dep_ == "advcl" and (c.dep_ == "mark" or c.dep_ == "aux")
+        ]
+        entire_cl = (
+            spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+        )
 
         ## Start with broad category, which is then re-evaluated for specific constructions.
-        if spanroot.dep_ in ['advcl', 'acl', 'punct', 'pcomp']: #'mark', 
+        if spanroot.dep_ in ["advcl", "acl", "punct", "pcomp"]:  #'mark',
             ## Adverbial clauses
-            subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
-            entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+            subjless = all(
+                c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+                for c in spanroot.children
+            )
+            entire_cl = (
+                spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+            )
 
             ### Finite-adverbial clauses
-            if "mark" in span_dep and (spanroot.pos_ in ['VERB', "AUX"] or "aux" in span_dep ):
+            if "mark" in span_dep and (
+                spanroot.pos_ in ["VERB", "AUX"] or "aux" in span_dep
+            ):
                 category = "Finite adverbial clause"
 
-            elif "mark" in span_dep and "aux" in span_dep :
+            elif "mark" in span_dep and "aux" in span_dep:
                 category = "Finite adverbial clause"
 
-            elif "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"] and "expl" in c_dep:
+            elif (
+                "mark" in span_dep
+                and spanroot.pos_ in ["VERB", "AUX"]
+                and "expl" in c_dep
+            ):
                 category = "Finite adverbial clause"
 
             elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
-                if spanroot.pos_ in ['VERB', "AUX"]:
+                if spanroot.pos_ in ["VERB", "AUX"]:
                     category = "Finite adverbial clause"
 
-                elif spanroot.pos_ not in ['VERB', "AUX"] and subjless:
+                elif spanroot.pos_ not in ["VERB", "AUX"] and subjless:
                     category = "Non-finite adv clause 1"
 
                 elif not argmentless:
-                    category =  "Finite adverbial clause"
+                    category = "Finite adverbial clause"
 
             ## non-finite
-            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+            elif (
+                str(spanroot.morph)
+                in [
+                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
+                    "Aspect=Perf|Tense=Past|VerbForm=Part",
+                ]
+                and "aux" not in c_dep
+            ):
                 # he doing his job
                 if argmentless:
-                    #e.g., frankly speaking, strictly speaking
+                    # e.g., frankly speaking, strictly speaking
                     category = "Adverbial Phrase"
                 else:
                     category = "Non-finite adv clause 2"
 
-            elif spanroot.pos_ not in ['VERB', "AUX"] and "mark" in span_dep and subjless:
-
+            elif (
+                spanroot.pos_ not in ["VERB", "AUX"] and "mark" in span_dep and subjless
+            ):
                 category = "Non-finite adv clause 3"
-            
+
             elif "aux" in c_dep and "TO" in c_tag:
                 category = "Adverbial Phrase"
 
-
-            elif "mark" not in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+            elif "mark" not in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                 category = "Dependent Verb phrase"
-            
+
             elif not argmentless:
-                category =  "Adverbial clause"
-            
+                category = "Adverbial clause"
+
             elif spanroot.dep_ == "advcl":
-                category =  "Adverbial phrase"
-            
+                category = "Adverbial phrase"
+
             else:
                 category = "Finite adverbial clause "
 
-        if spanroot.dep_ in ['relcl', 'ccomp', 'acl', 'punct', "pcomp"]:
-            
+        if spanroot.dep_ in ["relcl", "ccomp", "acl", "punct", "pcomp"]:
             head = spanroot.head
             if ";" in [t.norm_ for t in head.children]:
                 category = "Main verb 3"
@@ -630,66 +855,96 @@ def construction_classifier2(doc, span):
 
             elif "mark" in span_dep:
                 category = "Complement clause"
-            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+            elif (
+                str(spanroot.morph)
+                in [
+                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
+                    "Aspect=Perf|Tense=Past|VerbForm=Part",
+                ]
+                and "aux" not in c_dep
+            ):
                 category = "Non-finite complement clause"
-            elif spanroot.dep_ in ['relcl']:
+            elif spanroot.dep_ in ["relcl"]:
                 category = "Relative clause"
-            elif spanroot.dep_ in ['ccomp']:
+            elif spanroot.dep_ in ["ccomp"]:
                 category = "Complement clause"
-            elif spanroot.dep_ in ['acl']:
+            elif spanroot.dep_ in ["acl"]:
                 category = "Noun Complement clause"
 
         ## Specific constructions
         # Extraposed that-clause or to-infinitives
-        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in ["VERB", "AUX"]:
+        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in [
+            "VERB",
+            "AUX",
+        ]:
             # print(c_dep)
             if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
-                #eg it seems odd (oprd) that X.
-                #eg it is certain (acomp) that X.
-                category = "Extraposed that-cl (adj-complement)" #e.g., it is certain that X.
+                # eg it seems odd (oprd) that X.
+                # eg it is certain (acomp) that X.
+                category = (
+                    "Extraposed that-cl (adj-complement)"  # e.g., it is certain that X.
+                )
 
             elif "xcomp" in c_dep or ("advcl" in c_dep):
                 if "for_mark" in _check_for_to:
-                    category = "Extraposed to-cl (explicit subj)" #eg It is possible to .
+                    category = (
+                        "Extraposed to-cl (explicit subj)"  # eg It is possible to .
+                    )
                 elif _check_to:
-                    category = "Extraposed to-cl 1" #eg It is possible to .
+                    category = "Extraposed to-cl 1"  # eg It is possible to .
                 elif _check_ing:
-                    category = "Extraposed -ing 1" #eg It is possible to .
-            elif ("prep" in right_dep or "npadvmod" in right_dep) and "ccomp" in right_dep and spanroot.lemma_ == "be":
-                category = "Cleft construction" 
+                    category = "Extraposed -ing 1"  # eg It is possible to .
+            elif (
+                ("prep" in right_dep or "npadvmod" in right_dep)
+                and "ccomp" in right_dep
+                and spanroot.lemma_ == "be"
+            ):
+                category = "Cleft construction"
 
             elif "attr" in c_dep:
-                category = "Extraposed that-cl (copula)" #eg It is a wonder that X.
+                category = "Extraposed that-cl (copula)"  # eg It is a wonder that X.
 
             else:
-                category = "Extraposed that-cl (VERB)" 
+                category = "Extraposed that-cl (VERB)"
 
         # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
         #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
-        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "acomp" in c_dep:
+        elif (
+            "it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_
+        ) and "acomp" in c_dep:
             if "xcomp" in c_dep:
                 if _check_to:
-                    category = "Extraposed to-cl 2" #eg it is difficult to decide.
+                    category = "Extraposed to-cl 2"  # eg it is difficult to decide.
                 elif _check_ing:
-                    category = "Extraposed -ing 2" #eg it is difficult to decide.
-            
+                    category = "Extraposed -ing 2"  # eg it is difficult to decide.
+
             else:
                 category = "Extraposed that-cl (adj-complement) 2"
 
         elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
+            category = (
+                "Extraposed that-cl (adj-complement) 3"  # eg it seems odd that X.
+            )
 
-            category = "Extraposed that-cl (adj-complement) 3" #eg it seems odd that X.
-
-        
         # something without dummy subject "it"
-        elif (("nsubj" in c_dep and spanroot.lemma_ in ['be']) or "nsubjpass" in c_dep) and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm:
-            
+        elif (
+            (("nsubj" in c_dep and spanroot.lemma_ in ["be"]) or "nsubjpass" in c_dep)
+            and spanroot.pos_ in ["AUX", "VERB"]
+            and "it" not in c_norm
+        ):
             # store xcomp, if the head of the xcomp is acomp
-            _check_xcomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"]
-            _check_ccomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"]
+            _check_xcomp = [
+                c.dep_
+                for c in spanroot.subtree
+                if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"
+            ]
+            _check_ccomp = [
+                c.dep_
+                for c in spanroot.subtree
+                if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"
+            ]
             # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
             # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
-            
 
             if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
                 if any(root_before_ccomp):
@@ -703,13 +958,13 @@ def construction_classifier2(doc, span):
             elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
                 category = "Post-predicate to-cl"
 
-            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_to:
+            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_to:
                 category = "Subject predicate to-cl"
 
             elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
                 category = "Subject predicate to-cl (passive)"
 
-            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_ing:
+            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_ing:
                 category = "Subject predicate -ing"
             elif "ccomp" in c_dep:
                 category = "Subject predicate that-cl"
@@ -724,9 +979,27 @@ def construction_classifier2(doc, span):
                 category = "Main verb 1"
 
         ## without dummy subject it, and lexical verbs
-        elif ("nsubj" in c_dep or "nsubjpass" in c_dep)  in c_dep and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm and spanroot.lemma_ not in ['be']:
-            _check_wh = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["attr", "advmod", 'dobj', 'nsubj'] and c.tag_ in ["WP", "WRB", "WDT", "WP$"]) and c.head.dep_ == "ccomp"]
-            _check_if = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"]) and c.head.dep_ == "ccomp"]
+        elif (
+            ("nsubj" in c_dep or "nsubjpass" in c_dep) in c_dep
+            and spanroot.pos_ in ["AUX", "VERB"]
+            and "it" not in c_norm
+            and spanroot.lemma_ not in ["be"]
+        ):
+            _check_wh = [
+                c.dep_
+                for c in spanroot.subtree
+                if (
+                    c.dep_ in ["attr", "advmod", "dobj", "nsubj"]
+                    and c.tag_ in ["WP", "WRB", "WDT", "WP$"]
+                )
+                and c.head.dep_ == "ccomp"
+            ]
+            _check_if = [
+                c.dep_
+                for c in spanroot.subtree
+                if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"])
+                and c.head.dep_ == "ccomp"
+            ]
 
             # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
             # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
@@ -746,40 +1019,48 @@ def construction_classifier2(doc, span):
                 elif _check_ing:
                     category = "Post-predicate -ing"
 
-
-
         # Existential
         elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
             category = "There is/are NOUN"
-            
-        elif "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]:
+
+        elif (
+            "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]
+        ):
             category = "Cleft construction"
 
         ### The end of clausal analysis
-    
-    if spanroot.dep_ in ['parataxis']:
-        if "_".join(span_dep) in ["nsubj_parataxis", "aux_parataxis", "nsubj_aux_parataxis"]:
+
+    if spanroot.dep_ in ["parataxis"]:
+        if "_".join(span_dep) in [
+            "nsubj_parataxis",
+            "aux_parataxis",
+            "nsubj_aux_parataxis",
+        ]:
             category = "Comment clause"
         else:
             category = "Parataxis"
-    
 
-    if spanroot.dep_ in ['dep', "csubj", 'csubjpass']:
-        if spanroot.head.dep_ in ['ROOT', 'ccomp'] and spanroot.head.pos_ in ['AUX', 'VERB'] and spanroot.pos_ in ['AUX', 'VERB']:
+    if spanroot.dep_ in ["dep", "csubj", "csubjpass"]:
+        if (
+            spanroot.head.dep_ in ["ROOT", "ccomp"]
+            and spanroot.head.pos_ in ["AUX", "VERB"]
+            and spanroot.pos_ in ["AUX", "VERB"]
+        ):
             if spanroot.morph == spanroot.head.morph:
                 category = "Main verb 4"
             else:
                 category = "Dependent verb 2"
         elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
             category = "Gerund"
-        elif "VerbForm=Fin" in str(spanroot.morph) or "VerbForm=Inf" in str(spanroot.morph):
+        elif "VerbForm=Fin" in str(spanroot.morph) or "VerbForm=Inf" in str(
+            spanroot.morph
+        ):
             category = "Dependent verb 2"
-        elif spanroot.dep_ in ["csubj", 'csubjpass']:
+        elif spanroot.dep_ in ["csubj", "csubjpass"]:
             category = "Dependent verb (csubj)"
 
-
     # Appositive phrases
-    if spanroot.dep_ in ['appos']:
+    if spanroot.dep_ in ["appos"]:
         if "nummod" in c_dep:
             category = "Apposition"
         if spanroot.pos_ in ["PROPN"]:
@@ -787,16 +1068,18 @@ def construction_classifier2(doc, span):
         elif spanroot.pos_ in ["NOUN"]:
             category = "Appositive Noun Phrase"
         elif spanroot.pos_ in ["VERB", "AUX"]:
-            _check = any(c.dep_ in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            _check = any(
+                c.dep_ in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
+                for c in spanroot.children
+            )
             if _check:
                 category = "Appositive Finite-clause"
-    
 
-    if spanroot.dep_ in ['appos', "dep", "attr"]:
-        if not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+    if spanroot.dep_ in ["appos", "dep", "attr"]:
+        if not subjless and spanroot.pos_ in ["VERB", "AUX"]:
             category = "Main verb (likely parsing error)"
 
-    #sometimes the dep are on the conjunctions
+    # sometimes the dep are on the conjunctions
     if spanroot.dep_ in ["dep", "mark"]:
         if spanroot.tag_ in ["RB", "IN", "CC"]:
             category = "Conjunction"
@@ -804,9 +1087,12 @@ def construction_classifier2(doc, span):
     if spanroot.dep_ in ["intj"]:
         category = "Introjection"
 
-
-    #sometimes the extra-clausal links are not accurate
-    if spanroot.dep_ in ['aux', "auxpass", 'oprd', 'appos', "xcomp", "attr", 'dep', "meta", 'prt'] and category == None:
+    # sometimes the extra-clausal links are not accurate
+    if (
+        spanroot.dep_
+        in ["aux", "auxpass", "oprd", "appos", "xcomp", "attr", "dep", "meta", "prt"]
+        and category == None
+    ):
         if spanroot.head.dep_ == "ROOT":
             category = "Main verb"
         else:
@@ -814,7 +1100,7 @@ def construction_classifier2(doc, span):
 
     if span.label_ == "CITATION":
         if "NNP" in span_tag or "NNPS" in span_tag:
-            if span_dep[0] == 'punct' and span_dep[-1] == 'punct':
+            if span_dep[0] == "punct" and span_dep[-1] == "punct":
                 category = "Parenthetical Citation"
             elif span_tag[0] in ["NNP", "NNPS"]:
                 category = "Narrative Citation"
@@ -827,18 +1113,32 @@ def construction_classifier2(doc, span):
     return category
 
 
-
-def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
-                spans_key: str = "sc",
-                attrs: List[str] = SPAN_ATTRS):
-    columns = attrs + ["Conf. score", "sent no.", "grammatical realization", 'span dep', "ner", 
-                       "POS", 'span dep seq', "TAG sequence", "POS sequence", "head", "head dep", "children", "morphology", "sent"]
+def const_table(
+    doc: Union[spacy.tokens.Doc, Dict[str, str]],
+    spans_key: str = "sc",
+    attrs: List[str] = SPAN_ATTRS,
+):
+    columns = attrs + [
+        "Conf. score",
+        "sent no.",
+        "grammatical realization",
+        "span dep",
+        "ner",
+        "POS",
+        "span dep seq",
+        "TAG sequence",
+        "POS sequence",
+        "head",
+        "head dep",
+        "children",
+        "morphology",
+        "sent",
+    ]
     data = []
     # data = span_info_aggregator(doc, columns)
     sentences = {s: i for i, s in enumerate(doc.sents)}
 
-    for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores']):
-
+    for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs["scores"]):
         span_info = []
         span_info.extend([str(getattr(span, attr)) for attr in attrs])
 
@@ -854,7 +1154,7 @@ def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
         span_info.append(span.root.head.norm_)
         span_info.append(span.root.head.dep_)
         span_info.append("_".join([c.dep_ for c in span.root.children]))
-        span_info.append(span.root.morph)
+        span_info.append(str(span.root.morph))
         span_info.append(span.sent.text.strip())
 
         data.append(span_info)
@@ -862,27 +1162,27 @@ def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
     return data, columns
 
 
-def ngrammar(seq: list, n=2, concat = False, sep = "-"):
+def ngrammar(seq: list, n=2, concat=False, sep="-"):
     result = []
     n_item = len(seq)
     for idx, item in enumerate(seq):
         if idx + n <= n_item:
             if concat:
-                result.append(sep.join(seq[idx: idx + n]))
+                result.append(sep.join(seq[idx : idx + n]))
             else:
-                result.append(seq[idx: idx + n])
+                result.append(seq[idx : idx + n])
     return result
 
 
 def diversity_values(count_vec: list):
     result = {}
     if len(count_vec) == 0:
-        count_vec = [0,0,0,0,0,0,0,0,0,0]
+        count_vec = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
 
-    result['shannon'] = dv.alpha.shannon(list(count_vec), base=2)
-    result['brillouin_d'] = dv.alpha.brillouin_d(list(count_vec))
-    result["simpson_d"] = 1- dv.alpha.simpson(list(count_vec))    
-    result['simpson_e'] = dv.alpha.simpson_e(list(count_vec))
+    result["shannon"] = dv.alpha.shannon(list(count_vec), base=2)
+    result["brillouin_d"] = dv.alpha.brillouin_d(list(count_vec))
+    result["simpson_d"] = 1 - dv.alpha.simpson(list(count_vec))
+    result["simpson_e"] = dv.alpha.simpson_e(list(count_vec))
     # result['gini_index'] = dv.alpha.gini_index(list(count_vec))
     # result['faith_pd'] = dv.alpha.faith_pd(list(count_vec))
 

requirements.txt

@@ -1,4 +1,4 @@
-pip==24.2
+pip == 24.2
 spacy-streamlit #==1.0.4
 spacy>3.4.4, <3.7
 # spacy-experimental==0.6.1

utils/visualize.py

@@ -17,7 +17,12 @@ import streamlit as st
 from spacy_streamlit import visualize_spans
 from spacy_streamlit.util import load_model, process_text, get_svg, get_html, LOGO
 
-from pipeline.post_processors import simple_table, const_table, ngrammar, diversity_values
+from pipeline.post_processors import (
+    simple_table,
+    const_table,
+    ngrammar,
+    diversity_values,
+)
 from skbio import diversity as dv
 
 SPACY_VERSION = tuple(map(int, spacy.__version__.split(".")))
@@ -43,6 +48,9 @@ def visualize_spans(
     manual: bool = False,
     displacy_options: Optional[Dict] = None,
     simple: bool = True,
+    show_confidence: bool = False,
+    show_diversity: bool = False,
+    show_ngrams: bool = False,
 ):
     """
     Visualizer for spans.
@@ -100,13 +108,15 @@ def visualize_spans(
             df = pd.DataFrame(data, columns=cols)
             df = df.astype({"start": int, "end": int})
             df = df.sort_values(by= ['start'])
-            st.subheader("Span information")
+            st.subheader("Engagement span information")
+
             st.dataframe(
                 df.style.highlight_between(subset='Conf. score', right=.7))
 
-            if not simple:
+            counts = df['label_'].value_counts().reindex(CATEGORIES, fill_value=0)
+
+            if show_confidence:
                 st.subheader("Label counts & Diagnostic confidence score summary")
-                counts = df['label_'].value_counts().reindex(CATEGORIES, fill_value=0)
 
                 print(counts)
                 print(list(counts))
@@ -119,7 +129,9 @@ def visualize_spans(
                 st.dataframe(label_counts)
                 # print(list(label_counts))
 
+            if show_ngrams:
                 sequences = list(df['label_'])
+
                 # Engagement ngrams
                 span_bigrams = ngrammar(seq=sequences, n=2, concat=True)
                 span_trigrams = ngrammar(seq=sequences, n=3, concat=True)
@@ -132,20 +144,26 @@ def visualize_spans(
                 label_dep = pd.crosstab(df['grammatical realization'], df['label_'])
                 st.dataframe(label_dep)
 
-                st.subheader('Quantitative results')
+            if show_diversity:
+                st.subheader('Diversity of rhetorical features')
                 # st.markdown(
                 #     f"Shannon's index: {dv.alpha.shannon(list(counts), base=2): .3f}")
                 # st.markdown(
                 #     f"Simpson's e index: {1 - dv.alpha.simpson_e(list(counts)): .3f}")
                 
+                st.markdown("##### Entropy based diversity measures")
+
+                filename = "NA"
+
                 div = diversity_values(list(counts))
                 div_data = pd.DataFrame.from_dict(div, orient='index')
-                st.dataframe(div_data)            
+                # st.dataframe(div_data)            
                 
-                doc_data = pd.concat([counts, div_data], axis = 0).T
+                doc_data = pd.concat([div_data, counts, ], axis = 0).T
                 filename = "NA"
                 doc_data.insert(0, "filename", filename, True)
                 doc_data.insert(1, "nwords", len(doc), True)
                 st.dataframe(doc_data)            
+                
             # st.markdown(str(dv.alpha_diversity(metric = "shannon", counts=counts, ids = ['ENTERTAIN', 'ATTRIBUTE', 'CITATION', 'COUNTER', 'DENY', 'ENDORSE', 'PRONOUNCE', 'CONCUR', 'MONOGLOSS', 'SOURCES', 'JUSTIFYING'])))
             # print(dv.get_alpha_diversity_metrics())