✨ option to drop stopwords pre-summ

Signed-off-by: peter szemraj <peterszemraj@gmail.com>

app.py

@@ -32,6 +32,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+    datefmt="%Y-%b-%d %H:%M:%S",
 )
 
 import gradio as gr
@@ -50,6 +51,7 @@ from utils import (
     saves_summary,
     textlist2html,
     truncate_word_count,
+    remove_stopwords,
 )
 
 _here = Path(__file__).parent
@@ -194,6 +196,7 @@ def proc_submission(
     length_penalty: float,
     repetition_penalty: float,
     no_repeat_ngram_size: int,
+    predrop_stopwords: bool,
     max_input_length: int = 6144,
 ):
     """
@@ -230,11 +233,14 @@ def proc_submission(
         "do_sample": False,
     }
     max_input_length = int(os.environ.get("APP_MAX_WORDS", max_input_length))
-    logging.info(f"max_input_length set to: {max_input_length}")
+    logging.info(
+        f"max_input_length set to: {max_input_length}. pre-drop stopwords: {predrop_stopwords}"
+    )
 
     st = time.perf_counter()
     history = {}
     clean_text = clean(input_text, lower=False)
+    clean_text = remove_stopwords(clean_text) if predrop_stopwords else clean_text
     processed = truncate_word_count(clean_text, max_words=max_input_length)
 
     if processed["was_truncated"]:
@@ -296,6 +302,7 @@ def proc_submission(
     html += ""
 
     # save to file
+    settings["remove_stopwords"] = predrop_stopwords
     settings["model_name"] = model_name
     saved_file = saves_summary(summarize_output=_summaries, outpath=None, **settings)
     return html, full_summary, scores_out, saved_file
@@ -607,6 +614,10 @@ if __name__ == "__main__":
                     label="no repeat ngram size",
                     value=3,
                 )
+                predrop_stopwords = gr.Checkbox(
+                    label="Drop Stopwords (Pre-Truncation)",
+                    value=False,
+                )
         with gr.Column():
             gr.Markdown("## About")
             gr.Markdown(
@@ -638,6 +649,7 @@ if __name__ == "__main__":
                 length_penalty,
                 repetition_penalty,
                 no_repeat_ngram_size,
+                predrop_stopwords,
             ],
             outputs=[output_text, summary_text, summary_scores, text_file],
         )

utils.py

@@ -19,34 +19,74 @@ logging.basicConfig(
 
 import torch
 from natsort import natsorted
-from nltk.tokenize import word_tokenize
+from nltk.tokenize import word_tokenize, WhitespaceTokenizer
 from rapidfuzz import fuzz
 
-# Define stopwords
 STOPWORDS = set(
     "a about above after again against all am an and any are aren't as at be because been before being below between both but by can't cannot could couldn't did didn't do does doesn't doing don't down during each few for from further had hadn't has hasn't have haven't having he he'd he'll he's her here here's hers herself him himself his how how's i i'd i'll i'm i've if in into is isn't it it's its itself let's me more most mustn't my myself no nor not of off on once only or other ought our ours ourselves out over own same shan't she she'd she'll she's should shouldn't so some such than that that's the their theirs them themselves then there there's these they they'd they'll they're they've this those through to too under until up very was wasn't we we'd we'll we're we've were weren't what what's when when's where where's which while who who's whom why why's with won't would wouldn't you you'd you'll you're you've your yours yourself yourselves".split()
 )
 
 
-def remove_stopwords(text: str, stopwords: list = STOPWORDS) -> str:
+def custom_tokenize(text: str) -> List[str]:
+    """custom_tokenize - merges words containing apostrophes as one token."""
+
+    # Tokenize the text using the WhitespaceTokenizer
+    tokenizer = WhitespaceTokenizer()
+    tokens = tokenizer.tokenize(text)
+
+    merged_tokens = []
+    merged_token = ""
+
+    for token in tokens:
+        if re.search(r"\w+'\w+", token):
+            # Token contains an apostrophe, merge with previous token
+            merged_token += token
+        else:
+            # no apostrophe, add previous merged token (if any) and current
+            if merged_token:
+                merged_tokens.append(merged_token)
+                merged_token = ""
+            merged_tokens.append(token)
+
+    # Add the last merged token (if any)
+    if merged_token:
+        merged_tokens.append(merged_token)
+
+    return merged_tokens
+
+
+def remove_stopwords(
+    text: str, stopwords: List[str] = STOPWORDS, use_custom_tokenize: bool = True
+) -> str:
     """
-    remove_stopwords - Remove stopwords from a string.
+    remove_stopwords - Remove stopwords from text.
 
-    :param str text: text to remove stopwords from
-    :param list stopwords: list of stopwords to remove, defaults to STOPWORDS
-    :return, str: text with stopwords removed
+    :param str text: input text
+    :param List[str] stopwords: list of stopwords, defaults to STOPWORDS
+    :param bool use_custom_tokenize: use custom apostrophe tokenizer, defaults to True
+    :return str: text with stopwords removed
     """
-    words = word_tokenize(text)
-    filtered_words = []
+    words = custom_tokenize(text) if use_custom_tokenize else word_tokenize(text)
 
+    filtered_words = []
     for word in words:
-        word = word.strip(string.punctuation)  # remove punctuation
+        # Remove leading and trailing punctuation marks
+        word = word.strip(string.punctuation)
 
         if word.lower() not in stopwords:
             filtered_words.append(word)
 
     filtered_text = " ".join(filtered_words)
 
+    # Replace multiple consecutive whitespaces with a single space
+    filtered_text = re.sub(r"\s+", " ", filtered_text)
+    filtered_text = filtered_text.strip()
+
+    # Restore original whitespaces around punctuation marks
+    filtered_text = re.sub(
+        r"\s*([{}])\s*".format(re.escape(string.punctuation)), r"\1", filtered_text
+    )
+
     return filtered_text