Update functions.py

functions.py

@@ -106,10 +106,16 @@ def preprocess_plain_text(text,window_size=3):
     return passages
  
 @st.experimental_memo(suppress_st_warning=True)    
-def chunk_clean_text(text):
+def chunk_and_process_text(text):
     
     """Chunk text longer than 500 tokens"""
     
+    text = text.encode("ascii", "ignore").decode()  # unicode
+    text = re.sub(r"https*\S+", " ", text)  # url
+    text = re.sub(r"@\S+", " ", text)  # mentions
+    text = re.sub(r"#\S+", " ", text)  # hastags
+    text = re.sub(r"\s{2,}", " ", text)  # over spaces
+
     article = nlp(text)
     sentences = [i.text for i in list(article.sents)]