add stopword

app.py

@@ -3,7 +3,7 @@ import preprocessor
 import helper
 import matplotlib.pyplot as plt
 import seaborn as sns
-import pandas as pd 
+import pandas as pd
 
 st.set_option('deprecation.showPyplotGlobalUse', False)
 

preprocessor.py

@@ -1,5 +1,9 @@
 import re
 import pandas as pd
+from nltk.corpus import stopwords
+import nltk
+
+nltk.download('stopwords')
 
 def preprocess(data):
     pattern = '\d{1,2}/\d{1,2}/\d{2,4},\s\d{1,2}:\d{2}\s-\s'
@@ -28,6 +32,24 @@ def preprocess(data):
     df['message'] = messages
     df.drop(columns=['user_message'], inplace=True)
 
+    # Additional preprocessing
+    stop_words_en = set(stopwords.words('english'))
+    stop_words_fr = set(stopwords.words('french'))
+    combined_stop_words = stop_words_en.union(stop_words_fr)
+
+    def clean_message(message):
+        # Remove messages containing '<Media ...>'
+        if '<Media' in message:
+            return ''
+        # Remove words with less than 3 characters
+        words = [word for word in message.split() if len(word) >= 3]
+        # Remove stopwords
+        words = [word for word in words if word.lower() not in combined_stop_words]
+        return ' '.join(words)
+
+    df['message'] = df['message'].apply(clean_message)
+    df = df[df['message'] != '']  # Remove empty messages
+
     df['only_date'] = df['date'].dt.date
     df['year'] = df['date'].dt.year
     df['month_num'] = df['date'].dt.month

requirements.txt

@@ -4,4 +4,5 @@ seaborn
 urlextract
 wordcloud
 pandas
-emoji
+emoji
+nltk