Combining both sentiment and news collector

1. adding the sentiment analysis code.
1. adding helping functions for sentiment analyzer.
1. combining both the gradio rows for news collector and sentiment analyzer to work on the same page

app.py

@@ -4,6 +4,15 @@ import os
 import json
 import pandas as pd
 
+# -----imports for Sentiment Analyzer
+from sklearn.pipeline import Pipeline
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+
+
+#--------------------------------------------------------------------------------------
+#------------------------ NEWS DATA RETRIEVER------------------------------------------
+#--------------------------------------------------------------------------------------
 
 
 def creating_data_dir(directory_path):
@@ -147,11 +156,181 @@ def call_functions(domain):
 # iface.launch(debug=True)
 
 # GRADIO APP USING BLOCKS
-with gr.Blocks() as demo:
-  ui_domain = gr.Dropdown(["bbc", "forbes", "businessinsider_us"], label="Select Domain")
-  df_output = gr.Dataframe(type="pandas")
-  retrieve_button = gr.Button("Retrieve news")
 
-  retrieve_button.click(call_functions, inputs=ui_domain, outputs=df_output)
 
-demo.launch(debug=True)
+
+
+#--------------------------------------------------------------------------------------
+#------------------------ SENTIMENT ANALYZER------------------------------------------
+#--------------------------------------------------------------------------------------
+
+#----------------  Data Prepocessing ---------- 
+def re_breakline(text_list):
+    return [re.sub('[\n\r]', ' ', r) for r in text_list]
+   
+def re_hyperlinks(text_list):
+    # Applying regex
+    pattern = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+    return [re.sub(pattern, ' link ', r) for r in text_list]
+  
+def re_dates(text_list):
+    # Applying regex
+    pattern = '([0-2][0-9]|(3)[0-1])(\/|\.)(((0)[0-9])|((1)[0-2]))(\/|\.)\d{2,4}'
+    return [re.sub(pattern, ' date ', r) for r in text_list]
+
+
+def re_money(text_list):
+    # Applying regex
+    pattern = '[R]{0,1}\$[ ]{0,}\d+(,|\.)\d+'
+    return [re.sub(pattern, ' paisa ', r) for r in text_list]
+    
+def re_numbers(text_list):
+    # Applying regex
+    return [re.sub('[0-9]+', ' num ', r) for r in text_list]
+
+def re_negation(text_list):
+    # Applying regex
+    return [re.sub('([nN][ãÃaA][oO]|[ñÑ]| [nN] )', ' negate ', r) for r in text_list]
+
+def re_special_chars(text_list):
+    # Applying regex
+    return [re.sub('\W', ' ', r) for r in text_list]
+def re_whitespaces(text_list):
+    # Applying regex
+    white_spaces = [re.sub('\s+', ' ', r) for r in text_list]
+    white_spaces_end = [re.sub('[ \t]+$', '', r) for r in white_spaces]
+    return white_spaces_end
+
+# Class for regular expressions application
+class ApplyRegex(BaseEstimator, TransformerMixin):
+
+    def __init__(self, regex_transformers):
+        self.regex_transformers = regex_transformers
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        # Applying all regex functions in the regex_transformers dictionary
+        for regex_name, regex_function in self.regex_transformers.items():
+            X = regex_function(X)
+
+        return X
+        
+# Class for stopwords removal from the corpus
+class StopWordsRemoval(BaseEstimator, TransformerMixin):
+
+    def __init__(self, text_stopwords):
+        self.text_stopwords = text_stopwords
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        return [' '.join(stopwords_removal(comment, self.text_stopwords)) for comment in X]
+
+# Class for apply the stemming process
+class StemmingProcess(BaseEstimator, TransformerMixin):
+
+    def __init__(self, stemmer):
+        self.stemmer = stemmer
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        return [' '.join(stemming_process(comment, self.stemmer)) for comment in X]
+
+# Class for extracting features from corpus
+class TextFeatureExtraction(BaseEstimator, TransformerMixin):
+
+    def __init__(self, vectorizer):
+        self.vectorizer = vectorizer
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        return self.vectorizer.fit_transform(X).toarray()
+
+
+#----------------------------Creating Pipeline for Preparing the data-----
+# Defining regex transformers to be applied
+regex_transformers = {
+    'break_line': re_breakline,
+    'hiperlinks': re_hyperlinks,
+    'dates': re_dates,
+    'money': re_money,
+    'numbers': re_numbers,
+    'negation': re_negation,
+    'special_chars': re_special_chars,
+    'whitespaces': re_whitespaces
+}
+
+# Defining the vectorizer to extract features from text
+vectorizer = TfidfVectorizer(max_features=300, min_df=7, max_df=0.8, stop_words=en_stopwords)
+
+# Building the Pipeline
+text_pipeline = Pipeline([
+    ('regex', ApplyRegex(regex_transformers)),
+    ('stopwords', StopWordsRemoval(stopwords.words('portuguese'))),
+    ('stemming', StemmingProcess(RSLPStemmer())),
+    ('text_features', TextFeatureExtraction(vectorizer))
+])
+
+
+
+#----------------- Analyzing the Sentiments of whole dataset-------
+
+def sentiment_analyzer(csv_file_name='combined_news_response.csv'):
+
+  df = pd.read_csv(csv_file_name)
+  df.drop('Unnamed: 0',axis=1,inplace=True)
+
+  # Splitting into X and y
+  X = list(df['content'].values)
+  # Applying the pipeline
+  X_processed = text_pipeline.fit_transform(X)
+
+  # Load a saved model
+  loaded_model_nb = joblib.load("Naive Bayes_model.joblib")
+
+  # Use the loaded model for inference
+  loaded_predictions_nb = loaded_model_nb.predict(X_processed)
+  sentiments = loaded_predictions_nb
+  
+  # Sentiment mapping
+  sentiment_mapping = {0: 'negative', 1: 'neutral', 2: 'positive'}
+
+  print(f"df['content'].values ==> {len(df['content'].values)} \n sentiments length ==> {len(sentiments)}")
+  # Create a DataFrame
+  sentiment_df = pd.DataFrame({
+      'content': df['content'].values,
+      'sentiment': [sentiment_mapping[sent] for sent in sentiments]
+  })
+
+  return sentiment_df
+
+
+
+
+
+# Creating the app for both 
+
+with gr.Blocks() as demo:
+    with gr.Row():
+      with gr.Column(scale=1, min_width=600):
+        ui_domain = gr.Dropdown(["bbc", "forbes", "businessinsider_us"], label="Select Domain")
+        df_output = gr.Dataframe(type="pandas",wrap=True)
+        retrieve_button = gr.Button("Retrieve news")
+      
+        retrieve_button.click(call_functions, inputs=ui_domain, outputs=df_output)
+
+    with gr.Row():
+      with gr.Column(scale=1, min_width=600):
+          ui_input = gr.Textbox(value='combined_news_response.csv' , visible=False)
+          view_sentiment_bttn = gr.Button("Analyze Sentiment")
+          df_output = gr.Dataframe(type="pandas",wrap=True)
+          
+          view_sentiment_bttn.click(sentiment_analyzer, inputs=ui_input, outputs=df_output)
+
+demo.launch(debug=True)