Update tweet_analyzer.py

tweet_analyzer.py

@@ -1,4 +1,4 @@
-import os  
+import os
 from PyPDF2 import PdfReader
 import pandas as pd
 from dotenv import load_dotenv
@@ -10,6 +10,7 @@ from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.cluster import KMeans
 import random
+from joblib import Parallel, delayed
 
 class TweetDatasetProcessor:
     def __init__(self):
@@ -18,6 +19,7 @@ class TweetDatasetProcessor:
         self.tweets = []
         self.personality_profile = {}
         self.vectorizer = TfidfVectorizer(stop_words='english')
+        self.used_tweets = set()  # Track used tweets to avoid repetition
 
     def extract_text_from_pdf(self, pdf_path):
         """Extract text content from PDF file."""
@@ -29,40 +31,33 @@ class TweetDatasetProcessor:
 
     def process_pdf_content(self, text):
         """Process PDF content and clean extracted tweets."""
+        if not text.strip():
+            raise ValueError("The uploaded PDF appears to be empty.")
+        
         lines = text.split('\n')
-        clean_tweets = []
-        buffer = ""
-
-        for line in lines:
-            line = line.strip()
-            if not line:
-                if buffer:  # End of a tweet
-                    clean_tweets.append(buffer.strip())
-                    buffer = ""
-            elif line.startswith('http'):  # Skip URLs
-                continue
-            else:
-                buffer += " " + line  # Append lines to form complete tweets
-
-        if buffer:  # Add the last tweet
-            clean_tweets.append(buffer.strip())
-
-        # Build the tweet list with metadata
-        self.tweets = [
-            {
-                'content': tweet,
-                'timestamp': datetime.now(),  # Assign dummy timestamp
-                'mentions': self._extract_mentions(tweet),
-                'hashtags': self._extract_hashtags(tweet)
-            }
-            for tweet in clean_tweets
-        ]
+        clean_tweets = Parallel(n_jobs=-1)(delayed(self._process_line)(line) for line in lines)
+        self.tweets = [tweet for tweet in clean_tweets if tweet]
+
+        if not self.tweets:
+            raise ValueError("No tweets were extracted from the PDF. Ensure the content is properly formatted.")
 
         # Save the processed tweets to a CSV
         df = pd.DataFrame(self.tweets)
         df.to_csv('processed_tweets.csv', index=False)
         return df
 
+    def _process_line(self, line):
+        """Process a single line in parallel."""
+        line = line.strip()
+        if not line or line.startswith('http'):  # Skip empty lines and URLs
+            return None
+        return {
+            'content': line,
+            'timestamp': datetime.now(),
+            'mentions': self._extract_mentions(line),
+            'hashtags': self._extract_hashtags(line)
+        }
+
     def _extract_mentions(self, text):
         """Extract mentioned users from tweet."""
         return [word for word in text.split() if word.startswith('@')]
@@ -74,6 +69,9 @@ class TweetDatasetProcessor:
     def categorize_tweets(self):
         """Cluster tweets into categories using KMeans."""
         all_tweets = [tweet['content'] for tweet in self.tweets]
+        if not all_tweets:
+            raise ValueError("No tweets available for clustering.")
+        
         tfidf_matrix = self.vectorizer.fit_transform(all_tweets)
         kmeans = KMeans(n_clusters=5, random_state=1)
         kmeans.fit(tfidf_matrix)
@@ -82,30 +80,38 @@ class TweetDatasetProcessor:
             tweet['category'] = f"Category {kmeans.labels_[i]}"
         return pd.DataFrame(self.tweets)
 
-    def analyze_personality(self):
-        """Comprehensive personality analysis using all tweets."""
-        all_tweets = [tweet['content'] for tweet in self.tweets]
-        # Use a broader dataset for a comprehensive profile
+    def analyze_personality(self, max_tweets=50):
+        """Comprehensive personality analysis using a limited subset of tweets."""
+        if not self.tweets:
+            raise ValueError("No tweets available for personality analysis.")
+
+        all_tweets = [tweet['content'] for tweet in self.tweets][:max_tweets]
         analysis_prompt = f"""Perform a deep psychological analysis of the author based on these tweets:
         Core beliefs, emotional tendencies, cognitive patterns, etc.
         Tweets for analysis:
         {json.dumps(all_tweets, indent=2)}
         """
+        try:
+            response = self.groq_client.chat.completions.create(
+                messages=[
+                    {"role": "system", "content": "You are an expert psychologist."},
+                    {"role": "user", "content": analysis_prompt},
+                ],
+                model="llama-3.1-70b-versatile",
+                temperature=0.1,
+            )
+            self.personality_profile = response.choices[0].message.content
+            return self.personality_profile
+        except Exception as e:
+            return f"Error during personality analysis: {str(e)}"
 
-        response = self.groq_client.chat.completions.create(
-            messages=[
-                {"role": "system", "content": "You are an expert psychologist."},
-                {"role": "user", "content": analysis_prompt},
-            ],
-            model="llama-3.1-70b-versatile",
-            temperature=0.1,
-        )
-        self.personality_profile = response.choices[0].message.content
-        return self.personality_profile
-
-    def analyze_topics(self, n_topics=5):
+    def analyze_topics(self, n_topics=None):
         """Extract and identify different topics the author has tweeted about."""
         all_tweets = [tweet['content'] for tweet in self.tweets]
+        if not all_tweets:
+            return []
+
+        n_topics = n_topics or min(5, len(all_tweets) // 10)
         tfidf_matrix = self.vectorizer.fit_transform(all_tweets)
         nmf_model = NMF(n_components=n_topics, random_state=1)
         nmf_model.fit(tfidf_matrix)
@@ -120,34 +126,33 @@ class TweetDatasetProcessor:
         """Estimate the number of tokens in the given text."""
         return len(text.split())
 
-    def generate_tweet(self, context=""):
-        """Generate a new tweet based on personality profile and optional context."""
-        historical_topics = self.analyze_topics(n_topics=5)
-        additional_contexts = historical_topics + [
-            "Comment on a recent technological advancement.",
-            "Share a motivational thought.",
-            "Discuss a current trending topic.",
-            "Reflect on a past experience.",
-            "Provide advice to followers."
-        ]
-        selected_contexts = random.sample(additional_contexts, min(3, len(additional_contexts)))
-
-        # Select tweets close to the author's style
-        tfidf_matrix = self.vectorizer.transform([tweet['content'] for tweet in self.tweets])
-        similarity = cosine_similarity(tfidf_matrix)
-        tweet_sample_indices = similarity.sum(axis=1).argsort()[-5:]  # Top 5 similar tweets
-        all_tweets = [self.tweets[i]['content'] for i in tweet_sample_indices]
-
-        personality_profile_excerpt = self.personality_profile[:400]
+    def generate_tweet(self, context="", sample_size=3):
+        """Generate a new tweet by sampling random tweets and avoiding repetition."""
+        if not self.tweets:
+            return "Error: No tweets available for generation."
+
+        # Randomly sample unique tweets
+        available_tweets = [tweet for tweet in self.tweets if tweet['content'] not in self.used_tweets]
+        if len(available_tweets) < sample_size:
+            self.used_tweets.clear()  # Reset used tweets if all have been used
+            available_tweets = self.tweets
+
+        sampled_tweets = random.sample(available_tweets, sample_size)
+        sampled_contents = [tweet['content'] for tweet in sampled_tweets]
+
+        # Update the used tweets tracker
+        self.used_tweets.update(sampled_contents)
+
+        # Truncate personality profile to avoid token overflow
+        personality_profile_excerpt = self.personality_profile[:400] if len(self.personality_profile) > 400 else self.personality_profile
 
+        # Construct the prompt
         prompt = f"""Based on this personality profile:
         {personality_profile_excerpt}
         Current context or topic (if any):
         {context}
-        Additionally, consider these contexts to increase diversity:
-        {', '.join(selected_contexts)}
         Tweets for context:
-        {', '.join(all_tweets)}
+        {', '.join(sampled_contents)}
         **Only generate the tweet. Do not include analysis, explanation, or any other content.**
         """
         try:
@@ -163,5 +168,4 @@ class TweetDatasetProcessor:
             tweet = response.choices[0].message.content.strip()
             return tweet
         except Exception as e:
-            print(f"Error generating tweet: {e}")
-            return "Error generating tweet"
+            return f"Error generating tweet: {str(e)}"