Update app.py

app.py

@@ -1,10 +1,7 @@
 import os
 import streamlit as st
 from openai import OpenAI
-from PyPDF2 import PdfReader
 import requests
-from youtube_transcript_api import YouTubeTranscriptApi
-from urllib.parse import urlparse, parse_qs
 from pinecone import Pinecone
 import uuid
 from dotenv import load_dotenv
@@ -21,27 +18,22 @@ from pymongo import MongoClient
 from pymongo.errors import ConnectionFailure
 from datetime import datetime
 
-# Set page config at the very beginning
 st.set_page_config(layout="wide")
 
-# Load environment variables
 load_dotenv()
 
-# Set up OpenAI client
 client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
 
-# Set up Pinecone
 pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
 
-index_name = "lyca"  # Your index name
+index_name = "lyca"
 index = pc.Index(index_name)
 
-# Set up MongoDB connection
 mongo_uri = os.getenv("MONGODB_URI")
 
 try:
     client = MongoClient(mongo_uri, serverSelectionTimeoutMS=5000)
-    client.server_info()  # This will raise an exception if the connection fails
+    client.server_info()
     db = client['lyca']
     sim_swap_collection = db['sim_swap_requests']
 except ConnectionFailure:
@@ -52,47 +44,30 @@ def get_embedding(text):
     response = client.embeddings.create(input=text, model="text-embedding-3-large")
     return response.data[0].embedding
 
-def process_pdf(file):
-    reader = PdfReader(file)
-    text = ""
-    for page in reader.pages:
-        text += page.extract_text() + "\n"
-    return text
-
 def process_web_link(url):
     try:
-        # Set up Selenium options
         chrome_options = Options()
-        chrome_options.add_argument("--headless")  # Run in headless mode for performance
+        chrome_options.add_argument("--headless")
         chrome_options.add_argument("--no-sandbox")
         chrome_options.add_argument("--disable-dev-shm-usage")
         
-        # Install the Chrome driver automatically using webdriver-manager
         driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)
         
-        # Navigate to the URL
         driver.get(url)
         
-        # Give the page some time to load fully
         time.sleep(3)
         
-        # Extract the rendered page's content
         page_source = driver.page_source
         
-        # Close the browser after extracting content
         driver.quit()
 
-        # Parse the page content using BeautifulSoup
         soup = BeautifulSoup(page_source, 'lxml')
         
-        # Remove script and style elements
         for script in soup(["script", "style"]):
             script.decompose()
         
-        # Get text
         text = soup.get_text()
         
-        # Clean up the text
         lines = (line.strip() for line in text.splitlines())
         chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
         text = '\n'.join(chunk for chunk in chunks if chunk)
@@ -102,38 +77,14 @@ def process_web_link(url):
         print(f"Error processing web link {url}: {str(e)}")
         return f"Error processing {url}: {str(e)}"
 
-def process_youtube_link(url):
-    video_id = extract_video_id(url)
-    transcript = YouTubeTranscriptApi.get_transcript(video_id)
-    return " ".join([entry['text'] for entry in transcript])
-
-def extract_video_id(url):
-    parsed_url = urlparse(url)
-    if parsed_url.hostname == 'youtu.be':
-        return parsed_url.path[1:]
-    if parsed_url.hostname in ('www.youtube.com', 'youtube.com'):
-        if parsed_url.path == '/watch':
-            return parse_qs(parsed_url.query)['v'][0]
-        if parsed_url.path[:7] == '/embed/':
-            return parsed_url.path.split('/')[2]
-        if parsed_url.path[:3] == '/v/':
-            return parsed_url.path.split('/')[2]
-    return None
-
 def process_upload(upload_type, file_or_link, file_name=None):
     print(f"Starting process_upload for {upload_type}")
     doc_id = str(uuid.uuid4())
     print(f"Generated doc_id: {doc_id}")
 
-    if upload_type == "PDF":
-        content = process_pdf(file_or_link)
-        doc_name = file_name or "Uploaded PDF"
-    elif upload_type == "Web Link":
+    if upload_type == "Web Link":
         content = process_web_link(file_or_link)
         doc_name = file_or_link
-    elif upload_type == "YouTube Link":
-        content = process_youtube_link(file_or_link)
-        doc_name = f"YouTube: {file_or_link}"
     else:
         print("Invalid upload type")
         return "Invalid upload type"
@@ -141,7 +92,6 @@ def process_upload(upload_type, file_or_link, file_name=None):
     content_length = len(content)
     print(f"Content extracted, length: {content_length}")
 
-    # Dynamically adjust chunk size based on content length
     if content_length < 10000:
         chunk_size = 1000
     elif content_length < 100000:
@@ -158,7 +108,6 @@ def process_upload(upload_type, file_or_link, file_name=None):
         
         for future in as_completed(futures):
             vectors.append(future.result())
-            # Update progress
             progress = len(vectors) / len(chunks)
             st.session_state.upload_progress.progress(progress)
     
@@ -186,14 +135,12 @@ def get_relevant_context(query, top_k=5):
     search_results = index.query(vector=query_embedding, top_k=top_k, include_metadata=True)
     print(f"Found {len(search_results['matches'])} relevant results")
     
-    # Sort results by doc_id and chunk_index to maintain document structure
     sorted_results = sorted(search_results['matches'], key=lambda x: (x['metadata']['doc_id'], x['metadata']['chunk_index']))
     
     context = "\n".join([result['metadata']['text'] for result in sorted_results])
     return context, sorted_results
 
 def check_lyca_data_loaded():
-    # Check if there are any vectors in the index
     stats = index.describe_index_stats()
     return stats['total_vector_count'] > 0
 
@@ -218,12 +165,10 @@ def general_conversation(message):
 
 def is_sim_swap_request(message):
     sim_swap_keywords = {'sim', 'swap', 'change', 'new', 'replace'}
-    # Remove the question mark at the end if it exists
     message = message.rstrip('?')
     message_words = set(message.lower().split())
     return len(sim_swap_keywords.intersection(message_words)) >= 2
 
-# Add a print statement for debugging
 print(f"is_sim_swap_request result: {is_sim_swap_request('how to change my sim?')}")
 
 def trigger_sim_swap_workflow():
@@ -294,7 +239,6 @@ def chat_with_ai(message):
                 for result in results
             ]
         else:
-            # Fallback to general conversation if no relevant context is found or similarity is low
             ai_response = general_conversation(message)
             sources = []
         
@@ -309,7 +253,6 @@ def clear_database():
     print("Database cleared")
     return "Database cleared successfully."
 
-# Streamlit UI
 st.title("Lyca Mobile Assistant")
 
 if 'workflow' not in st.session_state:
@@ -320,7 +263,6 @@ if 'workflow' not in st.session_state:
 if 'chat_history' not in st.session_state:
     st.session_state.chat_history = []
 
-# Create two columns instead of three
 col1, col2 = st.columns([2, 1])
 
 with col1:
@@ -329,14 +271,12 @@ with col1:
     if st.session_state.workflow == 'sim_swap':
         process_sim_swap_workflow()
     else:
-        # Display chat history
         for message in st.session_state.chat_history:
             st.markdown(f"**{'You' if message['role'] == 'user' else 'AI'}:** {message['content']}")
         
         user_input = st.text_input("How can I assist you with Lyca Mobile today?")
         if st.button("Send"):
             if user_input:
-                # Add debug print
                 print(f"User input: {user_input}")
                 is_swap_request = is_sim_swap_request(user_input)
                 print(f"Is sim swap request: {is_swap_request}")
@@ -348,20 +288,16 @@ with col1:
                     st.session_state.workflow = 'sim_swap'
                 else:
                     print("Proceeding with regular chat flow")
-                    # Existing code for non-sim-swap requests
                     st.session_state.chat_progress = st.progress(0)
                     response, sources = chat_with_ai(user_input)
                     st.session_state.chat_progress.progress(1.0)
                     
-                    # Add to chat history
                     st.session_state.chat_history.append({"role": "user", "content": user_input})
                     st.session_state.chat_history.append({"role": "assistant", "content": response})
                     
-                    # Display the latest messages
                     st.markdown("**You:** " + user_input)
                     st.markdown("**AI:** " + response)
                     
-                    # Store sources in session state for display in col2
                     st.session_state.sources = sources
                     st.session_state.chat_progress.empty()
             else: