Update app.py

app.py

@@ -3,10 +3,11 @@ import openai, os
 
 from langchain.chains import LLMChain, RetrievalQA
 from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import PyPDFLoader
+from langchain.document_loaders import PyPDFLoader, WebBaseLoader
 from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
 from langchain.document_loaders.generic import GenericLoader
 from langchain.document_loaders.parsers import OpenAIWhisperParser
+
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.prompts import PromptTemplate
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -31,15 +32,11 @@ RAG_CHAIN_PROMPT = PromptTemplate(input_variables = ["context", "question"],
 CHROMA_DIR  = "/data/chroma"
 YOUTUBE_DIR = "/data/youtube"
 
+PDF_URL       = "https://arxiv.org/pdf/2303.08774.pdf"
+WEB_URL       = "https://openai.com/research/gpt-4"
 YOUTUBE_URL_1 = "https://www.youtube.com/watch?v=--khbXchTeE"
 YOUTUBE_URL_2 = "https://www.youtube.com/watch?v=hdhZwyf24mE"
 YOUTUBE_URL_3 = "https://www.youtube.com/watch?v=vw-KWfKwvTQ"
-YOUTUBE_URL_4 = "https://www.youtube.com/watch?v=kiHpqXNCPj8"
-YOUTUBE_URL_5 = "https://www.youtube.com/shorts/3x95mw35dJY"
-YOUTUBE_URL_6 = "https://www.youtube.com/shorts/zg-DS23wq0c"
-YOUTUBE_URL_7 = "https://www.youtube.com/shorts/cS4fyhKZ8bQ"
-
-PDF_URL = "https://arxiv.org/pdf/2303.08774.pdf"
 
 MODEL_NAME  = "gpt-4"
 
@@ -48,24 +45,25 @@ def invoke(openai_api_key, use_rag, prompt):
                      openai_api_key = openai_api_key, 
                      temperature = 0)
     if (use_rag):
-        # Document loading, splitting, and storage
+        # Document loading
         docs = []
-        #loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,
-        #                                           YOUTUBE_URL_2,
-        #                                           YOUTUBE_URL_3,
-        #                                           YOUTUBE_URL_4,
-        #                                           YOUTUBE_URL_5,
-        #                                           YOUTUBE_URL_6,
-        #                                           YOUTUBE_URL_7], YOUTUBE_DIR), 
-        #                       OpenAIWhisperParser())
-        #docs = loader.load()
-        ###docs.extend(loader.load())
+        # Load PDF
         loader = PyPDFLoader(PDF_URL)
         docs.extend(loader.load())
-        #
+        # Load Web
+        loader = WebBaseLoader(WEB_URL)
+        docs.extend(loader.load())
+        # Load YouTube
+        loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,
+                                                   YOUTUBE_URL_2,
+                                                   YOUTUBE_URL_3], YOUTUBE_DIR), 
+                               OpenAIWhisperParser())
+        docs.extend(loader.load())
+        # Document splitting
         text_splitter = RecursiveCharacterTextSplitter(chunk_overlap = 150,
                                                        chunk_size = 1500)
         splits = text_splitter.split_documents(docs)
+        # Document storage
         vector_db = Chroma.from_documents(documents = splits, 
                                           embedding = OpenAIEmbeddings(disallowed_special = ()), 
                                           persist_directory = CHROMA_DIR)