NonToxicGlazeAdvisor_Chat_with_Docs_Groq_Edition_1 - app.py - 27-03-2024, 10:25 CET

app.py

@@ -155,85 +155,98 @@ if "vector" not in st.session_state:
     # docs = loader.load()
     # st.session_state.docs = docs
 
-    # JB:
-    # https://python.langchain.com/docs/modules/data_connection/document_loaders/file_directory
-    # text_loader_kwargs={'autodetect_encoding': True}
-    text_loader_kwargs={'autodetect_encoding': False}
-    path = '../'
-    # loader = DirectoryLoader(path, glob="**/*.pdf", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)
-    # PyPDFDirectoryLoader (TEST):
-    # loader = PyPDFDirectoryLoader(path, glob="**/*.pdf", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)
-    # loader = PyPDFDirectoryLoader(path, glob="**/*.pdf", loader_kwargs=text_loader_kwargs)
-    loader = PyPDFDirectoryLoader(path, glob="**/*.pdf")
-    docs = loader.load()
-    st.session_state.docs = docs
-
-    # JB 18-03-2024:
-    # https://python.langchain.com/docs/integrations/document_loaders/
-    # MICROSOFT WORD:
-    # https://python.langchain.com/docs/integrations/document_loaders/microsoft_word
-    # 1 - Using Docx2txt
-    # Load .docx using Docx2txt into a document.
-    # %pip install --upgrade --quiet  docx2txt
-    # from langchain_community.document_loaders import Docx2txtLoader
-    # loader = Docx2txtLoader("example_data/fake.docx")
-    # data = loader.load()
-    # data
-    # [Document(page_content='Lorem ipsum dolor sit amet.', metadata={'source': 'example_data/fake.docx'})]
-    #
-    # 2A - Using Unstructured
-    # from langchain_community.document_loaders import UnstructuredWordDocumentLoader
-    # loader = UnstructuredWordDocumentLoader("example_data/fake.docx")
-    # data = loader.load()
-    # data
-    # [Document(page_content='Lorem ipsum dolor sit amet.', lookup_str='', metadata={'source': 'fake.docx'}, lookup_index=0)]
-    #
-    # 2B - Retain Elements
-    # Under the hood, Unstructured creates different “elements” for different chunks of text.
-    # By default we combine those together, but you can easily keep that separation by specifying mode="elements".
-    # loader = UnstructuredWordDocumentLoader("example_data/fake.docx", mode="elements")
-    # data = loader.load()
-    # data[0]
-    # Document(page_content='Lorem ipsum dolor sit amet.', lookup_str='', metadata={'source': 'fake.docx', 'filename': 'fake.docx', 'category': 'Title'}, lookup_index=0)
-    #
-    # 2A - Using Unstructured
-    # from langchain_community.document_loaders import UnstructuredWordDocumentLoader
-    # loader = UnstructuredWordDocumentLoader(path, glob="**/*.docx")
-    # docs = loader.load()
-    # st.session_state.docs = docs
 
 
-
-    
-    st.session_state.text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    st.session_state.documents = st.session_state.text_splitter.split_documents(st.session_state.docs)
-    # https://python.langchain.com/docs/integrations/vectorstores/faiss
-    # docs_and_scores = db.similarity_search_with_score(query)
-    # Saving and loading
-    # You can also save and load a FAISS index. 
-    # This is useful so you don’t have to recreate it everytime you use it.
-    # db.save_local("faiss_index")
-    # new_db = FAISS.load_local("faiss_index", embeddings)
-    # docs = new_db.similarity_search(query)
-    # docs[0]
-    # Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../../state_of_the_union.txt'})
-    #
-    st.session_state.vector = FAISS.from_documents(st.session_state.documents, st.session_state.embeddings) # ORIGINAL
+    with st.status("Downloading data...", expanded=True) as status:
+        # st.write("Searching for data...")
+        # time.sleep(2)
+        # st.write("Found URL.")
+        # time.sleep(1)
+        # st.write("Downloading data...")
+        # time.sleep(1)
+        #status.update(label="Download complete!", state="complete", expanded=False)
+
+        st.write("Laden van de PDF documenten...")
+        # JB:
+        # https://python.langchain.com/docs/modules/data_connection/document_loaders/file_directory
+        # text_loader_kwargs={'autodetect_encoding': True}
+        text_loader_kwargs={'autodetect_encoding': False}
+        path = '../'
+        # loader = DirectoryLoader(path, glob="**/*.pdf", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)
+        # PyPDFDirectoryLoader (TEST):
+        # loader = PyPDFDirectoryLoader(path, glob="**/*.pdf", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)
+        # loader = PyPDFDirectoryLoader(path, glob="**/*.pdf", loader_kwargs=text_loader_kwargs)
+        loader = PyPDFDirectoryLoader(path, glob="**/*.pdf")
+        docs = loader.load()
+        st.session_state.docs = docs
+
+        # JB 18-03-2024:
+        # https://python.langchain.com/docs/integrations/document_loaders/
+        # MICROSOFT WORD:
+        # https://python.langchain.com/docs/integrations/document_loaders/microsoft_word
+        # 1 - Using Docx2txt
+        # Load .docx using Docx2txt into a document.
+        # %pip install --upgrade --quiet  docx2txt
+        # from langchain_community.document_loaders import Docx2txtLoader
+        # loader = Docx2txtLoader("example_data/fake.docx")
+        # data = loader.load()
+        # data
+        # [Document(page_content='Lorem ipsum dolor sit amet.', metadata={'source': 'example_data/fake.docx'})]
+        #
+        # 2A - Using Unstructured
+        # from langchain_community.document_loaders import UnstructuredWordDocumentLoader
+        # loader = UnstructuredWordDocumentLoader("example_data/fake.docx")
+        # data = loader.load()
+        # data
+        # [Document(page_content='Lorem ipsum dolor sit amet.', lookup_str='', metadata={'source': 'fake.docx'}, lookup_index=0)]
+        #
+        # 2B - Retain Elements
+        # Under the hood, Unstructured creates different “elements” for different chunks of text.
+        # By default we combine those together, but you can easily keep that separation by specifying mode="elements".
+        # loader = UnstructuredWordDocumentLoader("example_data/fake.docx", mode="elements")
+        # data = loader.load()
+        # data[0]
+        # Document(page_content='Lorem ipsum dolor sit amet.', lookup_str='', metadata={'source': 'fake.docx', 'filename': 'fake.docx', 'category': 'Title'}, lookup_index=0)
+        #
+        # 2A - Using Unstructured
+        # from langchain_community.document_loaders import UnstructuredWordDocumentLoader
+        # loader = UnstructuredWordDocumentLoader(path, glob="**/*.docx")
+        # docs = loader.load()
+        # st.session_state.docs = docs
+
+
+        st.write("Splitting / chunking de teksten...")
+        st.session_state.text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+        st.session_state.documents = st.session_state.text_splitter.split_documents(st.session_state.docs)
+
+        st.write("Genereer de Vector Store (kan enige minuten duren)...")        
+        # https://python.langchain.com/docs/integrations/vectorstores/faiss
+        # docs_and_scores = db.similarity_search_with_score(query)
+        # Saving and loading
+        # You can also save and load a FAISS index. 
+        # This is useful so you don’t have to recreate it everytime you use it.
+        # db.save_local("faiss_index")
+        # new_db = FAISS.load_local("faiss_index", embeddings)
+        # docs = new_db.similarity_search(query)
+        # docs[0]
+        # Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../../state_of_the_union.txt'})
+        #
+        st.session_state.vector = FAISS.from_documents(st.session_state.documents, st.session_state.embeddings) # ORIGINAL
     
-    # st.session_state.vector = FAISS.from_documents(st.session_state.documents, st.session_state.embeddings) # ORIGINAL
-    #st.session_state.vector.save_local("faiss_index")
-    # The de-serialization relies loading a pickle file. 
-    # Pickle files can be modified to deliver a malicious payload that results in execution of arbitrary code on your machine.
-    # You will need to set `allow_dangerous_deserialization` to `True` to enable deserialization. If you do this, make sure that you trust the source of the data.
+        # st.session_state.vector = FAISS.from_documents(st.session_state.documents, st.session_state.embeddings) # ORIGINAL
+        #st.session_state.vector.save_local("faiss_index")
+        # The de-serialization relies loading a pickle file. 
+        # Pickle files can be modified to deliver a malicious payload that results in execution of arbitrary code on your machine.
+        # You will need to set `allow_dangerous_deserialization` to `True` to enable deserialization. If you do this, make sure that you trust the source of the data.
     
-    #st.session_state.vector = FAISS.load_local("faiss_index", st.session_state.embeddings, allow_dangerous_deserialization=True)
+        #st.session_state.vector = FAISS.load_local("faiss_index", st.session_state.embeddings, allow_dangerous_deserialization=True)
     
-    # ZIE: 
-    # ZIE VOOR EEN APP MET CHROMADB:
-    # https://github.com/vndee/local-rag-example/blob/main/rag.py
-    # https://raw.githubusercontent.com/vndee/local-rag-example/main/rag.py
-    # Chroma.from_documents(documents=chunks, embedding=FastEmbedEmbeddings())
-    # st.session_state.vector = Chroma.from_documents(st.session_state.documents, st.session_state.embeddings) # JB
+        # ZIE: 
+        # ZIE VOOR EEN APP MET CHROMADB:
+        # https://github.com/vndee/local-rag-example/blob/main/rag.py
+        # https://raw.githubusercontent.com/vndee/local-rag-example/main/rag.py
+        # Chroma.from_documents(documents=chunks, embedding=FastEmbedEmbeddings())
+        # st.session_state.vector = Chroma.from_documents(st.session_state.documents, st.session_state.embeddings) # JB
 
 st.write("---------------------------------")