Update app.py

app.py

@@ -46,14 +46,28 @@ from langchain_community.document_loaders import TextLoader
 def load_txt(path="./a.cv.ckaller.2024.txt"):
     loader = TextLoader(path)
     document = loader.load()
+
+
+
+    ####
+
+    from langchain_experimental.text_splitter import SemanticChunker 
+    with open(path) as f:
+        state_of_the_union = f.read()
+
+
+
+    ######
    # split the document into chunks
-    text_splitter = RecursiveCharacterTextSplitter( 
+    a_text_splitter = RecursiveCharacterTextSplitter( 
         chunk_size=1500,
         chunk_overlap=250,
         length_function=len,
         is_separator_regex=False,
     )
-    document_chunks = text_splitter.split_documents(document)
+    a_document_chunks = text_splitter.split_documents(document)
+
+    
     ####### 
     ''' 
         FAISS
@@ -66,6 +80,16 @@ def load_txt(path="./a.cv.ckaller.2024.txt"):
     embeddings = HuggingFaceBgeEmbeddings(
         model_name=model, encode_kwargs=encode_kwargs, model_kwargs={"device": "cpu"}
     )
+
+
+#####
+
+
+    text_splitter = SemanticChunker(embeddings)
+     
+    document_chunks = text_splitter.create_documents([state_of_the_union])
+    print(document_chunks[0].page_content)
+
     # load from disk
     vector_store = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)