Update app.py

app.py

@@ -40,43 +40,81 @@ if uploaded_files:
         documents = reader.load_data()
         st.success("File uploaded...")
 
+        # # Indexing
+        # index = PropertyGraphIndex.from_documents(
+        #     documents,
+        #     embed_model=OpenAIEmbedding(model_name="text-embedding-3-small"),
+        #     kg_extractors=[
+        #         ImplicitPathExtractor(),
+        #         SimpleLLMPathExtractor(
+        #             llm=OpenAI(model="gpt-3.5-turbo", temperature=0.3),
+        #             num_workers=4,
+        #             max_paths_per_chunk=10,
+        #         ),
+        #     ],
+        #     show_progress=True,
+        # )
+
+        # # Save Knowlege Graph
+        # index.property_graph_store.save_networkx_graph(name="./data/kg.html")
+
+        # # Display the graph in Streamlit
+        # st.success("File Processed...")
+        # st.success("Creating Knowledge Graph...")
+        # HtmlFile = open("./data/kg.html", 'r', encoding='utf-8')
+        # source_code = HtmlFile.read() 
+        # components.html(source_code, height= 500, width=700)
+
+        # # Retrieval
+        # kg_retriever = index.as_retriever(
+        #     include_text=True,  # include source text, default True
+        # )
+
+
         # Indexing
-        index = PropertyGraphIndex.from_documents(
-            documents,
-            embed_model=OpenAIEmbedding(model_name="text-embedding-3-small"),
-            kg_extractors=[
-                ImplicitPathExtractor(),
-                SimpleLLMPathExtractor(
-                    llm=OpenAI(model="gpt-3.5-turbo", temperature=0.3),
-                    num_workers=4,
-                    max_paths_per_chunk=10,
-                ),
-            ],
-            show_progress=True,
-        )
-
-        # Save Knowlege Graph
-        index.property_graph_store.save_networkx_graph(name="./data/kg.html")
-
-        # Display the graph in Streamlit
-        st.success("File Processed...")
-        st.success("Creating Knowledge Graph...")
-        HtmlFile = open("./data/kg.html", 'r', encoding='utf-8')
-        source_code = HtmlFile.read() 
-        components.html(source_code, height= 500, width=700)
+        splitter = SentenceSplitter(chunk_size=256)
+        nodes = splitter.get_nodes_from_documents(documents)
+        storage_context = StorageContext.from_defaults()
+        storage_context.docstore.add_documents(nodes)
+        index = VectorStoreIndex(nodes=nodes, storage_context=storage_context)
 
         # Retrieval
-        kg_retriever = index.as_retriever(
-            include_text=True,  # include source text, default True
-        )
+        bm25_retriever = BM25Retriever.from_defaults(nodes=nodes, similarity_top_k=10)
+        vector_retriever = index.as_retriever(similarity_top_k=10)
+
+        # Hybrid Retriever class
+        class HybridRetriever(BaseRetriever):
+            def __init__(self, vector_retriever, bm25_retriever):
+                self.vector_retriever = vector_retriever
+                self.bm25_retriever = bm25_retriever
+                super().__init__()
+
+            def _retrieve(self, query, **kwargs):
+                bm25_nodes = self.bm25_retriever.retrieve(query, **kwargs)
+                vector_nodes = self.vector_retriever.retrieve(query, **kwargs)
+                all_nodes = []
+                node_ids = set()
+                for n in bm25_nodes + vector_nodes:
+                    if n.node.node_id not in node_ids:
+                        all_nodes.append(n)
+                        node_ids.add(n.node.node_id)
+                return all_nodes
+
+        hybrid_retriever = HybridRetriever(vector_retriever, bm25_retriever)
 
         # Generation
         model = "gpt-3.5-turbo"
 
+        # def get_context(query):
+        #     contexts = kg_retriever.retrieve(query)
+        #     context_list = [n.text for n in contexts]
+        #     return context_list
+
         def get_context(query):
-            contexts = kg_retriever.retrieve(query)
-            context_list = [n.text for n in contexts]
+            contexts = hybrid_retriever.retrieve(query)
+            context_list = [n.get_content() for n in contexts]
             return context_list
+
         
 
         def res(prompt):