openapi generative pipeline integration

.vscode/launch.json

@@ -1,16 +1,33 @@
+// {
+//     // Use IntelliSense to learn about possible attributes.
+//     // Hover to view descriptions of existing attributes.
+//     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+//     "version": "0.2.0",
+//     "configurations": [
+//         {
+//             "name": "Python: Current File",
+//             "type": "python",
+//             "request": "launch",
+//             "program": "${file}",
+//             "console": "integratedTerminal",
+//             "justMyCode": true
+//         }
+//     ]
+// }
 {
     // Use IntelliSense to learn about possible attributes.
     // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    // For more information, visit: Debugging in Visual Studio Code
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Python: Current File",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal",
-            "justMyCode": true
-        }
+    {
+    "name": "debug streamlit",
+    "type": "python",
+    "request": "launch",
+    "program": "/Users/jorge.henao/oosource/ask2democracycol/venv/bin/streamlit",
+    "args": [
+    "run",
+    "main_page.py"]
+    }
     ]
 }

about.py

@@ -1,6 +1,6 @@
 import streamlit as st
 
-def about_ask2democracy():
+def about_ask2democracy(api_key):
     st.markdown("""
     <h1 style='
     text-align: center;

hallazgos.py

@@ -10,14 +10,15 @@ import logging
 logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)
 logging.getLogger("haystack").setLevel(logging.INFO)
 
-extractive_query = PinecodeProposalQueries (index_name= Config.index_name,
+
+def hallazgos_comision_verdad_2022(api_key):
+    
+    queries = PinecodeProposalQueries (index_name= Config.index_name,
                                         api_key = Config.es_password,
                                         environment = Config.pinecone_environment,
                                         embedding_dim = Config.embedding_dim,
                                         reader_name_or_path = Config.reader_model_name_or_path,
-                                        use_gpu = Config.use_gpu)
-
-def hallazgos_comision_verdad_2022():
+                                        use_gpu = Config.use_gpu, OPENAI_key= api_key)
     title = """
     <h1 style='
     text-align: center;
@@ -55,8 +56,7 @@ def hallazgos_comision_verdad_2022():
 
     def search(question, retriever_top_k, reader_top_k):
         filters = {"source_title": "Hallazgos y recomendaciones - 28 de Junio 2022"}
-        
-        query_result = extractive_query.search_by_query(query = question, 
+        query_result = queries.search_by_query(query = question, 
                                                         retriever_top_k = retriever_top_k, 
                                                         reader_top_k = reader_top_k,
                                                         filters = filters)
@@ -80,7 +80,7 @@ def hallazgos_comision_verdad_2022():
         elapsed_time = round(ent - stt, 2)
 
         # show which query was entered, and what was searching time
-        st.write(f"**Resultados relacionados con:** \"{query}\" ({elapsed_time} sec.)")
+        st.write(f"**Resultados encontrados de las fuentes** \"{query}\" ({elapsed_time} sec.):")
         # then we use loop to show results
         for i, answer in enumerate(results):
             # answer starts with header
@@ -92,9 +92,49 @@ def hallazgos_comision_verdad_2022():
             
         st.markdown("---")
 
+    def search_and_generate_answer(question, retriever_top_k, generator_top_k):
+        filters = {"source_title": "Hallazgos y recomendaciones - 28 de Junio 2022"}
+        query_result = queries.genenerate_answer_OpenAI(query = question, 
+                                                        retriever_top_k = retriever_top_k, 
+                                                        generator_top_k = generator_top_k,
+                                                        filters = filters, OPENAI_key = st.session_state.get("OPENAI_API_KEY"))
+        
+        result = []
+        for i in range(0, len(query_result)):
+            item = query_result[i]
+            source_title = item.meta['doc_metas'][0]['source_title']
+            source_url = item.meta['doc_metas'][0]['source_url']
+            chapter_titles = [source['title'] for source in item.meta['doc_metas']]
+            result.append([[i+1], item.answer.replace("\n",""), 
+                        source_title, source_url, str(chapter_titles)])
+            
+    def search_and_show_generative_results():
+        # set start time
+        stt = time.time()
+        
+        # retrieve top 5 documents
+        results = search_and_generate_answer(query, retriever_top_k=5, generator_top_k=1)
+        # set endtime
+        ent = time.time()
+        # measure resulting time
+        elapsed_time = round(ent - stt, 2)
+
+        # show which query was entered, and what was searching time
+        st.write(f"**Respuesta generada a partir de los resultados** \"{query}\" ({elapsed_time} sec.):")
+        # then we use loop to show results
+        if results != None:
+            for i, answer in enumerate(results):
+                # answer starts with header
+                st.subheader(f"{answer[1]}")
+                st.markdown(f"[**Lee más aquí**]({answer[3]})")
+                st.caption(f"Fuentes: {answer[2]} - {answer[4]}")
+                
+            st.markdown("---")
+        
     #results = search("que es el adres", retriever_top_k=5, reader_top_k=3)
     
-    st.markdown(f"""<br><p>Cuanto más contexto le des a la pregunta mejor funciona el sistema. 
+    
+    st.markdown(f"""<br><p>Cuanto más contexto le des a la pregunta mejores resultados se obtienen. 
              No es un sistema basado en palabras claves, puedes escribir preguntas elaboradas. 
              Una serie de modelos de lenguaje transformers intervienen en cada consulta para ayudar a entenderlas.""", 
              unsafe_allow_html=True)
@@ -106,4 +146,6 @@ def hallazgos_comision_verdad_2022():
             st.error("¡escribe una pregunta!")
         else:
             st.session_state["submit"] = True
+            if api_key:
+                search_and_show_generative_results()
             search_and_show_results()

main_page.py

@@ -6,7 +6,11 @@ from pinecode_quieries import PinecodeProposalQueries
 from config import Config
 
 # Define the sidebar
-api_key = st.sidebar.text_input("OpenAI API Key", type="password")
+api_key = st.sidebar.text_input("OpenAI API Key", type="password", 
+                                value=st.session_state.get("OPENAI_API_KEY", ""))
+if api_key:
+    st.session_state["OPENAI_API_KEY"] = api_key
+
 
 # Define the navigation between pages
 page_options = {
@@ -19,4 +23,5 @@ page_options = {
 selected_page = st.sidebar.radio("Selecciona la página que deseas explorar:", list(page_options.keys()))
 
 # Render the selected page
-page_options[selected_page]()
+#print("key: " + api_key)
+page_options[selected_page](api_key)

pinecode_quieries.py

@@ -76,24 +76,24 @@ class DocumentQueries(ABC):
 class PinecodeProposalQueries(DocumentQueries):
     
     def __init__(self, index_name: str, api_key, reader_name_or_path: str, use_gpu = True,
-                 embedding_dim = 384, environment = "us-east1-gcp") -> None:
+                 embedding_dim = 384, environment = "us-east1-gcp", OPENAI_key = None) -> None:
         
         reader = FARMReader(model_name_or_path = reader_name_or_path, 
                             use_gpu = use_gpu, num_processes = 1, 
                             context_window_size = 200)
         
         self._initialize_pipeline(index_name, api_key, reader = reader, embedding_dim= 
-                                  embedding_dim, environment = environment)
+                                  embedding_dim, environment = environment, OPENAI_key= OPENAI_key)
         #self.log = Log(es_host= es_host, es_index="log", es_user = es_user, es_password= es_password)
 
     def _initialize_pipeline(self, index_name, api_key, similarity = "cosine",
                              embedding_dim = 384, reader = None,
                              environment = "us-east1-gcp",
-                             metadata_config = {"indexed": ["title", "source_title"]}):
+                             metadata_config = {"indexed": ["title", "source_title"]},
+                             OPENAI_key = None):
         if reader is not None:
             self.reader = reader
-        self.OPENAI_generator = OpenAIAnswerGenerator(api_key="", 
-                                                      model="text-davinci-003", temperature=.5, max_tokens=60)
+                
         #pinecone.init(api_key=es_password, environment="us-east1-gcp")
         
         self.document_store = PineconeDocumentStore(
@@ -113,9 +113,14 @@ class PinecodeProposalQueries(DocumentQueries):
                      
         self.extractive_pipe = ExtractiveQAPipeline (reader = self.reader, 
                                           retriever = self.retriever)
-        self.generative_OPENAI_pipe = GenerativeQAPipeline(generator = self.OPENAI_generator, 
-                                                           retriever = self.retriever)
         
+        self.generative_OPENAI_pipe = None
+        if (OPENAI_key != None and OPENAI_key != ""):
+            OPENAI_generator = OpenAIAnswerGenerator(api_key = OPENAI_key, 
+                                                      model="text-davinci-003", temperature=.5, max_tokens=60)
+            self.generative_OPENAI_pipe = GenerativeQAPipeline(generator = OPENAI_generator, 
+                                                           retriever = self.retriever)
+            
     def search_by_query(self, query : str, retriever_top_k: int, reader_top_k: int, index_name: str = None, filters = None):
         #self.document_store.update_embeddings(self.retriever, update_existing_embeddings=False)        
         params = {"Retriever": {"top_k": retriever_top_k, 
@@ -124,19 +129,28 @@ class PinecodeProposalQueries(DocumentQueries):
         prediction = self.extractive_pipe.run( query = query, params = params, debug = True)      
         return prediction["answers"]
 
-    def genenerate_answer_OpenAI(self, query : str, retriever_top_k: int, reader_top_k: int, es_index: str = None, filters = None) :
-        params = {"Retriever": {"top_k": retriever_top_k, 
-                                "filters": filters}, 
-                  "Generator": {"top_k": reader_top_k}} 
-        prediction = self.generative_OPENAI_pipe.run( query = query, params = params)      
-        return prediction
+    def genenerate_answer_OpenAI(self, query : str, retriever_top_k: int, generator_top_k: int,  filters = None, OPENAI_key = None):
+        # if (OPENAI_key != None and OPENAI_key != ""):
+        #     OPENAI_generator = OpenAIAnswerGenerator(api_key=OPENAI_key, 
+        #                                               model="text-davinci-003", temperature=.5, max_tokens=60)
+        #     self.generative_OPENAI_pipe = GenerativeQAPipeline(generator = OPENAI_generator, 
+        #                                                    retriever = self.retriever)
+        
+        if (self.generative_OPENAI_pipe != None):
+            params = {"Retriever": {"top_k": retriever_top_k, 
+                                    "filters": filters}, 
+                    "Generator": {"top_k": generator_top_k}} 
+            prediction = self.generative_OPENAI_pipe.run( query = query, params = params)      
+            return prediction["answers"]
+        else:
+            return None
 
     def genenerate_answer_HF(self, query : str, retriever_top_k: int, reader_top_k: int, es_index: str = None, filters = None) :
         params = {"Retriever": {"top_k": retriever_top_k, 
                                 "filters": filters}, 
                   "Generator": {"top_k": reader_top_k}} 
         prediction = self.generative_HF_pipe.run( query = query, params = params)      
-        return prediction
+        return prediction["answers"]
 
 class Log():
     

reformas.py

@@ -10,14 +10,16 @@ import logging
 logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)
 logging.getLogger("haystack").setLevel(logging.INFO)
 
-extractive_query = PinecodeProposalQueries (index_name= Config.index_name,
+def reforma_salud_febrero_2023(api_key):
+    
+    queries = PinecodeProposalQueries (index_name= Config.index_name,
                                         api_key = Config.es_password,
                                         environment = Config.pinecone_environment,
                                         embedding_dim = Config.embedding_dim,
                                         reader_name_or_path = Config.reader_model_name_or_path,
-                                        use_gpu = Config.use_gpu)
+                                        use_gpu = Config.use_gpu,
+                                        OPENAI_key=api_key)
 
-def reforma_salud_febrero_2023():
     title = """
     <h1 style='
     text-align: center;
@@ -74,7 +76,7 @@ def reforma_salud_febrero_2023():
     def search(question, retriever_top_k, reader_top_k):
         filters = {"source_title": "Reforma de la salud 13 Febrero 2023"}
         
-        query_result = extractive_query.search_by_query(query = question, 
+        query_result = queries.search_by_query(query = question, 
                                                         retriever_top_k = retriever_top_k, 
                                                         reader_top_k = reader_top_k,
                                                         filters = filters)
@@ -86,7 +88,7 @@ def reforma_salud_febrero_2023():
                         int(item.meta['page']), item.meta['source_url']])
             #result.append([[i+1], item.answer, item.context[:200], item.meta['title']])
         return result
-
+    
     def search_and_show_results():
         # set start time
         stt = time.time()
@@ -98,7 +100,7 @@ def reforma_salud_febrero_2023():
         elapsed_time = round(ent - stt, 2)
 
         # show which query was entered, and what was searching time
-        st.write(f"**Resultados relacionados con:** \"{query}\" ({elapsed_time} sec.)")
+        st.write(f"**Resultados encontrados de las fuentes** \"{query}\" ({elapsed_time} sec.):")
         # then we use loop to show results
         for i, answer in enumerate(results):
             # answer starts with header
@@ -106,12 +108,54 @@ def reforma_salud_febrero_2023():
             # cropped answer
             doc = answer[2][:250] + "..."
             st.markdown(f"{doc}[**Lee más aquí**]({answer[6]})")
-            st.caption(f"Fuente: {answer[4]} - Capítulo: {answer[3]} - Página: {answer[5]}")
+            st.caption(f"Fuente: {answer[4]} - Artículo: {answer[3]} - Página: {answer[5]}")
             
-        st.markdown("---")
+        #st.markdown("---")
 
-    #results = search("que es el adres", retriever_top_k=5, reader_top_k=3)
+    def search_and_generate_answer(question, retriever_top_k, generator_top_k):
+        filters = {"source_title": "Reforma de la salud 13 Febrero 2023"}
+        
+        query_result = queries.genenerate_answer_OpenAI(query = question, 
+                                                        retriever_top_k = retriever_top_k, 
+                                                        generator_top_k = generator_top_k,
+                                                        filters = filters)
+        
+        result = []
+        for i in range(0, len(query_result)):
+            item = query_result[i]
+            source_title = item.meta['doc_metas'][0]['source_title']
+            source_url = item.meta['doc_metas'][0]['source_url']
+            chapter_titles = [source['title'] for source in item.meta['doc_metas']]
+            result.append([[i+1], item.answer.replace("\n",""), 
+                        source_title, source_url, str(chapter_titles)])
+        return result
     
+    def search_and_show_generative_results():
+        # set start time
+        stt = time.time()
+        
+        # retrieve top 5 documents
+        results = search_and_generate_answer(query, retriever_top_k=5, generator_top_k=1)
+        # set endtime
+        ent = time.time()
+        # measure resulting time
+        elapsed_time = round(ent - stt, 2)
+
+        # show which query was entered, and what was searching time
+        st.write(f"**Respuesta generada a partir de los resultados** \"{query}\" ({elapsed_time} sec.):")
+        if results != None:
+            for i, answer in enumerate(results):
+                # answer starts with header
+                st.subheader(f"{answer[1]}")
+                st.markdown(f"[**Lee más aquí**]({answer[3]})")
+                st.caption(f"Fuentes: {answer[2]} - {answer[4]}")
+                
+            st.markdown("---")
+            
+        #st.markdown("---")
+
+    #results = search("que es el adres", retriever_top_k=5, reader_top_k=3)
+
     st.write(f"""<br><p>Cuanto más contexto le des a la pregunta mejores resultados se obtienen. 
              No es un sistema basado en palabras claves, puedes escribir preguntas elaboradas. 
              Una serie de modelos de lenguaje transformers intervienen en cada consulta para ayudar a entenderlas."""
@@ -124,4 +168,11 @@ def reforma_salud_febrero_2023():
             st.error("¡escribe una pregunta!")
         else:
             st.session_state["submit"] = True
-            search_and_show_results()
+            #if not st.session_state.get("OPENAI_API_KEY"):
+            if api_key:
+                search_and_show_generative_results()
+            search_and_show_results()
+
+#r = search_and_generate_answer("que es el ADRES?", retriever_top_k = 5, generator_top_k = 1)
+
+