adding comments

app.py

@@ -1,9 +1,7 @@
 import gradio as gr
 import pandas as pd
 import logging
-import numpy as np
 import os
-import time
 import re
 import json
 from uuid import uuid4
@@ -11,15 +9,11 @@ from datetime import datetime
 from pathlib import Path
 from huggingface_hub import CommitScheduler
 from auditqa.sample_questions import QUESTIONS
-from auditqa.engine.prompts import audience_prompts
 from auditqa.reports import files, report_list
-from auditqa.doc_process import process_pdf, get_local_qdrant
 from langchain.schema import (
     HumanMessage,
     SystemMessage,
 )
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain_community.llms import HuggingFaceEndpoint
 from auditqa.process_chunks import load_chunks, getconfig
 from langchain_community.chat_models.huggingface import ChatHuggingFace
@@ -27,15 +21,19 @@ from langchain.retrievers import ContextualCompressionRetriever
 from langchain.retrievers.document_compressors import CrossEncoderReranker
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
 from qdrant_client.http import models as rest
-#from qdrant_client import QdrantClient
 from dotenv import load_dotenv
-import pkg_resources
 load_dotenv()
+# token to allow acces to Hub, This token should also be 
+# valid fo calls made to Inference endpoints
 HF_token = os.environ["HF_TOKEN"]
+
+# create the local logs repo
 JSON_DATASET_DIR = Path("json_dataset")
 JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
 JSON_DATASET_PATH = JSON_DATASET_DIR / f"logs-{uuid4()}.json"
 
+# the logs are written to dataset repo
+# https://huggingface.co/spaces/Wauplin/space_to_dataset_saver
 scheduler = CommitScheduler(
     repo_id="GIZ/spaces_logs",
     repo_type="dataset",
@@ -44,38 +42,36 @@ scheduler = CommitScheduler(
 )
 
 model_config = getconfig("model_params.cfg")
-#installed_packages = pkg_resources.working_set
-#package_list_ = ""
-#for package in installed_packages:
-#    package_list_ = package_list_ + f"{package.key}=={package.version}\n"
-#print(package_list_)
 
 
-######## Vector Store #######
-# process all files and get the vectorstores collections
-# vectorestore colection are stored on persistent storage so this needs to be run only once
-# hence, comment out line below when creating for first time
+
+#### VECTOR STORE ####
+# reports contain the already created chunks from Markdown version of pdf reports
+# document processing was done using : https://github.com/axa-group/Parsr
 vectorstores = load_chunks()
-# once the vectore embeddings  are created we will qdrant client to access these
-#vectorstores = get_local_qdrant()
 
-# -------------------------------------------------------------
-# Functions
-# -------------------------------------------------------------
+
+#### FUNCTIONS ####
+# App UI and and its functionality is inspired and adapted from
+# https://huggingface.co/spaces/Ekimetrics/climate-question-answering
+
 
 def save_logs(logs) -> None:
+    """ Every interaction with app saves the log of question and answer, 
+        this is to get the usage statistics of app and evaluate model performances 
+    """
     with scheduler.lock:
         with JSON_DATASET_PATH.open("a") as f:
             json.dump(logs, f)
             f.write("\n")
     logging.info("logging done")
-          
+
+
 def make_html_source(source,i):
     """
     takes the text and converts it into html format for display in "source" side tab
     """
     meta = source.metadata
-    # content = source.page_content.split(":",1)[1].strip()
     content = source.page_content.strip()
 
     name = meta['filename']
@@ -120,8 +116,9 @@ def finish_chat():
     return (gr.update(interactive = True,value = ""))
     
 async def chat(query,history,sources,reports,subtype,year):
-    """taking a query and a message history, use a pipeline (reformulation, retriever, answering) to yield a tuple of:
-    (messages in gradio format, messages in langchain format, source documents)"""
+    """taking a query and a message history, use a pipeline (reformulation, retriever, answering) 
+       to yield a tuple of:(messages in gradio format/messages in langchain format, source documents)
+    """
 
     logging.info(f">> NEW QUESTION : {query}")
     logging.info(f"history:{history}")
@@ -133,13 +130,9 @@ async def chat(query,history,sources,reports,subtype,year):
     docs_html = ""
     output_query = ""
 
-    ##------------------------decide which collection to fetch------------------------------
-    if len(reports) == 0:
-        vectorstore = vectorstores["allreports"]
-    else:
-        vectorstore = vectorstores["allreports"]
-
-    ###-------------------------------------Construct Filter------------------------------------
+    ##------------------------fetch collection from vectorstore------------------------------
+    vectorstore = vectorstores["allreports"]
+    ##---------------------construct filter for metdata filtering---------------------------
     if len(reports) == 0:
         ("defining filter for:{}:{}:{}".format(sources,subtype,year))
         filter=rest.Filter(
@@ -165,12 +158,18 @@ async def chat(query,history,sources,reports,subtype,year):
                     )])
         
 
-    ##------------------------------get context----------------------------------------------------  
+    ##------------------------------get context---------------------------------------------- 
     context_retrieved_lst = []
     question_lst= [query]
+
     for question in question_lst:
+        # similarity score threshold can be used to make adjustments in quality and quantity for Retriever
+        # However need to make balancing, as retrieved results are again used by Ranker to fetch best among
+        # retreived results
         retriever = vectorstore.as_retriever(
-          search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.6, "k": int(model_config.get('retriever','TOP_K')), "filter":filter})
+          search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.6, 
+                                                                   "k": int(model_config.get('retriever','TOP_K')), 
+                                                                   "filter":filter})
         model = HuggingFaceCrossEncoder(model_name="BAAI/bge-reranker-base")
         compressor = CrossEncoderReranker(model=model, top_n=3)
         compression_retriever = ContextualCompressionRetriever(
@@ -187,7 +186,7 @@ async def chat(query,history,sources,reports,subtype,year):
         context_retrieved_formatted = format_docs(context_retrieved)
         context_retrieved_lst.append(context_retrieved_formatted)
 
-    ##-------------------Prompt---------------------------------------------------------------
+    ##------------------- -------------Prompt--------------------------------------------------
     SYSTEM_PROMPT = """
         You are AuditQ&A, an AI Assistant created by Auditors and Data Scientist. You are given a question and extracted passages of the consolidated/departmental/thematic focus audit reports. Provide a clear and structured answer based on the passages/context provided and the guidelines.
         Guidelines:
@@ -209,15 +208,13 @@ async def chat(query,history,sources,reports,subtype,year):
         """.format(context = context_retrieved_lst, question=query)
 
     messages = [
-    SystemMessage(content=SYSTEM_PROMPT),
-    HumanMessage(
-        content=USER_PROMPT
-    ),]
+                SystemMessage(content=SYSTEM_PROMPT),
+                HumanMessage(
+                    content=USER_PROMPT
+                ),]
 
-    ###-----------------getting inference endpoints------------------------------
+    ##-----------------------getting inference endpoints------------------------------
 
-    # llama-3_1 endpoint = https://howaqfw0lpap12sg.us-east-1.aws.endpoints.huggingface.cloud
-    # llama-3 endpoint = https://nhe9phsr2zhs0e36.eu-west-1.aws.endpoints.huggingface.cloud
     #callbacks = [StreamingStdOutCallbackHandler()]
     llm_qa = HuggingFaceEndpoint(
         endpoint_url= model_config.get('reader','ENDPOINT'),
@@ -226,10 +223,10 @@ async def chat(query,history,sources,reports,subtype,year):
         timeout=70,
         huggingfacehub_api_token=HF_token,)
 
-    # create rag chain
+    # create RAG
     chat_model = ChatHuggingFace(llm=llm_qa)
     
-    ###-------------------------- get answers ---------------------------------------
+    ##-------------------------- get answers ---------------------------------------
     answer_lst = []
     for question, context in zip(question_lst , context_retrieved_lst):
         answer = chat_model.invoke(messages)
@@ -249,10 +246,9 @@ async def chat(query,history,sources,reports,subtype,year):
         
     yield history,docs_html
 
-
+    # logging the event
     try:
         timestamp = str(datetime.now().timestamp())
-        #file_store = "/data/logs/" + timestamp + ".json"
         logs = {
                 "system_prompt": SYSTEM_PROMPT,
                 "sources":sources,
@@ -271,12 +267,10 @@ async def chat(query,history,sources,reports,subtype,year):
     except Exception as e:
         logging.error(e)
 
-#process_pdf()
 
 
-# --------------------------------------------------------------------
-# Gradio
-# --------------------------------------------------------------------
+
+#### Gradio App ####
 
 # Set up Gradio Theme
 theme = gr.themes.Base(
@@ -323,13 +317,13 @@ with gr.Blocks(title="Audit Q&A", css= "style.css", theme=theme,elem_id = "main-
             with gr.Column(scale=1, variant="panel",elem_id = "right-panel"):
                 # creating tabs on right panel
                 with gr.Tabs() as tabs:
-                    ################## tab for REPORTS SELECTION ##########
+                    #---------------- tab for REPORTS SELECTION ----------------------
                     
                     with gr.Tab("Reports",elem_id = "tab-config",id = 2):
                         gr.Markdown("Reminder: To get better results select the specific report/reports")
 
                         
-                        #### First level filter for selecting Report source/category
+                        #----- First level filter for selecting Report source/category ----------
                         dropdown_sources = gr.Radio(
                             ["Consolidated", "District","Ministry"],
                             label="Select Report Category",
@@ -337,19 +331,19 @@ with gr.Blocks(title="Audit Q&A", css= "style.css", theme=theme,elem_id = "main-
                             interactive=True,
                         )
 
-                        #### second level filter for selecting subtype within the report category selected above
+                        #------ second level filter for selecting subtype within the report category selected above
                         dropdown_category = gr.Dropdown(
                             list(files["Consolidated"].keys()),
                             value = list(files["Consolidated"].keys())[0],
                             label = "Filter for Sub-Type",
                             interactive=True)
 
-                        #### update the secodn level filter abse don values from first level
+                        #----------- update the secodn level filter abse don values from first level ----------------
                         def rs_change(rs):
                             return gr.update(choices=files[rs], value=list(files[rs].keys())[0])
                         dropdown_sources.change(fn=rs_change, inputs=[dropdown_sources], outputs=[dropdown_category])
 
-                        #### Select the years for reports
+                        #--------- Select the years for reports -------------------------------------
                         dropdown_year = gr.Dropdown(
                             ['2018','2019','2020','2021','2022'],
                             label="Filter for year",
@@ -358,7 +352,7 @@ with gr.Blocks(title="Audit Q&A", css= "style.css", theme=theme,elem_id = "main-
                             interactive=True,
                         )
                         gr.Markdown("-------------------------------------------------------------------------")
-                        ##### Another way to select reports across category and sub-type
+                        #---------------- Another way to select reports across category and sub-type ------------
                         dropdown_reports = gr.Dropdown(
                         report_list,
                         label="Or select specific reports",
@@ -396,7 +390,7 @@ with gr.Blocks(title="Audit Q&A", css= "style.css", theme=theme,elem_id = "main-
                                 )
                             
                             samples.append(group_examples)
-                    ########## tab for Sources reporting #################
+                    ##------------------- tab for Sources reporting ##------------------
                     with gr.Tab("Sources",elem_id = "tab-citations",id = 1):
                         sources_textbox = gr.HTML(show_label=False, elem_id="sources-textbox")
                         docs_textbox = gr.State("")

auditqa/doc_process.py

@@ -10,6 +10,9 @@ from qdrant_client import QdrantClient
 from auditqa.reports import files, report_list
 device = 'cuda' if cuda.is_available() else 'cpu'
 
+### This script is NO MORE IN USE ##### 
+# Preprocessed report pdf is brought along with chunks and added to existing reports database
+
 # path to the pdf files
 path_to_data = "./data/pdf/"
 

auditqa/engine/prompts.py

@@ -1,68 +0,0 @@
-llama_propmt =  """<|begin_of_text|>
-        <|start_header_id|>system<|end_header_id|>
-        You are AuditQ&A, an AI Assistant created by Auditors and Data Scientist. You are given a question and extracted passages of the consolidated/departmental/thematic focus audit reports. Provide a clear and structured answer based on the passages provided, the context and the guidelines.
-        Guidelines:
-        - If the passages have useful facts or numbers, use them in your answer.
-        - When you use information from a passage, mention where it came from by using [Doc i] at the end of the sentence. i stands for the number of the document.
-        - Do not use the sentence 'Doc i says ...' to say where information came from.
-        - If the same thing is said in more than one document, you can mention all of them like this: [Doc i, Doc j, Doc k]
-        - Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.
-        - If it makes sense, use bullet points and lists to make your answers easier to understand.
-        - You do not need to use every passage. Only use the ones that help answer the question.
-        - If the documents do not have the information needed to answer the question, just say you do not have enough information.
-        <|eot_id|>
-        <|start_header_id|>user<|end_header_id|>
-        Passages:
-        {context}
-        -----------------------
-        Question: {question} - Explained to {audience}
-        Answer in {language} with the passages citations:
-        <|eot_id|>
-        <|start_header_id|>assistant<|end_header_id|>
-        """
-system_propmt = """
-You are AuditQ&A, an AI Assistant created by Auditors and Data Scientist. You are given a question and extracted passages of the consolidated/departmental/thematic focus audit reports. Provide a clear and structured answer based on the passages provided, the context and the guidelines.
-Guidelines:
-- If the passages have useful facts or numbers, use them in your answer.
-- When you use information from a passage, mention where it came from by using [Doc i] at the end of the sentence. i stands for the number of the document.
-- Do not use the sentence 'Doc i says ...' to say where information came from.
-- If the same thing is said in more than one document, you can mention all of them like this: [Doc i, Doc j, Doc k]
-- Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.
-- If it makes sense, use bullet points and lists to make your answers easier to understand.
-- You do not need to use every passage. Only use the ones that help answer the question.
-- If the documents do not have the information needed to answer the question, just say you do not have enough information.
-"""
-user_propmt = """
-Passages:
-{context}
------------------------
-Question: {question} - Explained to {audience}
-Answer in {language} with the passages citations:
-"""
-
-answer_prompt_template = """
-You are AuditQ&A, an AI Assistant created by Auditors and Data Scientist. You are given a question and extracted passages of the consolidated/departmental/thematic focus audit reports. Provide a clear and structured answer based on the passages provided, the context and the guidelines.
-Guidelines:
-- If the passages have useful facts or numbers, use them in your answer.
-- When you use information from a passage, mention where it came from by using [Doc i] at the end of the sentence. i stands for the number of the document.
-- Do not use the sentence 'Doc i says ...' to say where information came from.
-- If the same thing is said in more than one document, you can mention all of them like this: [Doc i, Doc j, Doc k]
-- Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.
-- If it makes sense, use bullet points and lists to make your answers easier to understand.
-- You do not need to use every passage. Only use the ones that help answer the question.
-- If the documents do not have the information needed to answer the question, just say you do not have enough information.
-- Consider by default that the question is about the past century unless it is specified otherwise. 
-- If the passage is the caption of a picture, you can still use it as part of your answer as any other document.
------------------------
-Passages:
-{context}
------------------------
-Question: {question} - Explained to {audience}
-Answer in {language} with the passages citations:
-"""
-
-audience_prompts = {
-    "children": "6 year old children that don't know anything about audit and governance and need metaphors to learn",
-    "general": "the general public who know the basics in audit and governance and want to learn more about it without technical terms. Still use references to passages.",
-    "experts": "expert and climate scientists that are not afraid of technical terms",
-}

auditqa/reports.py

@@ -1,9 +1,3 @@
-POSSIBLE_REPORTS = [
-    "Consolidated2021",
-    "MWTS2021",
-    "MWTS2022"
-]
-
 report_list = ['Annual Consolidated OAG audit reports 2018',
  'Annual Consolidated OAG audit reports 2019',
  'Annual Consolidated OAG audit reports 2020',

requirements.txt

@@ -1,6 +1,5 @@
 langchain~=0.1.0
 langchain-huggingface==0.0.3
-#langchainhub~=0.1.14
 python-dotenv
 transformers>=4.35.2
 huggingface_hub==0.23.5
@@ -8,4 +7,4 @@ sentence_transformers~=3.0.1
 langchain-qdrant==0.1.3
 qdrant-client~=1.10.1
 PyMuPDF~=1.23.7
-sentencepiece
+sentencepiece==0.2.0