Upload 10 files

utils/chatbot.py

@@ -0,0 +1,171 @@
+import gradio as gr
+import time
+import os
+from langchain.vectorstores import Chroma
+from typing import List, Tuple
+import re
+import ast
+import html
+from utils.load_config import LoadConfig
+from langchain.embeddings import HuggingFaceEmbeddings
+import requests
+import torch
+FLASK_APP_ENDPOINT = "http://127.0.0.1:8888/generate_text"
+
+APPCFG = LoadConfig()
+URL = ""
+hyperlink = f"[RAG]({URL})"
+
+
+class ChatBot:
+    """
+    Class representing a chatbot with document retrieval and response generation capabilities.
+
+    This class provides static methods for responding to user queries, handling feedback, and
+    cleaning references from retrieved documents.
+    """
+    @staticmethod
+    def respond(chatbot: List,
+                message: str,
+                data_type: str = "Preprocessed doc",
+                temperature: float = 0.1,
+                top_k: int = 10,
+                top_p: float = 0.1) -> Tuple:
+        """
+        Generate a response to a user query using document retrieval and language model completion.
+
+        Parameters:
+            chatbot (List): List representing the chatbot's conversation history.
+            message (str): The user's query.
+            data_type (str): Type of data used for document retrieval ("Preprocessed doc" or "Upload doc: Process for RAG").
+            temperature (float): Temperature parameter for language model completion.
+
+        Returns:
+            Tuple: A tuple containing an empty string, the updated chat history, and references from retrieved documents.
+        """
+
+        # Retrieve embedding function from code env resources
+        # emb_model = "sentence-transformers/all-MiniLM-L6-v2"
+        embedding_function = HuggingFaceEmbeddings(
+            model_name="NeuML/pubmedbert-base-embeddings",
+            # cache_folder=os.getenv('SENTENCE_TRANSFORMERS_HOME')
+        )
+        if data_type == "Preprocessed doc":
+            # directories
+            if os.path.exists(APPCFG.persist_directory):
+                vectordb = Chroma(persist_directory=APPCFG.persist_directory,
+                                  embedding_function=embedding_function)
+            else:
+                chatbot.append(
+                    (message, f"VectorDB does not exist. Please first execute the 'upload_data_manually.py' module. For further information please visit {hyperlink}."))
+                return "", chatbot, None
+
+        elif data_type == "Upload doc: Process for RAG":
+            if os.path.exists(APPCFG.custom_persist_directory):
+                vectordb = Chroma(persist_directory=APPCFG.custom_persist_directory,
+                                  embedding_function=embedding_function)
+            else:
+                chatbot.append(
+                    (message, f"No file was uploaded. Please first upload your files using the 'upload' button."))
+                return "", chatbot, None
+
+        docs = vectordb.similarity_search(message, k=APPCFG.k)
+        question = "# Prompt that you have to answer:\n" + message
+        retrieved_content, markdown_documents = ChatBot.clean_references(docs)
+        # Memory: previous two Q&A pairs
+        chat_history = f"Chat history:\n {str(chatbot[-APPCFG.number_of_q_a_pairs:])}\n\n"
+        if APPCFG.add_history:
+            prompt_wrapper = f"{APPCFG.llm_system_role_with_history}\n\n{chat_history}\n\n{retrieved_content}{question}"
+        else:
+            prompt_wrapper = f"{APPCFG.llm_system_role_without_history}\n\n{question}\n\n{retrieved_content}"
+
+        print("========================")
+        print(prompt_wrapper)
+        print("========================")
+        messages = [
+            {"role": "user", "content": prompt_wrapper},
+        ]
+        data = {
+            "prompt": messages,
+            "max_new_tokens": APPCFG.max_new_tokens,
+            "do_sample": APPCFG.do_sample,
+            "temperature": temperature,
+            "top_k": top_k,
+            "top_p": top_p
+        }
+        response = requests.post(FLASK_APP_ENDPOINT, json=data)
+        # print(response.text)
+        response_json = response.json()
+
+        chatbot.append(
+            (message, response_json["response"]))
+        # Clean up GPU memory
+        del vectordb
+        del docs
+        torch.cuda.empty_cache()
+        return "", chatbot, markdown_documents
+
+    @staticmethod
+    def clean_references(documents: List) -> str:
+        """
+        Clean and format references from retrieved documents.
+
+        Parameters:
+            documents (List): List of retrieved documents.
+
+        Returns:
+            str: A string containing cleaned and formatted references.
+        """
+        server_url = "http://localhost:8000"
+        documents = [str(x)+"\n\n" for x in documents]
+        markdown_documents = ""
+        retrieved_content = ""
+        counter = 1
+        for doc in documents:
+            # Extract content and metadata
+            content, metadata = re.match(
+                r"page_content=(.*?)( metadata=\{.*\})", doc).groups()
+            metadata = metadata.split('=', 1)[1]
+            metadata_dict = ast.literal_eval(metadata)
+
+            # Decode newlines and other escape sequences
+            content = bytes(content, "utf-8").decode("unicode_escape")
+
+            # Replace escaped newlines with actual newlines
+            content = re.sub(r'\\n', '\n', content)
+            content = re.sub(r'\s*<EOS>\s*<pad>\s*', ' ', content)
+            content = re.sub(r'\s+', ' ', content).strip()
+
+
+            # Decode HTML entities
+            content = html.unescape(content)
+
+            # Replace incorrect unicode characters with correct ones
+            #content = content.encode('utf-8').decode('utf-8', 'ignore')
+            # Use UTF-8 encoding instead of latin-1 to avoid encoding issues
+            content = content.encode('utf-8', 'ignore').decode('utf-8', 'ignore')
+            
+
+
+            # Remove or replace special characters and mathematical symbols
+            # This step may need to be customized based on the specific symbols in your documents
+            content = re.sub(r'–', '-', content)
+            content = re.sub(r'∈', '∈', content)
+            content = re.sub(r'×', '×', content)
+            content = re.sub(r'fi', 'fi', content)
+            content = re.sub(r'∈', '∈', content)
+            content = re.sub(r'·', '·', content)
+            content = re.sub(r'fl', 'fl', content)
+
+            pdf_url = f"{server_url}/{os.path.basename(metadata_dict['source'])}"
+            retrieved_content += f"# Content {counter}:\n" + \
+                content + "\n\n"
+
+            # Append cleaned content to the markdown string with two newlines between documents
+            markdown_documents += f"# Retrieved content {counter}:\n" + content + "\n\n" + \
+                f"Source: {os.path.basename(metadata_dict['source'])}" + " | " +\
+                f"Page number: {str(metadata_dict['page'])}" + " | " +\
+                f"[View PDF]({pdf_url})" "\n\n"
+            counter += 1
+
+        return retrieved_content, markdown_documents

utils/load_config.py

@@ -0,0 +1,89 @@
+
+import os
+from dotenv import load_dotenv
+import yaml
+from pyprojroot import here
+import shutil
+
+load_dotenv()
+
+
+class LoadConfig:
+    """
+    A class for loading configuration settings and managing directories.
+
+    This class loads various configuration settings from the 'app_config.yml' file,
+    including language model (LLM) configurations, retrieval configurations, summarizer
+    configurations, and memory configurations. It also sets up OpenAI API credentials
+    and performs directory-related operations such as creating and removing directories.
+    """
+
+    def __init__(self) -> None:
+        with open(here("configs/app_config.yml")) as cfg:
+            app_config = yaml.load(cfg, Loader=yaml.FullLoader)
+
+        # LLM configs
+        self.llm_engine = app_config["llm_config"]["engine"]
+        self.llm_system_role_with_history = app_config["llm_config"]["llm_system_role_with_history"]
+        self.llm_system_role_without_history = app_config[
+            "llm_config"]["llm_system_role_without_history"]
+        self.persist_directory = str(here(
+            app_config["directories"]["persist_directory"]))  # needs to be strin for summation in chromadb backend: self._settings.require("persist_directory") + "/chroma.sqlite3"
+        self.custom_persist_directory = str(here(
+            app_config["directories"]["custom_persist_directory"]))
+        self.gemma_token = os.getenv("GEMMA_TOKEN")
+        self.device = app_config["llm_config"]["device"]
+        # Retrieval configs
+        self.data_directory = app_config["directories"]["data_directory"]
+        self.k = app_config["retrieval_config"]["k"]
+        self.chunk_size = int(app_config["splitter_config"]["chunk_size"])
+        self.chunk_overlap = int(
+            app_config["splitter_config"]["chunk_overlap"])
+        self.temperature = float(app_config["llm_config"]["temperature"])
+        self.add_history = bool(app_config["llm_config"]["add_history"])
+        self.top_k = int(app_config["llm_config"]["top_k"])
+        self.top_p = float(app_config["llm_config"]["top_p"])
+        self.max_new_tokens = int(app_config["llm_config"]["max_new_tokens"])
+        self.do_sample = bool(app_config["llm_config"]["do_sample"])
+        self.embedding_model = app_config["llm_config"]["embedding_model"]
+
+        # Memory
+        self.number_of_q_a_pairs = int(
+            app_config["memory"]["number_of_q_a_pairs"])
+
+        # clean up the upload doc vectordb if it exists
+        self.create_directory(self.persist_directory)
+        self.remove_directory(self.custom_persist_directory)
+
+    def create_directory(self, directory_path: str):
+        """
+        Create a directory if it does not exist.
+
+        Parameters:
+            directory_path (str): The path of the directory to be created.
+        """
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+
+    def remove_directory(self, directory_path: str):
+        """
+        Removes the specified directory.
+
+        Parameters:
+            directory_path (str): The path of the directory to be removed.
+
+        Raises:
+            OSError: If an error occurs during the directory removal process.
+
+        Returns:
+            None
+        """
+        if os.path.exists(directory_path):
+            try:
+                shutil.rmtree(directory_path)
+                print(
+                    f"The directory '{directory_path}' has been successfully removed.")
+            except OSError as e:
+                print(f"Error: {e}")
+        else:
+            print(f"The directory '{directory_path}' does not exist.")

utils/prepare_vectordb.py

@@ -0,0 +1,117 @@
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import os
+from typing import List
+from langchain.embeddings import HuggingFaceEmbeddings
+
+
+class PrepareVectorDB:
+    """
+    A class for preparing and saving a VectorDB using OpenAI embeddings.
+
+    This class facilitates the process of loading documents, chunking them, and creating a VectorDB
+    with OpenAI embeddings. It provides methods to prepare and save the VectorDB.
+
+    Parameters:
+        data_directory (str or List[str]): The directory or list of directories containing the documents.
+        persist_directory (str): The directory to save the VectorDB.
+        chunk_size (int): The size of the chunks for document processing.
+        chunk_overlap (int): The overlap between chunks.
+    """
+
+    def __init__(
+            self,
+            data_directory: str,
+            persist_directory: str,
+            chunk_size: int,
+            chunk_overlap: int
+    ) -> None:
+        """
+        Initialize the PrepareVectorDB instance.
+
+        Parameters:
+            data_directory (str or List[str]): The directory or list of directories containing the documents.
+            persist_directory (str): The directory to save the VectorDB.
+            chunk_size (int): The size of the chunks for document processing.
+            chunk_overlap (int): The overlap between chunks.
+
+        """
+
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        """Other options: CharacterTextSplitter, TokenTextSplitter, etc."""
+        self.data_directory = data_directory
+        self.persist_directory = persist_directory
+        self.embedding_function = HuggingFaceEmbeddings(
+            model_name="NeuML/pubmedbert-base-embeddings",
+            # cache_folder=os.getenv('SENTENCE_TRANSFORMERS_HOME')
+        )
+
+    def __load_all_documents(self) -> List:
+        """
+        Load all documents from the specified directory or directories.
+
+        Returns:
+            List: A list of loaded documents.
+        """
+        doc_counter = 0
+        if isinstance(self.data_directory, list):
+            print("Loading the uploaded documents...")
+            docs = []
+            for doc_dir in self.data_directory:
+                docs.extend(PyPDFLoader(doc_dir).load())
+                doc_counter += 1
+            print("Number of loaded documents:", doc_counter)
+            print("Number of pages:", len(docs), "\n\n")
+        else:
+            print("Loading documents manually...")
+            document_list = os.listdir(self.data_directory)
+            docs = []
+            for doc_name in document_list:
+                docs.extend(PyPDFLoader(os.path.join(
+                    self.data_directory, doc_name)).load())
+                doc_counter += 1
+            print("Number of loaded documents:", doc_counter)
+            print("Number of pages:", len(docs), "\n\n")
+
+        return docs
+
+    def __chunk_documents(self, docs: List) -> List:
+        """
+        Chunk the loaded documents using the specified text splitter.
+
+        Parameters:
+            docs (List): The list of loaded documents.
+
+        Returns:
+            List: A list of chunked documents.
+
+        """
+        print("Chunking documents...")
+        chunked_documents = self.text_splitter.split_documents(docs)
+        print("Number of chunks:", len(chunked_documents), "\n\n")
+        return chunked_documents
+
+    def prepare_and_save_vectordb(self):
+        """
+        Load, chunk, and create a VectorDB with OpenAI embeddings, and save it.
+
+        Returns:
+            Chroma: The created VectorDB.
+        """
+        docs = self.__load_all_documents()
+        chunked_documents = self.__chunk_documents(docs)
+        print("Preparing vectordb...")
+        vectordb = Chroma.from_documents(
+            documents=chunked_documents,
+            embedding=self.embedding_function,
+            persist_directory=self.persist_directory
+        )
+        print("VectorDB is created and saved.")
+        print("Number of vectors in vectordb:",
+              vectordb._collection.count(), "\n\n")
+        return vectordb

utils/ui_settings.py

@@ -0,0 +1,35 @@
+import gradio as gr
+
+
+class UISettings:
+    """
+    Utility class for managing UI settings.
+
+    This class provides static methods for toggling UI components, such as a sidebar.
+    """
+    @staticmethod
+    def toggle_sidebar(state):
+        """
+        Toggle the visibility state of a UI component.
+
+        Parameters:
+            state: The current state of the UI component.
+
+        Returns:
+            Tuple: A tuple containing the updated UI component state and the new state.
+        """
+        state = not state
+        return gr.update(visible=state), state
+
+    @staticmethod
+    def feedback(data: gr.LikeData):
+        """
+        Process user feedback on the generated response.
+
+        Parameters:
+            data (gr.LikeData): Gradio LikeData object containing user feedback.
+        """
+        if data.liked:
+            print("You upvoted this response: " + data.value)
+        else:
+            print("You downvoted this response: " + data.value)

utils/upload_file.py

@@ -0,0 +1,39 @@
+from utils.prepare_vectordb import PrepareVectorDB
+from typing import List, Tuple
+from utils.load_config import LoadConfig
+
+APPCFG = LoadConfig()
+
+
+class UploadFile:
+    """
+    Utility class for handling file uploads and processing.
+
+    This class provides static methods for checking directories and processing uploaded files
+    to prepare a VectorDB.
+    """
+
+    @staticmethod
+    def process_uploaded_files(files_dir: List, chatbot: List, rag_with_dropdown: str) -> Tuple:
+        """
+        Process uploaded files to prepare a VectorDB.
+
+        Parameters:
+            files_dir (List): List of paths to the uploaded files.
+            chatbot: An instance of the chatbot for communication.
+
+        Returns:
+            Tuple: A tuple containing an empty string and the updated chatbot instance.
+        """
+        if rag_with_dropdown == "Upload doc: Process for RAG":
+            prepare_vectordb_instance = PrepareVectorDB(data_directory=files_dir,
+                                                        persist_directory=APPCFG.custom_persist_directory,
+                                                        chunk_size=APPCFG.chunk_size,
+                                                        chunk_overlap=APPCFG.chunk_overlap)
+            prepare_vectordb_instance.prepare_and_save_vectordb()
+            chatbot.append(
+                (" ", "Uploaded files are ready. Please ask your question"))
+        else:
+            chatbot.append(
+                (" ", "If you would like to upload a PDF, please select your desired action in 'rag_with' dropdown."))
+        return "", chatbot