Upload 10 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pdf_resource/BN81-25561C-300_EUG_ROPDVBEUD_EU_ENG_240507.0.pdf filter=lfs diff=lfs merge=lfs -text
+pdf_resource/D3300_NT(En)02.pdf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,48 @@
+import streamlit as st
+import os
+from src.utils.ingest_text import create_vector_database
+from src.utils.ingest_image import extract_and_store_images
+from src.utils.text_qa import qa_bot
+from src.utils.image_qa import query_and_print_results
+import nest_asyncio
+nest_asyncio.apply()
+
+from dotenv import load_dotenv
+load_dotenv()
+
+def get_answer(query,chain):
+    response = chain.invoke(query)
+    return response['result']
+
+st.title("MULTIMODAL DOC QA")
+uploaded_file = st.file_uploader("File upload",type="pdf")
+if uploaded_file is not None:
+    # Save the uploaded file to a temporary location
+    with open(uploaded_file.name, "wb") as f:
+        f.write(uploaded_file.getbuffer())
+
+    # Get the absolute path of the saved file
+    path = os.path.abspath(uploaded_file.name)
+    st.write(f"File saved to: {path}")
+    print(path)
+
+st.write("Document uploaded successfuly!")
+
+
+if st.button("Start Processing"):
+    with st.spinner("Processing"):
+        client = create_vector_database(path)
+        image_vdb = extract_and_store_images(path)
+    chain = qa_bot(client) 
+
+
+    if user_input := st.chat_input("User Input"):
+        with st.chat_message("user"):
+            st.markdown(user_input)
+
+        with st.spinner("Generating Response..."):
+            response = get_answer(chain,user_input)
+            answer = response['result']
+            st.markdown(answer)
+            query_and_print_results(image_vdb,user_input)
+            

data/parsed_data.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfd4afa8be2cd891a38e907116138938acf6ad75db94020d878e6d6725b31c12
+size 125666

pdf_resource/BN81-25561C-300_EUG_ROPDVBEUD_EU_ENG_240507.0.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1cf9a6367dcb45d25e0674217193c0c1c1fbe4f284432febe721fb68b05f91
+size 4355285

pdf_resource/D3300_NT(En)02.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d46bf0dc31ea22547354e845c58d22501fece0e6c5a78e5684e88f3806559d0
+size 5830738

requirements.txt

@@ -0,0 +1,26 @@
+langchain==0.1.15
+langchain-community==0.0.34
+langchain-experimental==0.0.57
+langchain-google-genai==1.0.1
+langchain-openai==0.0.7
+langchain-groq==0.0.1
+langchain-text-splitters==0.0.1
+langchainhub==0.1.15
+fastembed==0.2.5
+llama-index-core==0.10.25.post1
+llama-parse==0.4.0
+llamaindex-py-client==0.1.15
+qdrant-client==1.8.2
+streamlit==1.33.0
+streamlit-float==0.3.2
+unstructured==0.12.5
+unstructured-client==0.18.0
+pypdf==4.3.0
+python-docx==1.1.2
+PyMuPDF==1.24.7
+chromadb
+langchain-core
+open-clip-torch
+langchain_chroma
+qdrant_client
+fastembed

src/utils/image_qa.py

@@ -0,0 +1,20 @@
+import streamlit as st
+from io import BytesIO
+from IPython.display import Image, display
+from PIL import Image as PILImage
+
+def query_and_print_results(image_vdb,query):
+    results=3
+    # Query the database
+    query_results = image_vdb.query(
+        query_texts=[query],
+        n_results=results,
+        include=['uris', 'distances']
+    )
+
+    # Print the results
+    for idx, uri in enumerate(query_results['uris'][0]):
+        img = Image(filename=uri, width=300)
+        st.image(img) # type: ignore
+
+# Testing it out

src/utils/ingest_image.py

@@ -0,0 +1,49 @@
+import os
+import fitz 
+import chromadb
+from chromadb.utils.data_loaders import ImageLoader
+from chromadb.utils.embedding_functions import OpenCLIPEmbeddingFunction # type: ignore
+
+def extract_and_store_images(pdf_path, db_path='image_vdb', images_dir='extracted_images'):
+    # Step 1: Extract images from PDF
+    pdf_document = fitz.open(pdf_path)
+    os.makedirs(images_dir, exist_ok=True)
+
+    for page_num in range(len(pdf_document)):
+        page = pdf_document.load_page(page_num)
+        image_list = page.get_images(full=True)
+
+        for image_index, img in enumerate(image_list):
+            xref = img[0]
+            base_image = pdf_document.extract_image(xref)
+            image_bytes = base_image["image"]
+            image_ext = base_image["ext"]
+            image_filename = f"{images_dir}/page_{page_num+1}_img_{image_index+1}.{image_ext}"
+
+            with open(image_filename, "wb") as image_file:
+                image_file.write(image_bytes)
+            print(f"Saved: {image_filename}")
+
+    print("Image extraction complete.")
+
+    # Step 2: Add extracted images to ChromaDB
+    chroma_client = chromadb.PersistentClient(path=db_path)
+    image_loader = ImageLoader()
+    CLIP = OpenCLIPEmbeddingFunction()
+    image_vdb = chroma_client.get_or_create_collection(name="image", embedding_function=CLIP, data_loader=image_loader)
+
+    ids = []
+    uris = []
+
+    for i, filename in enumerate(sorted(os.listdir(images_dir))):
+        if filename.endswith('.jpeg') or filename.endswith('.png'):
+            file_path = os.path.join(images_dir, filename)
+            ids.append(str(i))
+            uris.append(file_path)
+
+    image_vdb.add(ids=ids, uris=uris)
+    print("Images added to the database.")
+
+    return image_vdb
+
+# Example usage

src/utils/ingest_text.py

@@ -0,0 +1,116 @@
+from llama_parse import LlamaParse
+from langchain_chroma import Chroma
+from qdrant_client import QdrantClient
+from langchain_community.vectorstores.qdrant import Qdrant
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
+from langchain_community.document_loaders.directory import DirectoryLoader
+import os
+from fastembed import TextEmbedding
+from typing import List
+
+import nest_asyncio
+nest_asyncio.apply()
+
+llamaparse_api_key = os.getenv("LLAMA_CLOUD_API_KEY")
+#qdrant_url = os.getenv("QDRANT_URL ")
+#qdrant_api_key = os.getenv("QDRANT_API_KEY")
+groq_api_key = os.getenv("GROQ_API_KEY")
+
+
+parsed_data_file = r"C:\Users\Naresh Kumar Lahajal\Desktop\multimodal\data\parsed_data.pkl"
+output_md = r"C:\Users\Naresh Kumar Lahajal\Desktop\multimodal\data\output.md"
+loki = r"C:\Users\Naresh Kumar Lahajal\Desktop\multimodal\data"
+
+import pickle
+# Define a function to load parsed data if available, or parse if not
+def load_or_parse_data(loc):
+    data_file = parsed_data_file
+
+    if os.path.exists(data_file):
+        # Load the parsed data from the file
+        with open(data_file, "rb") as f:
+            parsed_data = pickle.load(f)
+    else:
+        # Perform the parsing step and store the result in llama_parse_documents
+        parsingInstructiontest10k = """The provided document is an user guide or a manual.
+        It contains many images and tables.
+        Try to be precise while answering the questions"""
+        parser = LlamaParse(api_key=llamaparse_api_key, result_type="markdown", parsing_instruction=parsingInstructiontest10k) # type: ignore
+        llama_parse_documents = parser.load_data(loc)
+
+
+        # Save the parsed data to a file
+        with open(data_file, "wb") as f:
+            pickle.dump(llama_parse_documents, f)
+
+        # Set the parsed data to the variable
+        parsed_data = llama_parse_documents
+
+    return parsed_data
+
+
+# Create vector database
+def create_vector_database(loc):
+    """
+    Creates a vector database using document loaders and embeddings.
+
+    This function loads urls,
+    splits the loaded documents into chunks, transforms them into embeddings using OllamaEmbeddings,
+    and finally persists the embeddings into a Chroma vector database.
+
+    """
+    # Call the function to either load or parse the data
+    llama_parse_documents = load_or_parse_data(loc)
+    #print(llama_parse_documents[1].text[:100])
+
+    #with open('data/output.md', 'a') as f:  # Open the file in append mode ('a')
+      #  for doc in llama_parse_documents:
+       #     f.write(doc.text + '\n')
+    with open(output_md,'a', encoding='utf-8') as f:  # Open the file in append mode ('a')
+        for doc in llama_parse_documents:
+            f.write(doc.text + '\n')
+
+    loader = DirectoryLoader(loki, glob="**/*.md", show_progress=True)
+    documents = loader.load()
+    # Split loaded documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
+    print('data chunckex')
+    docs = text_splitter.split_documents(documents)
+    print(len(docs))
+
+    #len(docs)
+    #docs[0]
+
+    # Initialize Embeddings
+    embeddings = FastEmbedEmbeddings() # type: ignore
+    #embeddings = TextEmbedding()
+
+    print('Vector DB started!')
+
+    # Create and persist a Chroma vector database from the chunked documents
+    qdrant = Qdrant.from_documents(
+        documents=docs,
+        embedding=embeddings,
+        path="local_qdrant",
+        #url=qdrant_url,
+        collection_name="rag"
+        #api_key=qdrant_api_key
+    )
+    # save to disk
+    #db2 = Chroma.from_documents(docs, embeddings, persist_directory="./chroma_db")
+    #docs = db2.similarity_search(query)
+
+# load from disk
+    #db3 = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
+
+    #query it
+    #query = "what is the agend of Financial Statements for 2022 ?"
+    #found_doc = qdrant.similarity_search(query, k=3)
+    #print(found_doc[0][:100])
+    #
+    print('Vector DB created successfully !')
+    #query = "Switching between external devices connected to the TV"
+    #found_doc = qdrant.similarity_search(query, k=3)
+    #print(found_doc)
+    return qdrant

src/utils/text_qa.py

@@ -0,0 +1,91 @@
+import os
+from typing import List
+from langchain_groq import ChatGroq
+from langchain.prompts import PromptTemplate
+from langchain_community.vectorstores.qdrant import Qdrant
+from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
+from qdrant_client import QdrantClient
+#from langchain_community.chat_models import ChatOllama
+
+
+#import chainlit as cl
+from langchain.chains import RetrievalQA
+
+# bring in our GROQ_API_KEY
+from dotenv import load_dotenv
+load_dotenv()
+
+
+llamaparse_api_key = os.getenv("LLAMA_CLOUD_API_KEY")
+qdrant_url = os.getenv("QDRANT_URL ")
+qdrant_api_key = os.getenv("QDRANT_API_KEY")
+groq_api_key = os.getenv("GROQ_API_KEY")
+
+custom_prompt_template = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know,if it is out of context say that it is out of context and also try to provide the answer and don't be rude.
+
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+
+def set_custom_prompt():
+    """
+    Prompt template for QA retrieval for each vectorstore
+    """
+    prompt = PromptTemplate(template=custom_prompt_template,
+                            input_variables=['context', 'question'])
+    return prompt
+
+
+chat_model = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768",api_key=groq_api_key) # type: ignore
+#chat_model = ChatGroq(temperature=0, model_name="Llama2-70b-4096")
+#chat_model = ChatOllama(model="llama2", request_timeout=30.0)
+
+#client = QdrantClient(api_key=qdrant_api_key, url=qdrant_url,)
+
+
+def retrieval_qa_chain(llm, prompt, vectorstore):
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=llm,
+        chain_type="stuff",
+        retriever=vectorstore.as_retriever(search_kwargs={'k': 2}),
+        return_source_documents=True,
+        chain_type_kwargs={'prompt': prompt}
+    )
+    return qa_chain
+
+
+def qa_bot(qdrant):
+    embeddings = FastEmbedEmbeddings() # type: ignore
+    vectorstore = qdrant
+    llm = chat_model
+    qa_prompt=set_custom_prompt()
+    qa = retrieval_qa_chain(llm, qa_prompt, vectorstore)
+    return qa
+
+#---------------------------------------------------------------------#
+ 
+#qdrant_cloud_api_key="your_qdrant_cloud_api_key"
+#qdrant_url="your_qdrant_url"
+
+#qdrant_cloud = Qdrant.from_documents(
+#    docs,
+#   embeddings,
+#   url=qdrant_url,
+#   prefer_grpc=True,
+#   api_key=qdrant_cloud_api_key,
+#  collection_name="qdrant_cloud_documents",
+#)
+
+#---------------------------------------------------------------------#
+#query="how to make coffee"
+#print(query)
+
+#chain = qa_bot()
+#response = chain.invoke(query,)
+
+#print(response['result'])
+