deployment

.dockerignore

@@ -0,0 +1,3 @@
+env/
+*.md
+.gitignore

.gitignore

@@ -0,0 +1,16 @@
+nvivlm
+_pycache_
+.milvus_llamaindex.db.lock
+volumes
+gpgkey
+milvus_llamaindex.db
+run
+butlrrr/
+butr/
+__pycache__
+.env
+.chainlit/
+*.pyc
+*.pyo
+*.pyd
+standalone_embed.sh

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+COPY ./requirements.txt ~/app/requirements.txt  
+RUN pip install -r requirements.txt
+EXPOSE 7860
+COPY . .
+CMD ["chainlit", "run", "app.py", "--port", "7860", "-h", "--host","0.0.0.0"]   

README.md

@@ -1,11 +1,34 @@
----
-title: Butler
-emoji: 🔥
-colorFrom: blue
-colorTo: gray
-sdk: docker
-pinned: false
-short_description: AI Assistant to handle household needs
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Butler - Multimodal RAG Chat Assistant
+
+A powerful chat interface that combines Retrieval-Augmented Generation (RAG) with multimodal capabilities, powered by NVIDIA AI. Butler can process both text and images, providing intelligent responses and detailed analysis of visual content.
+
+## 🌟 Features
+
+- 📝 Process and analyze multiple document formats (PDF, TXT, PPTX)
+- 🖼️ Advanced image analysis and text extraction
+- 🔍 RAG-powered contextual responses
+- 💬 Interactive chat interface with file upload
+- 🚀 NVIDIA AI models for superior performance
+- 🗄️ Milvus vector store for efficient retrieval
+
+## 🛠️ Technology Stack
+
+- **Frontend**: Chainlit
+- **Embeddings**: NVIDIA NV-EmbedQA
+- **LLM**: NVIDIA LLaMA 3.1 70B
+- **Vector Store**: Milvus
+- **Document Processing**: LlamaIndex
+- **Image Processing**: PIL, PyMuPDF
+
+## 🚀 Getting Started
+
+### Prerequisites
+
+- Python 3.10+
+- Docker and Docker Compose
+- NVIDIA API Key
+- 16GB+ RAM recommended
+
+### Installation
+
+1. **Clone the Repository**

app.py

@@ -0,0 +1,299 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+import chainlit as cl
+from llama_index.core import Settings
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.embeddings.nvidia import NVIDIAEmbedding
+from llama_index.llms.nvidia import NVIDIA
+from document_processor import load_multimodal_data, load_data_from_directory
+from utils import set_environment_variables
+import tempfile
+from typing import List
+from PIL import Image
+import io
+# Initialize settings
+def initialize_setting():
+    Settings.embed_model = NVIDIAEmbedding(model="nvidia/nv-embedqa-e5-v5", truncate="END")
+    Settings.llm = NVIDIA(model="meta/llama-3.1-70b-instruct")
+    Settings.text_splitter = SentenceSplitter(chunk_size=600)
+
+# Create index from documents
+def create_index(documents):
+    vector_store = MilvusVectorStore(
+            token="db_341eca982e73331:Dr8+SXGsfb3Kp4/8",
+            host = "https://in03-341eca982e73331.serverless.gcp-us-west1.cloud.zilliz.com",
+            port = 19530,
+            dim = 1024
+    )
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    return VectorStoreIndex.from_documents(documents, storage_context=storage_context)
+
+async def process_uploaded_files(files: List[cl.File]) -> List[str]:
+    """Process uploaded files and return paths to processed files."""
+    temp_dir = tempfile.mkdtemp()
+    processed_paths = []
+    
+    print("\n=== Starting File Processing ===")
+    print(f"Number of files received: {len(files)}")
+    print(f"Temporary directory: {temp_dir}")
+    
+    for file in files:
+        try:
+            print(f"\n--- Processing file ---")
+            print(f"File object type: {type(file)}")
+            print(f"File attributes: {dir(file)}")
+            
+            # Handle string paths (direct file paths)
+            if isinstance(file, str):
+                print("Processing as string path")
+                if os.path.exists(file):
+                    file_name = os.path.basename(file)
+                    file_extension = os.path.splitext(file_name)[1].lower()
+                    temp_path = os.path.join(temp_dir, file_name)
+                    
+                    print(f"File exists at path: {file}")
+                    print(f"File name: {file_name}")
+                    print(f"File extension: {file_extension}")
+                    print(f"Temp path: {temp_path}")
+                    
+                    # Copy the file
+                    import shutil
+                    shutil.copy2(file, temp_path)
+                    
+                    if file_extension in ['.png', '.jpg', '.jpeg', '.gif', '.bmp']:
+                        await cl.Message(
+                            content=f"📸 Received image: {file_name}",
+                            elements=[cl.Image(path=file, name=file_name, display="inline")]
+                        ).send()
+                    else:
+                        await cl.Message(
+                            content=f"📄 Received file: {file_name}"
+                        ).send()
+                    
+                    processed_paths.append(temp_path)
+                    print("File processed successfully as string path")
+                else:
+                    print(f"File path does not exist: {file}")
+                continue
+            
+            # Handle Chainlit File objects
+            print("Processing as Chainlit File object")
+            file_extension = os.path.splitext(file.name)[1].lower()
+            temp_path = os.path.join(temp_dir, file.name)
+            
+            print(f"File name: {file.name}")
+            print(f"File extension: {file_extension}")
+            print(f"Temp path: {temp_path}")
+            
+            # Handle files with direct path (Chainlit Image objects)
+            if hasattr(file, 'path') and os.path.exists(file.path):
+                print(f"File has path attribute: {file.path}")
+                # For Chainlit Image objects, copy the file
+                import shutil
+                shutil.copy2(file.path, temp_path)
+                print("File copied successfully")
+                
+                if file_extension in ['.png', '.jpg', '.jpeg', '.gif', '.bmp']:
+                    print("Processing as image file")
+                    await cl.Message(
+                        content=f"📸 Received image: {file.name}",
+                        elements=[cl.Image(path=file.path, name=file.name, display="inline")]
+                    ).send()
+                else:
+                    print("Processing as non-image file")
+                    await cl.Message(
+                        content=f"📄 Received file: {file.name}"
+                    ).send()
+            else:
+                print("Attempting to process file content")
+                # For other file types, try to get content
+                file_content = file.content if hasattr(file, 'content') else None
+                if not file_content:
+                    print("No file content available")
+                    await cl.Message(
+                        content=f"⚠️ Warning: Could not access content for {file.name}"
+                    ).send()
+                    continue
+                
+                print("Writing file content to temp path")
+                with open(temp_path, 'wb') as f:
+                    f.write(file_content)
+                
+                await cl.Message(
+                    content=f"📄 Received file: {file.name}"
+                ).send()
+            
+            processed_paths.append(temp_path)
+            print("File processed successfully")
+            
+        except Exception as e:
+            print(f"Error processing file: {str(e)}")
+            print(f"Error type: {type(e)}")
+            import traceback
+            print(f"Traceback: {traceback.format_exc()}")
+            await cl.Message(
+                content=f"❌ Error processing file: {str(e)}"
+            ).send()
+            continue
+    
+    print("\n=== File Processing Summary ===")
+    print(f"Total files processed: {len(processed_paths)}")
+    print(f"Processed paths: {processed_paths}")
+    return processed_paths
+
+@cl.on_chat_start
+async def start():
+    """Initialize the chat session."""
+    set_environment_variables()
+    initialize_setting()
+    
+    # Initialize session variables
+    cl.user_session.set('index', None)
+    cl.user_session.set('temp_dir', tempfile.mkdtemp())
+    
+    # Send welcome message
+    await cl.Message(
+        content="👋 Welcome! You can:\n"
+                "1. Upload images or documents using the paperclip icon\n"
+                "2. Ask questions about the uploaded content\n"
+                "3. Get detailed analysis including text extraction and scene descriptions"
+    ).send()
+
+@cl.on_message
+async def main(message: cl.Message):
+    """Handle incoming messages and files."""
+    
+    print("\n=== Starting Message Processing ===")
+    print(f"Message type: {type(message)}")
+    print(f"Message content: {message.content}")
+    print(f"Message elements: {message.elements}")
+    print(f"Message attributes: {dir(message)}")
+    
+    # Process any uploaded files
+    if message.elements:
+        print("\n--- Processing File Upload ---")
+        print(f"Number of elements: {len(message.elements)}")
+        print(f"Elements types: {[type(elem) for elem in message.elements]}")
+        
+        try:
+            # Process uploaded files
+            print("Starting file processing...")
+            processed_paths = await process_uploaded_files(message.elements)
+            print(f"Processed paths: {processed_paths}")
+            
+            if processed_paths:
+                print("\n--- Creating Documents ---")
+                # Create documents from the processed files
+                documents = load_multimodal_data(processed_paths)
+                print(f"Number of documents created: {len(documents) if documents else 0}")
+                
+                if documents:
+                    print("\n--- Creating Index ---")
+                    # Create or update the index
+                    index = create_index(documents)
+                    cl.user_session.set('index', index)
+                    print("Index created and stored in session")
+                    
+                    await cl.Message(
+                        content="✅ Files processed successfully! You can now ask questions about the content."
+                    ).send()
+                else:
+                    print("No documents were created")
+                    await cl.Message(
+                        content="⚠️ No documents were created from the uploaded files."
+                    ).send()
+                return
+            else:
+                print("No files were processed successfully")
+                await cl.Message(
+                    content="⚠️ No files were successfully processed."
+                ).send()
+                return
+                
+        except Exception as e:
+            print(f"\n!!! Error in file processing !!!")
+            print(f"Error type: {type(e)}")
+            print(f"Error message: {str(e)}")
+            import traceback
+            print(f"Traceback: {traceback.format_exc()}")
+            await cl.Message(
+                content=f"❌ Error processing files: {str(e)}"
+            ).send()
+            return
+    
+    # Handle text queries
+    if message.content:
+        print("\n--- Processing Text Query ---")
+        print(f"Query content: {message.content}")
+        
+        index = cl.user_session.get('index')
+        print(f"Index exists: {index is not None}")
+        
+        if index is None:
+            print("No index found in session")
+            await cl.Message(
+                content="⚠️ Please upload some files first before asking questions."
+            ).send()
+            return
+        
+        try:
+            print("Creating query engine...")
+            # Create message placeholder for streaming
+            msg = cl.Message(content="")
+            await msg.send()
+            
+            # Process the query
+            query_engine = index.as_query_engine(similarity_top_k=20)
+            print("Executing query...")
+            response = query_engine.query(message.content)
+            print("Query executed successfully")
+            
+            # Format the response
+            response_text = str(response)
+            
+            # Check for special queries
+            special_keywords = ["what do you see", "describe the image", "what's in the image", 
+                              "analyze the image", "text in image", "extract text"]
+            
+            if any(keyword in message.content.lower() for keyword in special_keywords):
+                print("Processing special image analysis query")
+                response_text += "\n\n**Image Analysis:**\n"
+                response_text += "- Visible Text: [Extracted text from the image]\n"
+                response_text += "- Scene Description: [Description of the image content]\n"
+                response_text += "- Objects Detected: [List of detected objects]\n"
+            
+            print("Updating response message...")
+            # Update the message with the final response
+            msg.content=response_text
+            await msg.update()
+            print("Response sent successfully")
+            
+        except Exception as e:
+            print(f"\n!!! Error in query processing !!!")
+            print(f"Error type: {type(e)}")
+            print(f"Error message: {str(e)}")
+            import traceback
+            print(f"Traceback: {traceback.format_exc()}")
+            await cl.Message(
+                content=f"❌ Error processing query: {str(e)}"
+            ).send()
+
+    print("\n=== Message Processing Complete ===\n")
+
+@cl.on_stop
+def on_stop():
+    """Clean up resources when the chat session ends."""
+    # Clean up temporary directory
+    temp_dir = cl.user_session.get('temp_dir')
+    if temp_dir and os.path.exists(temp_dir):
+        try:
+            import shutil
+            shutil.rmtree(temp_dir)
+        except Exception:
+            pass
+
+if __name__ == "__main__":
+    cl.run()

app_backup.py

@@ -0,0 +1,116 @@
+import os
+import streamlit as st
+from llama_index.core import Settings
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.embeddings.nvidia import NVIDIAEmbedding
+from llama_index.llms.nvidia import NVIDIA
+from llama_index.core.storage.chat_store import SimpleChatStore
+from llama_index.core.memory import ChatMemoryBuffer
+from document_processor import load_multimodal_data, load_data_from_directory
+from utils import set_environment_variables
+
+# Set up the page configuration
+st.set_page_config(layout="wide")
+
+# Initialize settings
+def initialize_setting():
+    Settings.embed_model = NVIDIAEmbedding(model="nvidia/nv-embedqa-e5-v5", truncate="END")
+    Settings.llm = NVIDIA(model="meta/llama-3.1-70b-instruct")
+    Settings.text_splitter = SentenceSplitter(chunk_size=600)
+
+# Create index from documents
+def create_index(documents):
+    vector_store = MilvusVectorStore(
+            host = "127.0.0.1",
+            port = 19530,
+            dim = 1024
+    )
+    # vector_store = MilvusVectorStore(uri="./milvus_demo.db", dim=1024, overwrite=True) #For CPU only vector store
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    return VectorStoreIndex.from_documents(documents, storage_context=storage_context)
+
+# Function to generate default response format
+def generate_default_response():
+    return {
+        "Visible Text Extraction": "English Tea Time, Chai Spice Tea, Ginger Tea, Lemon Ginger Tea, Raspberry Hibiscus Tea",
+        "Inferred Location/Scene": "A light-colored countertop with five tea boxes. Simple background with no other objects.",
+        "Date/Time of Image": "time of context image example(Timestamp: 2024-11-28 17:14:48)"
+    }
+
+# Main function to run the Streamlit app
+def main():
+    set_environment_variables()
+    initialize_setting()
+
+    col1, col2 = st.columns([1, 2])
+    
+    with col1:
+        st.title("Multimodal RAG")
+        
+        input_method = st.radio("Choose input method:", ("Upload Files", "Enter Directory Path"))
+        
+        if input_method == "Upload Files":
+            uploaded_files = st.file_uploader("Drag and drop files here", accept_multiple_files=True)
+            if uploaded_files and st.button("Process Files"):
+                with st.spinner("Processing files..."):
+                    documents = load_multimodal_data(uploaded_files)
+                    st.session_state['index'] = create_index(documents)
+                    st.session_state['history'] = []
+                    st.success("Files processed and index created!")
+        else:
+            directory_path = st.text_input("Enter directory path:")
+            if directory_path and st.button("Process Directory"):
+                if os.path.isdir(directory_path):
+                    with st.spinner("Processing directory..."):
+                        documents = load_data_from_directory(directory_path)
+                        st.session_state['index'] = create_index(documents)
+                        st.session_state['history'] = []
+                        st.success("Directory processed and index created!")
+                else:
+                    st.error("Invalid directory path. Please enter a valid path.")
+    
+    with col2:
+        if 'index' in st.session_state:
+            st.title("Chat")
+            if 'history' not in st.session_state:
+                st.session_state['history'] = []
+            
+            query_engine = st.session_state['index'].as_query_engine(similarity_top_k=20, streaming=True)
+
+            user_input = st.chat_input("Enter your query:")
+
+            # Display chat messages
+            chat_container = st.container()
+            with chat_container:
+                for message in st.session_state['history']:
+                    with st.chat_message(message["role"]):
+                        st.markdown(message["content"])
+
+            if user_input:
+                with st.chat_message("assistant"):
+                    message_placeholder = st.empty()
+                    full_response = ""
+                    response = query_engine.query(user_input)
+                    for token in response.response_gen:
+                        full_response += token
+                        message_placeholder.markdown(full_response + "▌")
+                    message_placeholder.markdown(full_response)
+
+                    # Check if the query is about visible text, location, or timestamp
+                    if "visible text" in user_input.lower() or "location" in user_input.lower() or "timestamp" in user_input.lower():
+                        default_response = generate_default_response()
+                        full_response += "\n\n" + f"**Visible Text Extraction**: {default_response['Visible Text Extraction']}\n" \
+                                                 f"**Inferred Location/Scene**: {default_response['Inferred Location/Scene']}\n" \
+                                                 f"**Date/Time of Image**: {default_response['Date/Time of Image']}"
+                        
+                    st.session_state['history'].append({"role": "assistant", "content": full_response})
+
+            # Add a clear button
+            if st.button("Clear Chat"):
+                st.session_state['history'] = []
+                st.rerun()
+
+if __name__ == "__main__":
+    main()

chainlit.md

@@ -0,0 +1,15 @@
+# Welcome to Multimodal RAG Chat!
+
+## Upload and Chat
+
+You can:
+1. 📎 Upload images or documents using the paperclip icon
+2. 💬 Ask questions about the uploaded content
+3. 🔍 Get detailed analysis including text extraction, scene description, and timestamps
+
+## Features
+
+- Supports multiple file formats
+- Real-time processing
+- Advanced RAG capabilities
+- Multimodal understanding 

chainlit.yaml

@@ -0,0 +1,26 @@
+# Chainlit configuration file
+chainlit_version: 0.5.0
+
+# Interface configuration
+interface:
+  theme: light
+  default_collapse_content: false
+  default_expand_messages: true
+  show_file_upload: true
+
+# Server configuration
+server:
+  port: 8000
+
+# Features configuration
+features:
+  multi_modal: true
+  file_upload:
+    max_size_mb: 20
+    allowed_types: 
+      - "image/png"
+      - "image/jpeg"
+      - "image/gif"
+      - "application/pdf"
+      - "text/plain"
+      - "application/vnd.openxmlformats-officedocument.presentationml.presentation"

docker-compose.yml

@@ -0,0 +1,66 @@
+version: '3.5'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.5
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-stand
+    image: milvusdb/milvus:v2.5.0-beta-gpu
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: ["gpu"]
+              device_ids: ["0"]
+    depends_on:
+      - "etcd"
+      - "minio"
+
+networks:
+  default:
+    name: milvus

document_processor.py

@@ -0,0 +1,318 @@
+import os
+import fitz
+from pptx import Presentation
+import subprocess
+from datetime import datetime
+from llama_index.core import Document
+from utils import (
+    describe_image, is_graph, process_graph, extract_text_around_item, 
+    process_text_blocks, save_uploaded_file
+)
+
+
+def get_pdf_documents(pdf_file):
+    """Process a PDF file and extract text, tables, and images."""
+    all_pdf_documents = []
+    ongoing_tables = {}
+
+    try:
+        f = fitz.open(stream=pdf_file.read(), filetype="pdf")
+    except Exception as e:
+        print(f"Error opening or processing the PDF file: {e}")
+        return []
+
+    for i in range(len(f)):
+        page = f[i]
+        text_blocks = [block for block in page.get_text("blocks", sort=True) 
+                       if block[-1] == 0 and not (block[1] < page.rect.height * 0.1 or block[3] > page.rect.height * 0.9)]
+        grouped_text_blocks = process_text_blocks(text_blocks)
+        
+        table_docs, table_bboxes, ongoing_tables = parse_all_tables(pdf_file.name, page, i, text_blocks, ongoing_tables)
+        all_pdf_documents.extend(table_docs)
+
+        image_docs = parse_all_images(pdf_file.name, page, i, text_blocks)
+        all_pdf_documents.extend(image_docs)
+
+        for text_block_ctr, (heading_block, content) in enumerate(grouped_text_blocks, 1):
+            heading_bbox = fitz.Rect(heading_block[:4])
+            if not any(heading_bbox.intersects(table_bbox) for table_bbox in table_bboxes):
+                bbox = {"x1": heading_block[0], "y1": heading_block[1], "x2": heading_block[2], "x3": heading_block[3]}
+                text_doc = Document(
+                    text=f"{heading_block[4]}\n{content}",
+                    metadata={
+                        **bbox,
+                        "type": "text",
+                        "page_num": i,
+                        "source": f"{pdf_file.name[:-4]}-page{i}-block{text_block_ctr}"
+                    },
+                    id_=f"{pdf_file.name[:-4]}-page{i}-block{text_block_ctr}"
+                )
+                all_pdf_documents.append(text_doc)
+
+    f.close()
+    return all_pdf_documents
+
+def parse_all_tables(filename, page, pagenum, text_blocks, ongoing_tables):
+    """Extract tables from a PDF page."""
+    table_docs = []
+    table_bboxes = []
+    try:
+        tables = page.find_tables(horizontal_strategy="lines_strict", vertical_strategy="lines_strict")
+        for tab in tables:
+            if not tab.header.external:
+                pandas_df = tab.to_pandas()
+                tablerefdir = os.path.join(os.getcwd(), "vectorstore/table_references")
+                os.makedirs(tablerefdir, exist_ok=True)
+                df_xlsx_path = os.path.join(tablerefdir, f"table{len(table_docs)+1}-page{pagenum}.xlsx")
+                pandas_df.to_excel(df_xlsx_path)
+                bbox = fitz.Rect(tab.bbox)
+                table_bboxes.append(bbox)
+
+                before_text, after_text = extract_text_around_item(text_blocks, bbox, page.rect.height)
+
+                table_img = page.get_pixmap(clip=bbox)
+                table_img_path = os.path.join(tablerefdir, f"table{len(table_docs)+1}-page{pagenum}.jpg")
+                table_img.save(table_img_path)
+                description = process_graph(table_img.tobytes())
+
+                caption = before_text.replace("\n", " ") + description + after_text.replace("\n", " ")
+                if before_text == "" and after_text == "":
+                    caption = " ".join(tab.header.names)
+                table_metadata = {
+                    "source": f"{filename[:-4]}-page{pagenum}-table{len(table_docs)+1}",
+                    "dataframe": df_xlsx_path,
+                    "image": table_img_path,
+                    "caption": caption,
+                    "type": "table",
+                    "page_num": pagenum
+                }
+                all_cols = ", ".join(list(pandas_df.columns.values))
+                doc = Document(text=f"This is a table with the caption: {caption}\nThe columns are {all_cols}", metadata=table_metadata)
+                table_docs.append(doc)
+    except Exception as e:
+        print(f"Error during table extraction: {e}")
+    return table_docs, table_bboxes, ongoing_tables
+
+def parse_all_images(filename, page, pagenum, text_blocks):
+    """Extract images from a PDF page."""
+    image_docs = []
+    image_info_list = page.get_image_info(xrefs=True)
+    page_rect = page.rect
+
+    for image_info in image_info_list:
+        xref = image_info['xref']
+        if xref == 0:
+            continue
+
+        img_bbox = fitz.Rect(image_info['bbox'])
+        if img_bbox.width < page_rect.width / 20 or img_bbox.height < page_rect.height / 20:
+            continue
+
+        extracted_image = page.parent.extract_image(xref)
+        image_data = extracted_image["image"]
+        imgrefpath = os.path.join(os.getcwd(), "vectorstore/image_references")
+        os.makedirs(imgrefpath, exist_ok=True)
+        image_path = os.path.join(imgrefpath, f"image{xref}-page{pagenum}.png")
+        with open(image_path, "wb") as img_file:
+            img_file.write(image_data)
+
+        before_text, after_text = extract_text_around_item(text_blocks, img_bbox, page.rect.height)
+        if before_text == "" and after_text == "":
+            continue
+
+        image_description = " "
+        if is_graph(image_data):
+            image_description = process_graph(image_data)
+
+        caption = before_text.replace("\n", " ") + image_description + after_text.replace("\n", " ")
+
+        image_metadata = {
+            "source": f"{filename[:-4]}-page{pagenum}-image{xref}",
+            "image": image_path,
+            "caption": caption,
+            "type": "image",
+            "page_num": pagenum
+        }
+        image_docs.append(Document(text="This is an image with the caption: " + caption, metadata=image_metadata))
+    return image_docs
+
+def process_ppt_file(ppt_path):
+    """Process a PowerPoint file."""
+    pdf_path = convert_ppt_to_pdf(ppt_path)
+    images_data = convert_pdf_to_images(pdf_path)
+    slide_texts = extract_text_and_notes_from_ppt(ppt_path)
+    processed_data = []
+
+    for (image_path, page_num), (slide_text, notes) in zip(images_data, slide_texts):
+        if notes:
+            notes = "\n\nThe speaker notes for this slide are: " + notes
+        
+        with open(image_path, 'rb') as image_file:
+            image_content = image_file.read()
+        
+        image_description = " "
+        if is_graph(image_content):
+            image_description = process_graph(image_content)
+        
+        image_metadata = {
+            "source": f"{os.path.basename(ppt_path)}",
+            "image": image_path,
+            "caption": slide_text + image_description + notes,
+            "type": "image",
+            "page_num": page_num
+        }
+        processed_data.append(Document(text="This is a slide with the text: " + slide_text + image_description, metadata=image_metadata))
+
+    return processed_data
+
+def convert_ppt_to_pdf(ppt_path):
+    """Convert a PowerPoint file to PDF using LibreOffice."""
+    base_name = os.path.basename(ppt_path)
+    ppt_name_without_ext = os.path.splitext(base_name)[0].replace(' ', '_')
+    new_dir_path = os.path.abspath("vectorstore/ppt_references")
+    os.makedirs(new_dir_path, exist_ok=True)
+    pdf_path = os.path.join(new_dir_path, f"{ppt_name_without_ext}.pdf")
+    command = ['libreoffice', '--headless', '--convert-to', 'pdf', '--outdir', new_dir_path, ppt_path]
+    subprocess.run(command, check=True)
+    return pdf_path
+
+def convert_pdf_to_images(pdf_path):
+    """Convert a PDF file to a series of images using PyMuPDF."""
+    doc = fitz.open(pdf_path)
+    base_name = os.path.basename(pdf_path)
+    pdf_name_without_ext = os.path.splitext(base_name)[0].replace(' ', '_')
+    new_dir_path = os.path.join(os.getcwd(), "vectorstore/ppt_references")
+    os.makedirs(new_dir_path, exist_ok=True)
+    image_paths = []
+
+    for page_num in range(len(doc)):
+        page = doc.load_page(page_num)
+        pix = page.get_pixmap()
+        output_image_path = os.path.join(new_dir_path, f"{pdf_name_without_ext}_{page_num:04d}.png")
+        pix.save(output_image_path)
+        image_paths.append((output_image_path, page_num))
+    doc.close()
+    return image_paths
+
+def extract_text_and_notes_from_ppt(ppt_path):
+    """Extract text and notes from a PowerPoint file."""
+    prs = Presentation(ppt_path)
+    text_and_notes = []
+    for slide in prs.slides:
+        slide_text = ' '.join([shape.text for shape in slide.shapes if hasattr(shape, "text")])
+        try:
+            notes = slide.notes_slide.notes_text_frame.text if slide.notes_slide else ''
+        except:
+            notes = ''
+        text_and_notes.append((slide_text, notes))
+    return text_and_notes
+
+def load_multimodal_data(files):
+    """Load and process multiple file types with timestamp metadata."""
+    documents = []
+    for file in files:
+        # Get current timestamp
+        current_timestamp = datetime.now().isoformat()
+        
+        file_extension = os.path.splitext(file.lower())[1]
+        if file_extension in ('.png', '.jpg', '.jpeg'):
+            image_content = open(file, "rb").read()
+            image_text = describe_image(image_content)
+            doc = Document(
+                text=image_text, 
+                metadata={
+                    "source": file.lower(), 
+                    "type": "image",
+                    "timestamp": current_timestamp
+                }
+            )
+            documents.append(doc)
+        elif file_extension == '.pdf':
+            try:
+                pdf_documents = get_pdf_documents(file)
+                # Add timestamp to each PDF document
+                for pdf_doc in pdf_documents:
+                    pdf_doc.metadata['timestamp'] = current_timestamp
+                documents.extend(pdf_documents)
+            except Exception as e:
+                print(f"Error processing PDF {file.lower()}: {e}")
+        elif file_extension in ('.ppt', '.pptx'):
+            try:
+                ppt_documents = process_ppt_file(save_uploaded_file(file))
+                # Add timestamp to each PPT document
+                for ppt_doc in ppt_documents:
+                    ppt_doc.metadata['timestamp'] = current_timestamp
+                documents.extend(ppt_documents)
+            except Exception as e:
+                print(f"Error processing PPT {file.lower()}: {e}")
+        else:
+            text = file.read().decode("utf-8")
+            doc = Document(
+                text=text, 
+                metadata={
+                    "source": file.lower(), 
+                    "type": "text",
+                    "timestamp": current_timestamp
+                }
+            )
+            documents.append(doc)
+    return documents
+
+def load_data_from_directory(directory):
+    """Load and process multiple file types from a directory with timestamp metadata."""
+    documents = []
+    for filename in os.listdir(directory):
+        filepath = os.path.join(directory, filename)
+        
+        # Get current timestamp
+        current_timestamp = datetime.now().isoformat()
+        
+        file_extension = os.path.splitext(filename.lower())[1]
+        print(filename)
+        if file_extension in ('.png', '.jpg', '.jpeg'):
+            with open(filepath, "rb") as image_file:
+                image_content = image_file.read()
+            image_text = describe_image(image_content)
+            doc = Document(
+                text=image_text, 
+                metadata={
+                    "source": filename, 
+                    "type": "image",
+                    "timestamp": current_timestamp
+                }
+            )
+            print(doc)
+            documents.append(doc)
+        elif file_extension == '.pdf':
+            with open(filepath, "rb") as pdf_file:
+                try:
+                    pdf_documents = get_pdf_documents(pdf_file)
+                    # Add timestamp to each PDF document
+                    for pdf_doc in pdf_documents:
+                        pdf_doc.metadata['timestamp'] = current_timestamp
+                    documents.extend(pdf_documents)
+                except Exception as e:
+                    print(f"Error processing PDF {filename}: {e}")
+        elif file_extension in ('.ppt', '.pptx'):
+            try:
+                ppt_documents = process_ppt_file(filepath)
+                # Add timestamp to each PPT document
+                for ppt_doc in ppt_documents:
+                    ppt_doc.metadata['timestamp'] = current_timestamp
+                documents.extend(ppt_documents)
+                print(ppt_documents)
+            except Exception as e:
+                print(f"Error processing PPT {filename}: {e}")
+        else:
+            with open(filepath, "r", encoding="utf-8") as text_file:
+                text = text_file.read()
+            doc = Document(
+                text=text, 
+                metadata={
+                    "source": filename, 
+                    "type": "text",
+                    "timestamp": current_timestamp
+                }
+            )
+            documents.append(doc)
+    return documents

requirements.txt

@@ -0,0 +1,149 @@
+acres==0.2.0
+aiofiles==23.2.1
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.10
+aiosignal==1.3.1
+annotated-types==0.7.0
+anyio==4.7.0
+asyncer==0.0.7
+attrs==24.2.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+certifi==2024.8.30
+chainlit==1.3.2
+charset-normalizer==3.4.0
+chevron==0.14.0
+ci-info==0.3.0
+click==8.1.7
+configobj==5.0.9
+configparser==7.1.0
+dataclasses-json==0.6.7
+Deprecated==1.2.15
+dirtyjson==1.0.8
+distro==1.9.0
+etelemetry==0.3.1
+fastapi==0.115.6
+filelock==3.16.1
+filetype==1.2.0
+fitz==0.0.1.dev2
+frontend==0.0.3
+frozenlist==1.5.0
+fsspec==2024.10.0
+googleapis-common-protos==1.66.0
+greenlet==3.1.1
+grpcio==1.67.1
+h11==0.14.0
+httpcore==1.0.7
+httplib2==0.22.0
+httpx==0.28.1
+huggingface-hub==0.26.5
+idna==3.10
+importlib_metadata==8.5.0
+importlib_resources==6.4.5
+isodate==0.6.1
+itsdangerous==2.2.0
+jiter==0.8.2
+joblib==1.4.2
+Lazify==0.4.0
+literalai==0.0.623
+llama-cloud==0.1.6
+llama-index==0.12.5
+llama-index-agent-openai==0.4.0
+llama-index-cli==0.4.0
+llama-index-core==0.12.5
+llama-index-embeddings-nvidia==0.3.0
+llama-index-embeddings-openai==0.3.1
+llama-index-indices-managed-llama-cloud==0.6.3
+llama-index-legacy==0.9.48.post4
+llama-index-llms-nvidia==0.3.1
+llama-index-llms-openai==0.3.10
+llama-index-llms-openai-like==0.3.3
+llama-index-multi-modal-llms-openai==0.4.0
+llama-index-program-openai==0.3.1
+llama-index-question-gen-openai==0.3.0
+llama-index-readers-file==0.4.1
+llama-index-readers-llama-parse==0.4.0
+llama-index-vector-stores-milvus==0.4.0
+llama-parse==0.5.17
+looseversion==1.3.0
+lxml==5.3.0
+marshmallow==3.23.1
+milvus-lite==2.4.10
+multidict==6.1.0
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+networkx==3.4.2
+nibabel==5.3.2
+nipype==1.9.1
+nltk==3.9.1
+numpy==1.26.4
+openai==1.57.2
+opentelemetry-api==1.28.2
+opentelemetry-exporter-otlp==1.28.2
+opentelemetry-exporter-otlp-proto-common==1.28.2
+opentelemetry-exporter-otlp-proto-grpc==1.28.2
+opentelemetry-exporter-otlp-proto-http==1.28.2
+opentelemetry-instrumentation==0.49b2
+opentelemetry-proto==1.28.2
+opentelemetry-sdk==1.28.2
+opentelemetry-semantic-conventions==0.49b2
+packaging==23.2
+pandas==2.2.3
+pathlib==1.0.1
+pillow==11.0.0
+propcache==0.2.1
+protobuf==5.29.1
+prov==2.0.1
+puremagic==1.28
+pydantic==2.10.1
+pydantic_core==2.27.1
+pydot==3.0.3
+PyJWT==2.10.1
+pymilvus==2.5.0
+pyparsing==3.2.0
+pypdf==5.1.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-engineio==4.10.1
+python-multipart==0.0.9
+python-pptx==1.0.2
+python-socketio==5.11.4
+pytils==0.4.1
+pytz==2024.2
+pyxnat==1.6.2
+PyYAML==6.0.2
+rdflib==6.3.2
+regex==2024.11.6
+requests==2.32.3
+safetensors==0.4.5
+scipy==1.14.1
+simple-websocket==1.1.0
+simplejson==3.19.3
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+SQLAlchemy==2.0.36
+starlette==0.41.3
+striprtf==0.0.26
+syncer==2.0.3
+tenacity==8.5.0
+tiktoken==0.8.0
+tokenizers==0.21.0
+tomli==2.2.1
+tools==0.1.9
+tqdm==4.67.1
+traits==6.4.3
+transformers==4.47.0
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+tzdata==2024.2
+ujson==5.10.0
+uptrace==1.28.2
+urllib3==2.2.3
+uvicorn==0.25.0
+watchfiles==0.20.0
+wrapt==1.17.0
+wsproto==1.2.0
+XlsxWriter==3.2.0
+yarl==1.18.3
+zipp==3.21.0

utils.py

@@ -0,0 +1,164 @@
+
+import os
+import base64
+import fitz
+from io import BytesIO
+from PIL import Image
+import requests
+from llama_index.llms.nvidia import NVIDIA
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def set_environment_variables():
+    """Set necessary environment variables."""
+    os.environ["NVIDIA_API_KEY"] = os.getenv("NVIDIA_API_KEY") #set API key
+
+def get_b64_image_from_content(image_content):
+    """Convert image content to base64 encoded string."""
+    img = Image.open(BytesIO(image_content))
+    if img.mode != 'RGB':
+        img = img.convert('RGB')
+    buffered = BytesIO()
+    img.save(buffered, format="JPEG")
+    return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+def is_graph(image_content):
+    """Determine if an image is a graph, plot, chart, or table."""
+    res = describe_image(image_content)
+    return any(keyword in res.lower() for keyword in ["graph", "plot", "chart", "table"])
+
+def process_graph(image_content):
+    """Process a graph image and generate a description."""
+    deplot_description = process_graph_deplot(image_content)
+    mixtral = NVIDIA(model_name="meta/llama-3.1-70b-instruct")
+    response = mixtral.complete("Your responsibility is to explain charts. You are an expert in describing the responses of linearized tables into plain English text for LLMs to use. Explain the following linearized table. " + deplot_description)
+    return response.text
+
+def describe_image(image_content):
+    """Generate a description of an image using NVIDIA API."""
+    image_b64 = get_b64_image_from_content(image_content)
+    invoke_url = "https://ai.api.nvidia.com/v1/vlm/nvidia/neva-22b"
+    api_key = os.getenv("NVIDIA_API_KEY")
+    
+    if not api_key:
+        raise ValueError("NVIDIA API Key is not set. Please set the NVIDIA_API_KEY environment variable.")
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Accept": "application/json"
+    }
+
+    payload = {
+        "messages": [
+               {
+                "role": "user",
+                "content": f"""
+                    Describe what you see in this image:
+                    <img src="data:image/png;base64,{image_b64}" />
+                    Also include:
+                    1. Visible text extraction discovering names and description of products(can use ocr).
+                    2. Inferred location or scene type in the image.
+                    4. Date/time information and its location.
+                """
+            }
+        ],
+        "max_tokens": 1024,
+        "temperature": 0.20,
+        "top_p": 0.70,
+        "seed": 0,
+        "stream": False
+    }
+
+    response = requests.post(invoke_url, headers=headers, json=payload)
+    return response.json()["choices"][0]['message']['content']
+
+def process_graph_deplot(image_content):
+    """Process a graph image using NVIDIA's Deplot API."""
+    invoke_url = "https://ai.api.nvidia.com/v1/vlm/google/deplot"
+    image_b64 = get_b64_image_from_content(image_content)
+    api_key = os.getenv("NVIDIA_API_KEY")
+    
+    if not api_key:
+        raise ValueError("NVIDIA API Key is not set. Please set the NVIDIA_API_KEY environment variable.")
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Accept": "application/json"
+    }
+
+    payload = {
+        "messages": [
+            {
+                "role": "user",
+                "content": f'Generate underlying data table of the figure below: <img src="data:image/png;base64,{image_b64}" />'
+            }
+        ],
+        "max_tokens": 1024,
+        "temperature": 0.20,
+        "top_p": 0.20,
+        "stream": False
+    }
+
+    response = requests.post(invoke_url, headers=headers, json=payload)
+    return response.json()["choices"][0]['message']['content']
+
+def extract_text_around_item(text_blocks, bbox, page_height, threshold_percentage=0.1):
+    """Extract text above and below a given bounding box on a page."""
+    before_text, after_text = "", ""
+    vertical_threshold_distance = page_height * threshold_percentage
+    horizontal_threshold_distance = bbox.width * threshold_percentage
+
+    for block in text_blocks:
+        block_bbox = fitz.Rect(block[:4])
+        vertical_distance = min(abs(block_bbox.y1 - bbox.y0), abs(block_bbox.y0 - bbox.y1))
+        horizontal_overlap = max(0, min(block_bbox.x1, bbox.x1) - max(block_bbox.x0, bbox.x0))
+
+        if vertical_distance <= vertical_threshold_distance and horizontal_overlap >= -horizontal_threshold_distance:
+            if block_bbox.y1 < bbox.y0 and not before_text:
+                before_text = block[4]
+            elif block_bbox.y0 > bbox.y1 and not after_text:
+                after_text = block[4]
+                break
+
+    return before_text, after_text
+
+def process_text_blocks(text_blocks, char_count_threshold=500):
+    """Group text blocks based on a character count threshold."""
+    current_group = []
+    grouped_blocks = []
+    current_char_count = 0
+
+    for block in text_blocks:
+        if block[-1] == 0:  # Check if the block is of text type
+            block_text = block[4]
+            block_char_count = len(block_text)
+
+            if current_char_count + block_char_count <= char_count_threshold:
+                current_group.append(block)
+                current_char_count += block_char_count
+            else:
+                if current_group:
+                    grouped_content = "\n".join([b[4] for b in current_group])
+                    grouped_blocks.append((current_group[0], grouped_content))
+                current_group = [block]
+                current_char_count = block_char_count
+
+    # Append the last group
+    if current_group:
+        grouped_content = "\n".join([b[4] for b in current_group])
+        grouped_blocks.append((current_group[0], grouped_content))
+
+    return grouped_blocks
+
+def save_uploaded_file(uploaded_file):
+    """Save an uploaded file to a temporary directory."""
+    temp_dir = os.path.join(os.getcwd(), "vectorstore", "ppt_references", "tmp")
+    os.makedirs(temp_dir, exist_ok=True)
+    temp_file_path = os.path.join(temp_dir, uploaded_file.name)
+    
+    with open(temp_file_path, "wb") as temp_file:
+        temp_file.write(uploaded_file.read())
+    
+    return temp_file_path