hooha

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[theme]
+base="dark"

pages/1_🔐_Credentials.py

@@ -0,0 +1,75 @@
+import streamlit as st
+
+st.set_page_config(
+    page_title="Credentials",
+    page_icon="🔐",
+)
+
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+
+
+# """
+# Initialising session states  
+# """
+if 'openai_api_key' not in st.session_state:
+    st.session_state['openai_api_key'] = None
+
+if 'pinecone_api_key' not in st.session_state:
+    st.session_state['pinecone_api_key'] = None
+
+if 'pinecone_env' not in st.session_state:
+    st.session_state['pinecone_env'] = None
+
+if 'pinecone_index_namespace' not in st.session_state:
+    st.session_state['pinecone_index_namespace'] = None
+
+if 'requests' not in st.session_state:
+    st.session_state['requests'] = []
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'buffer_memory' not in st.session_state:
+    st.session_state.buffer_memory=ConversationBufferWindowMemory(k=3,return_messages=True)
+
+st.write("# Enter your Credentials! 🔐")
+
+# """
+# Page Content  
+# """
+st.markdown(
+"""
+Enter the following credentials to start uploading and querying documents.
+"""
+)
+
+# OpenAI API Key input
+openai_api_key = st.text_input("Enter your OpenAI API Key", type='password')
+if openai_api_key:
+    # Use the OpenAI API key (e.g., validate it, make a request to an API, etc.)
+    st.session_state['openai_api_key'] = openai_api_key
+    
+# Pinecone API Key input
+pinecone_api_key = st.text_input("Enter your Pinecone API Key", type='password')
+if pinecone_api_key:
+    st.session_state['pinecone_api_key'] = pinecone_api_key
+
+# Pinecone Environment input
+pinecone_env = st.text_input("Enter your Pinecone Environment", type='password')
+if pinecone_env:
+    st.session_state['pinecone_env'] = pinecone_env
+
+# Index Namespace input
+pinecone_index_namespace = st.text_input("Enter your Pinecone Index Namespace", type='password')
+if pinecone_index_namespace:
+    st.session_state['pinecone_index_namespace'] = pinecone_index_namespace
+
+# Check if all required fields are filled
+all_fields_filled = all([st.session_state['pinecone_api_key'], 
+                         st.session_state['openai_api_key'], 
+                         st.session_state['pinecone_env'], 
+                         st.session_state['pinecone_index_namespace']])
+
+if all_fields_filled:
+    st.success('Credentials Stored')
+

pages/2_📄_Upload_Documents.py

@@ -0,0 +1,58 @@
+import streamlit as st
+
+st.set_page_config(
+    page_title="Upload Documents",
+    page_icon="📄",
+)
+
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+from utils import ingest
+
+# """
+# Initialising session states  
+# """
+if 'openai_api_key' not in st.session_state:
+    st.session_state['openai_api_key'] = None
+
+if 'pinecone_api_key' not in st.session_state:
+    st.session_state['pinecone_api_key'] = None
+
+if 'pinecone_env' not in st.session_state:
+    st.session_state['pinecone_env'] = None
+
+if 'pinecone_index_namespace' not in st.session_state:
+    st.session_state['pinecone_index_namespace'] = None
+
+if 'requests' not in st.session_state:
+    st.session_state['requests'] = []
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'buffer_memory' not in st.session_state:
+    st.session_state.buffer_memory=ConversationBufferWindowMemory(k=3,return_messages=True)
+
+
+# """
+# Page Content  
+# """
+st.write("# Upload your Documents! 📄")
+
+# Check if all required fields are filled
+all_fields_filled = all([st.session_state['pinecone_api_key'], 
+                         st.session_state['openai_api_key'], 
+                         st.session_state['pinecone_env'], 
+                         st.session_state['pinecone_index_namespace']])
+
+if not all_fields_filled:
+    st.error('Credentials 🔐 not found. Enter Credentials 🔐 to activate uploader')
+    uploaded_file = st.file_uploader("Upload Document", type=['pdf'], disabled=True)
+
+else:
+    uploaded_file = st.file_uploader("Upload Document", type=['pdf', 'docx'], disabled=False)
+    if uploaded_file:
+        ingest(uploaded_file, 
+               pinecone_api_key=st.session_state['pinecone_api_key'],
+               pinecone_env=st.session_state['pinecone_env'],
+               pinecone_index_namespace=st.session_state['pinecone_index_namespace']
+               )

pages/3_🤖_Chatbot.py

@@ -0,0 +1,105 @@
+# 🤖
+import streamlit as st
+
+st.set_page_config(
+    page_title="Chatbot",
+    page_icon="📄",
+)
+
+from streamlit_chat import message
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationChain
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+from langchain.prompts import (
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+    ChatPromptTemplate,
+    MessagesPlaceholder
+)
+from utils import *
+
+# """
+# Initialising session states  
+# """
+if 'openai_api_key' not in st.session_state:
+    st.session_state['openai_api_key'] = None
+
+if 'pinecone_api_key' not in st.session_state:
+    st.session_state['pinecone_api_key'] = None
+
+if 'pinecone_env' not in st.session_state:
+    st.session_state['pinecone_env'] = None
+
+if 'pinecone_index_namespace' not in st.session_state:
+    st.session_state['pinecone_index_namespace'] = None
+
+if 'requests' not in st.session_state:
+    st.session_state['requests'] = []
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'buffer_memory' not in st.session_state:
+    st.session_state.buffer_memory=ConversationBufferWindowMemory(k=3,return_messages=True)
+
+system_msg_template = SystemMessagePromptTemplate.from_template(template="""Answer the question as truthfully as possible using the provided context, 
+and if the answer is not contained within the text below, say 'I don't know'""")
+
+human_msg_template = HumanMessagePromptTemplate.from_template(template="{input}")
+
+prompt_template = ChatPromptTemplate.from_messages([system_msg_template, MessagesPlaceholder(variable_name="history"), human_msg_template])
+
+empty_openai_api_key = False
+
+try:
+    llm = ChatOpenAI(model_name="gpt-3.5-turbo", openai_api_key=st.session_state['openai_api_key'])
+    conversation = ConversationChain(memory=st.session_state.buffer_memory, prompt=prompt_template, llm=llm, verbose=True)
+except:
+    empty_openai_api_key = True
+
+# """
+# Page Content  
+# """
+st.write("# Chat with your Documents! 🤖")
+
+if empty_openai_api_key:
+    st.error('Enter OpenAI API key in credentials tab')
+
+else:
+    try:
+        # if st.button('Say hello'):
+        #     st.write('Why hello there')
+        
+        # container for chat history
+        response_container = st.container()
+        # container for text box
+        textcontainer = st.container()
+
+
+        with textcontainer:
+            query = st.text_input("Query: ", key="input")
+            if query:
+                with st.spinner("typing..."):
+                    conversation_string = get_conversation_string()
+                    # st.code(conversation_string)
+                    refined_query = query_refiner(conversation_string, query)
+                    # st.subheader("Refined Query:")
+                    # st.write(refined_query)
+                    context = find_match(refined_query,
+                                        pinecone_api_key=st.session_state['pinecone_api_key'],
+                                        pinecone_env=st.session_state['pinecone_env'],
+                                        pinecone_index_namespace=st.session_state['pinecone_index_namespace']
+                                        )
+                    # print(context)  
+                    response = conversation.predict(input=f"Context:\n {context} \n\n Query:\n{query}")
+                st.session_state.requests.append(query)
+                st.session_state.responses.append(response) 
+        with response_container:
+            if st.session_state['responses']:
+
+                for i in range(len(st.session_state['responses'])):
+                    message(st.session_state['responses'][i],key=str(i))
+                    if i < len(st.session_state['requests']):
+                        message(st.session_state["requests"][i], is_user=True,key=str(i)+ '_user')
+    except Exception as error_message:
+        st.error("Error occured. Check your API key")

requirements.txt

@@ -0,0 +1,12 @@
+altair<5
+streamlit==1.17.0
+streamlit-chat
+langchain
+openai
+sentence-transformers
+unstructured
+unstructured[local-inference]
+pinecone-client
+tabulate
+chromadb
+pypdf

utils.py

@@ -0,0 +1,83 @@
+from langchain.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import SentenceTransformerEmbeddings
+from sentence_transformers import SentenceTransformer
+import pinecone 
+from langchain.vectorstores import Pinecone
+from langchain.document_loaders import PyPDFLoader
+import tempfile
+import streamlit as st
+import openai
+
+@st.cache(allow_output_mutation=True)
+def get_embeddings_model():
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    return model, embeddings
+
+model, embeddings = get_embeddings_model()
+
+def ingest(
+        uploaded_document, 
+        pinecone_api_key, 
+        pinecone_env, 
+        pinecone_index_namespace, 
+        chunk_size=500,
+        chunk_overlap=20
+        ):
+    with tempfile.NamedTemporaryFile(delete=False) as tf:
+        tf.write(uploaded_document.getbuffer())
+        file_path = tf.name
+    loader = PyPDFLoader(file_path)
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    docs = text_splitter.split_documents(documents)
+    # embeddings = get_embeddings_model()
+    pinecone.init(
+        api_key=pinecone_api_key,
+        environment=pinecone_env
+    )
+    index_name = pinecone_index_namespace
+    try:
+        index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
+        st.success('Document uploaded to Pinecone database successfully')
+    except Exception as error_message:
+        st.error(error_message)
+
+# # To create embeddings on hard disk
+# # !pip install chromadb
+# # from langchain.vectorstores import Chroma
+# # persist_directory = './data/embeddings'
+# # vStore = Chroma.from_documents(docs, embeddings, persist_directory=persist_directory)
+
+
+def query_refiner(conversation, query):
+    response = openai.Completion.create(
+    model="text-davinci-003",
+    prompt=f"Given the following user query and conversation log, formulate a question that would be the most relevant to provide the user with an answer from a knowledge base.\n\nCONVERSATION LOG: \n{conversation}\n\nQuery: {query}\n\nRefined Query:",
+    temperature=0.7,
+    max_tokens=256,
+    top_p=1,
+    frequency_penalty=0,
+    presence_penalty=0
+    )
+    return response['choices'][0]['text']
+
+
+def find_match(input, pinecone_api_key, pinecone_env, pinecone_index_namespace):
+    pinecone.init(
+        api_key=pinecone_api_key,
+        environment=pinecone_env
+    )
+    index = pinecone.Index(pinecone_index_namespace)
+    input_em = model.encode(input).tolist()
+    result = index.query(input_em, top_k=2, includeMetadata=True)
+    return result['matches'][0]['metadata']['text']+"\n"+result['matches'][1]['metadata']['text']
+
+
+def get_conversation_string():
+    conversation_string = ""
+    for i in range(len(st.session_state['responses'])-1):        
+        conversation_string += "Human: "+st.session_state['requests'][i] + "\n"
+        conversation_string += "Bot: "+ st.session_state['responses'][i+1] + "\n"
+    return conversation_string

👋_Introduction.py

@@ -0,0 +1,57 @@
+import streamlit as st
+
+st.set_page_config(
+    page_title="Introduction",
+    page_icon="👋",
+)
+
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+
+# """
+# Initialising session states  
+# """
+if 'openai_api_key' not in st.session_state:
+    st.session_state['openai_api_key'] = None
+
+if 'pinecone_api_key' not in st.session_state:
+    st.session_state['pinecone_api_key'] = None
+
+if 'pinecone_env' not in st.session_state:
+    st.session_state['pinecone_env'] = None
+
+if 'pinecone_index_namespace' not in st.session_state:
+    st.session_state['pinecone_index_namespace'] = None
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'requests' not in st.session_state:
+    st.session_state['requests'] = []
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'buffer_memory' not in st.session_state:
+    st.session_state.buffer_memory=ConversationBufferWindowMemory(k=3,return_messages=True)
+
+
+# """
+# Page Content  
+# """
+st.write("# Welcome to ChatPDF! 👋")
+
+st.markdown(
+"""
+ChatPDF is a user-friendly software that allows you to ask questions 
+and get answers from your personal and organizational documents. 
+
+This software uses OpenAI ChatGPT to query documents and Pinecone Vector Database to store documents.
+
+How to use:
+1. Make an accout on [OpenAI](https://platform.openai.com/) and [Pinecone](https://www.pinecone.io/).
+2. Enter your credentials. This include OpenAI API Key, Pinecone API Key, Pinecone Environment and Pinecone Index Name.
+3. Upload Documents. These documents will be uploaded to Pinecone Database.
+4. Chat with your documents. Chatbot is built on top of ChatGPT Engine.
+
+"""
+)