first deployment

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gte_large/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,156 @@
+from dotenv import load_dotenv, find_dotenv
+from langchain.chains import LLMChain
+import streamlit as st
+from decouple import config
+from langchain.llms import OpenAI
+from langchain.document_loaders import PyPDFLoader
+from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.retrievers.document_compressors import LLMChainExtractor
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.chains import RetrievalQA
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.evaluation.qa import QAGenerateChain
+from langchain.chains import RetrievalQA
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import CSVLoader
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.vectorstores import DocArrayInMemorySearch
+from langchain.prompts import ChatPromptTemplate
+from langchain.document_loaders.generic import GenericLoader
+from langchain.document_loaders.parsers import OpenAIWhisperParser
+from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
+from langchain.prompts import PromptTemplate
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+import time
+from htmlTemplates import css, bot_template, user_template
+from pathlib import Path
+import pathlib
+import platform
+plt = platform.system()
+if plt == 'Linux':
+    pathlib.WindowsPath = pathlib.PosixPath
+
+_ = load_dotenv(find_dotenv())  # read local .env file
+
+
+def timeit(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()  # Start time
+        result = func(*args, **kwargs)  # Function execution
+        end_time = time.time()  # End time
+        print(
+            f"Function {func.__name__} took {end_time - start_time} seconds to execute.")
+        return result
+    return wrapper
+
+
+@timeit
+def get_llm():
+    return OpenAI(temperature=0.0)
+
+
+@timeit
+def get_memory():
+    return ConversationBufferMemory(
+        memory_key="chat_history",
+        return_messages=True
+    )
+
+
+@timeit
+def generate_response(question, vectordb, llm, memory, chat_history):
+    template = """Use the provided context to answer the user's question.
+    you are honest petroleum engineer specialist in hydraulic fracture stimulation and reservoir engineering.
+    when you asked about code numer like SPE-19***-MS or any thing like that it's a paper search for it and give an introduction.
+    If you don't know the answer, respond with "Sorry Sir, I do not know".
+    Context: {context}
+    Question: {question}
+    Answer:
+    """
+
+    prompt = PromptTemplate(
+        template=template,
+        input_variables=[ 'question','context'])
+
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=vectordb.as_retriever(search_type="mmr", k=5, fetch_k=10),
+        memory=memory,
+        combine_docs_chain_kwargs={"prompt": prompt}
+    )
+
+    handle_userinput(
+        (qa_chain({"question": question, "chat_history": chat_history})))
+
+
+@timeit
+def create_embeding_function():
+    # embedding_func_all_mpnet_base_v2 = SentenceTransformerEmbeddings(
+    #     model_name="all-mpnet-base-v2")
+    # # embedding_func_all_MiniLM_L6_v2 = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    # embedding_func_jina_embeddings_v2_base_en = SentenceTransformerEmbeddings(
+    #     model_name="jinaai/jina-embeddings-v2-base-en"
+    # )
+    # embedding_func_jina_embeddings_v2_small_en = SentenceTransformerEmbeddings(
+    #     model_name="jinaai/jina-embeddings-v2-small-en"
+    # )
+    embedding_func_jgte_large = SentenceTransformerEmbeddings(
+    model_name="thenlper/gte-large"
+    )
+    return embedding_func_jgte_large
+
+
+@timeit
+def get_vector_db(embedding_function):
+    vector_db = Chroma(persist_directory=str(Path('gte_large')),
+                       embedding_function=embedding_function)
+    return vector_db
+
+
+def handle_userinput(user_question):
+    response = user_question
+    if 'chat_history' not in st.session_state:
+        st.session_state.chat_history = []
+
+    st.session_state.chat_history = response['chat_history']
+
+    for i, message in enumerate(st.session_state.chat_history):
+        if i % 2 == 0:
+            st.write(user_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+        else:
+            st.write(bot_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+
+
+if __name__ == "__main__":
+
+    st.set_page_config(
+        page_title="Hydraulic Fracture Stimulation Chat", page_icon=":books:")
+    st.write(css, unsafe_allow_html=True)
+    st.title("Hydraulic Fracture Stimulation Chat")
+    st.write(
+        "This is a chatbot that can answer questions related to petroleum engineering specially in hydraulic fracture stimulation.")
+
+    # get embeding function
+    embeding_function = create_embeding_function()
+    # get vector db
+    vector_db = get_vector_db(embeding_function)
+     # get llm
+    llm = get_llm()
+
+    # get memory
+    if 'memory' not in st.session_state:
+        st.session_state['memory'] = get_memory()
+    memory = st.session_state['memory']
+
+    # chat history
+    chat_history = []
+
+    prompt_question = st.chat_input("Please ask a question:")
+    if prompt_question:
+        generate_response(question=prompt_question, vectordb=vector_db,
+                          llm=llm, memory=memory, chat_history=chat_history)

gte_large/67650f97-d36f-459f-af8f-ff25b716cd81/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3e93c7d71a7def052f3331a5611bcab93e76407c291c29158bd308fd8ac1a7
+size 8472000

gte_large/67650f97-d36f-459f-af8f-ff25b716cd81/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:623782542530941eccea3efc3e80bfb767b479b3cf65a6d85dec916277b0b962
+size 100

gte_large/67650f97-d36f-459f-af8f-ff25b716cd81/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4da42581592e740de75828a7ee71890e66dbe3d16307ecb9e5d1a841bfeb52b9
+size 113989

gte_large/67650f97-d36f-459f-af8f-ff25b716cd81/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2388906e8c52245056cf576860661b5752a625a42c8c422e9918af4a3029c86c
+size 8000

gte_large/67650f97-d36f-459f-af8f-ff25b716cd81/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1efd5680b5399e4190cf6fe79b8f4441981427a9c6edb0a50f206617c377875c
+size 16976

gte_large/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c0d5ea2f5f5dfd09b5b0f30fb438331aba46bd53d56b74cc79e88481d4ac25
+size 21815296

htmlTemplates.py

@@ -0,0 +1,44 @@
+css = '''
+<style>
+.chat-message {
+    padding: 1.5rem; border-radius: 0.5rem; margin-bottom: 1rem; display: flex
+}
+.chat-message.user {
+    background-color: #2b313e
+}
+.chat-message.bot {
+    background-color: #475063
+}
+.chat-message .avatar {
+  width: 20%;
+}
+.chat-message .avatar img {
+  max-width: 78px;
+  max-height: 78px;
+  border-radius: 50%;
+  object-fit: cover;
+}
+.chat-message .message {
+  width: 80%;
+  padding: 0 1.5rem;
+  color: #fff;
+}
+'''
+
+bot_template = '''
+<div class="chat-message bot">
+    <div class="avatar">
+        <img src="https://i.ibb.co/cN0nmSj/Screenshot-2023-05-28-at-02-37-21.png" style="max-height: 78px; max-width: 78px; border-radius: 50%; object-fit: cover;">
+    </div>
+    <div class="message">{{MSG}}</div>
+</div>
+'''
+
+user_template = '''
+<div class="chat-message user">
+    <div class="avatar">
+        <img src="https://th.bing.com/th/id/OIP.xXHQ5dk4qJH74WMGNezDjwHaHa?rs=1&pid=ImgDetMain">
+    </div>    
+    <div class="message">{{MSG}}</div>
+</div>
+'''

requirements.txt

@@ -0,0 +1,170 @@
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+bcrypt==4.0.1
+blinker==1.7.0
+Brotli==1.1.0
+cachetools==5.3.2
+certifi==2023.7.22
+cffi==1.16.0
+charset-normalizer==3.3.2
+Chroma==0.2.0
+chroma-hnswlib==0.7.3
+chromadb==0.4.17
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.0
+cryptography==41.0.5
+ctransformers==0.2.27
+dataclasses-json==0.6.2
+debugpy==1.8.0
+decorator==5.1.1
+Deprecated==1.2.14
+distro==1.8.0
+exceptiongroup==1.1.3
+executing==2.0.1
+fastapi==0.104.1
+filelock==3.13.1
+flatbuffers==23.5.26
+frozenlist==1.4.0
+fsspec==2023.10.0
+gitdb==4.0.11
+GitPython==3.1.40
+google-auth==2.23.4
+googleapis-common-protos==1.61.0
+greenlet==3.0.1
+grpcio==1.59.2
+h11==0.14.0
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.25.1
+huggingface-hub==0.19.3
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.1
+ipykernel==6.26.0
+ipython==8.17.2
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.2
+jsonschema-specifications==2023.11.1
+jupyter_client==8.6.0
+jupyter_core==5.5.0
+kubernetes==28.1.0
+langchain==0.0.336
+langsmith==0.0.64
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mutagen==1.47.0
+mypy-extensions==1.0.0
+nest-asyncio==1.5.8
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.16.2
+openai==1.3.0
+opentelemetry-api==1.21.0
+opentelemetry-exporter-otlp-proto-common==1.21.0
+opentelemetry-exporter-otlp-proto-grpc==1.21.0
+opentelemetry-proto==1.21.0
+opentelemetry-sdk==1.21.0
+opentelemetry-semantic-conventions==0.42b0
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.3
+parso==0.8.3
+Pillow==10.1.0
+platformdirs==4.0.0
+posthog==3.0.2
+prompt-toolkit==3.0.41
+protobuf==4.25.1
+psutil==5.9.6
+pulsar-client==3.3.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==14.0.1
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycparser==2.21
+pycryptodomex==3.19.0
+pydantic==2.5.1
+pydantic_core==2.14.3
+pydeck==0.8.1b0
+pydub==0.25.1
+Pygments==2.16.1
+pypdf==3.17.1
+PyPika==0.48.9
+pyreadline3==3.4.1
+python-dateutil==2.8.2
+python-decouple==3.8
+python-dotenv==1.0.0
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.1
+referencing==0.31.0
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.7.0
+rpds-py==0.12.0
+rsa==4.9
+safetensors==0.4.0
+scikit-learn==1.3.2
+scipy==1.11.3
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+SQLAlchemy==2.0.23
+stack-data==0.6.3
+starlette==0.27.0
+streamlit==1.28.2
+sympy==1.12
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tiktoken==0.5.1
+tokenizers==0.15.0
+toml==0.10.2
+toolz==0.12.0
+torch==2.1.1
+torchvision==0.16.1
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.13.0
+transformers==4.35.2
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tzdata==2023.3
+tzlocal==5.2
+urllib3==1.26.18
+uvicorn==0.24.0.post1
+validators==0.22.0
+watchdog==3.0.0
+watchfiles==0.21.0
+wcwidth==0.2.10
+websocket-client==1.6.4
+websockets==12.0
+wrapt==1.16.0
+yarl==1.9.2
+yt-dlp==2023.11.16
+zipp==3.17.0