added Jine

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+___pycache__
+Data/
+Front-end/
+.env

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.10
+
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,2 @@
----
-title: Jin E
-emoji: 👀
-colorFrom: red
-colorTo: purple
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# JIN-e
+This is a bot based on Chatgpt using langchain  to answers Questions related to Policies.

app.py

@@ -0,0 +1,40 @@
+from fastapi import FastAPI, HTTPException, Query
+
+# Import the Jine class and other necessary modules
+from jine import Jine  # Replace 'your_module_name' with the actual module name
+from pydantic import BaseModel
+
+# Load your environment variables
+from dotenv import load_dotenv
+import os
+
+
+load_dotenv()
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+DATA_DIRECTORY = os.getenv("DATA_DIRECTORY")
+VECTOR_STORE_DIRECTORY = os.getenv("VECTOR_STORE_DIRCTORY")
+VECTOR_STORE_CHECK = os.getenv("VECTOR_STORE_CHECK")
+DEBUG = os.getenv("DEBUG")
+
+# Initialize Jine
+jine = Jine(OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG)
+jine.load_model()
+
+# Create a FastAPI app
+app = FastAPI()
+
+# Define a request model
+class ChatRequest(BaseModel):
+    user_question: str
+
+# Define a response model
+class ChatResponse(BaseModel):
+    user_question: str
+    chatbot_response: str
+
+# Define the chatbot endpoint
+@app.post("/chatbot/")
+def chat_with_bot(request: ChatRequest):
+    user_question = request.user_question
+    chatbot_response = jine.chat(user_question)
+    return ChatResponse(user_question=user_question, chatbot_response=chatbot_response)

environment.yml

@@ -0,0 +1,210 @@
+name: jine
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - asttokens=2.4.0=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - bzip2=1.0.8=he774522_0
+  - ca-certificates=2023.7.22=h56e8100_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - debugpy=1.6.7=py310hd77b12b_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - exceptiongroup=1.1.3=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - importlib-metadata=6.8.0=pyha770c72_0
+  - importlib_metadata=6.8.0=hd8ed1ab_0
+  - ipykernel=6.25.2=pyh60829e3_0
+  - ipython=8.16.1=pyh5737063_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jupyter_client=8.4.0=pyhd8ed1ab_0
+  - jupyter_core=5.4.0=py310h5588dad_0
+  - libffi=3.4.4=hd77b12b_0
+  - libsodium=1.0.18=h8d14728_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - openssl=1.1.1l=h8ffe710_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pip=23.3=py310haa95532_0
+  - platformdirs=3.11.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.39=pyha770c72_0
+  - prompt_toolkit=3.0.39=hd8ed1ab_0
+  - psutil=5.9.0=py310h2bbff1b_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - python=3.10.0=h96c0403_3
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.10=2_cp310
+  - pyzmq=23.2.1=py310h73ada01_0
+  - setuptools=68.0.0=py310haa95532_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h2bbff1b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h2bbff1b_0
+  - tornado=6.2=py310he2412df_0
+  - traitlets=5.11.2=pyhd8ed1ab_0
+  - typing-extensions=4.8.0=hd8ed1ab_0
+  - typing_extensions=4.8.0=pyha770c72_0
+  - vc=14.2=h21ff451_1
+  - vs2015_runtime=14.27.29016=h5e58377_2
+  - wcwidth=0.2.8=pyhd8ed1ab_0
+  - wheel=0.41.2=py310haa95532_0
+  - xz=5.4.2=h8cc25b3_0
+  - zeromq=4.3.4=h0e60522_1
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=h8cc25b3_0
+  - pip:
+    - aiofiles==23.2.1
+    - aiohttp==3.8.6
+    - aiosignal==1.3.1
+    - annotated-types==0.6.0
+    - antlr4-python3-runtime==4.9.3
+    - anyio==3.7.1
+    - async-timeout==4.0.3
+    - asyncer==0.0.2
+    - attrs==23.1.0
+    - backoff==2.2.1
+    - beautifulsoup4==4.12.2
+    - bidict==0.22.1
+    - certifi==2023.7.22
+    - cffi==1.16.0
+    - chainlit==0.7.301
+    - chardet==5.2.0
+    - charset-normalizer==3.3.0
+    - click==8.1.7
+    - contourpy==1.1.1
+    - cryptography==41.0.4
+    - cycler==0.12.1
+    - dataclasses-json==0.5.14
+    - deprecated==1.2.14
+    - effdet==0.4.1
+    - emoji==2.8.0
+    - fastapi==0.99.1
+    - fastapi-socketio==0.0.10
+    - filelock==3.12.4
+    - filetype==1.2.0
+    - flatbuffers==23.5.26
+    - fonttools==4.43.1
+    - frozenlist==1.4.0
+    - fsspec==2023.10.0
+    - googleapis-common-protos==1.61.0
+    - greenlet==3.0.0
+    - grpcio==1.59.0
+    - h11==0.14.0
+    - httpcore==0.18.0
+    - httptools==0.6.1
+    - httpx==0.25.0
+    - huggingface-hub==0.17.3
+    - humanfriendly==10.0
+    - idna==3.4
+    - importlib-resources==6.1.0
+    - iopath==0.1.10
+    - jinja2==3.1.2
+    - joblib==1.3.2
+    - jsonpatch==1.33
+    - jsonpointer==2.4
+    - kiwisolver==1.4.5
+    - langchain==0.0.320
+    - langdetect==1.0.9
+    - langsmith==0.0.49
+    - layoutparser==0.3.4
+    - lazify==0.4.0
+    - lxml==4.9.3
+    - markupsafe==2.1.3
+    - marshmallow==3.20.1
+    - matplotlib==3.8.0
+    - monotonic==1.6
+    - mpmath==1.3.0
+    - multidict==6.0.4
+    - mypy-extensions==1.0.0
+    - networkx==3.2
+    - nltk==3.8.1
+    - nodeenv==1.8.0
+    - numpy==1.26.1
+    - omegaconf==2.3.0
+    - onnx==1.14.1
+    - openai==0.28.1
+    - opencv-python==4.8.1.78
+    - opentelemetry-api==1.20.0
+    - opentelemetry-exporter-otlp==1.20.0
+    - opentelemetry-exporter-otlp-proto-common==1.20.0
+    - opentelemetry-exporter-otlp-proto-grpc==1.20.0
+    - opentelemetry-exporter-otlp-proto-http==1.20.0
+    - opentelemetry-instrumentation==0.41b0
+    - opentelemetry-proto==1.20.0
+    - opentelemetry-sdk==1.20.0
+    - opentelemetry-semantic-conventions==0.41b0
+    - overrides==7.4.0
+    - pdf2image==1.16.3
+    - pdfminer-six==20221105
+    - pdfplumber==0.10.2
+    - pillow==10.1.0
+    - portalocker==2.8.2
+    - prisma==0.10.0
+    - protobuf==4.24.4
+    - pulsar-client==3.3.0
+    - pycocotools==2.0.7
+    - pycparser==2.21
+    - pydantic==1.10.13
+    - pydantic-core==2.10.1
+    - pyjwt==2.8.0
+    - pymupdf==1.23.5
+    - pymupdfb==1.23.5
+    - pyparsing==3.1.1
+    - pypdfium2==4.22.0
+    - pypika==0.48.9
+    - pyreadline3==3.4.1
+    - pytesseract==0.3.10
+    - python-docx==1.0.1
+    - python-dotenv==1.0.0
+    - python-engineio==4.8.0
+    - python-graphql-client==0.4.3
+    - python-iso639==2023.6.15
+    - python-magic==0.4.27
+    - python-multipart==0.0.6
+    - python-socketio==5.10.0
+    - pytz==2023.3.post1
+    - pywin32==306
+    - pyyaml==6.0.1
+    - rank-bm25==0.2.2
+    - rapidfuzz==3.4.0
+    - regex==2023.10.3
+    - requests==2.31.0
+    - safetensors==0.4.0
+    - scipy==1.11.3
+    - simple-websocket==1.0.0
+    - sniffio==1.3.0
+    - soupsieve==2.5
+    - sqlalchemy==2.0.22
+    - sympy==1.12
+    - syncer==2.0.3
+    - tabulate==0.9.0
+    - tenacity==8.2.3
+    - tiktoken==0.5.1
+    - timm==0.9.8
+    - tokenizers==0.14.1
+    - tomli==2.0.1
+    - tomlkit==0.12.1
+    - torch==2.1.0
+    - torchvision==0.16.0
+    - tqdm==4.66.1
+    - transformers==4.34.1
+    - typing-inspect==0.9.0
+    - tzdata==2023.3
+    - unstructured==0.10.25
+    - unstructured-inference==0.7.9
+    - unstructured-pytesseract==0.3.12
+    - uptrace==1.20.2
+    - urllib3==2.0.7
+    - uvicorn==0.23.2
+    - watchfiles==0.20.0
+    - websockets==11.0.3
+    - wrapt==1.15.0
+    - wsproto==1.2.0
+    - yarl==1.9.2
+prefix: D:\anaconda3\envs\jine

jine.py

@@ -0,0 +1,130 @@
+import logging
+import os
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import DirectoryLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.retrievers.multi_query import MultiQueryRetriever
+from langchain.prompts import PromptTemplate
+from langchain.chains import RetrievalQA
+from dotenv import load_dotenv
+
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
+from langchain.llms import OpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains import LLMChain, HypotheticalDocumentEmbedder
+
+## Setting up Log configuration
+logging.basicConfig(
+    filename='Logs/chatbot.log', # Name of the log file
+    level=logging.INFO,      # Logging level (you can use logging.DEBUG for more detailed logs)
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+class Jine:
+
+    def __init__(self, OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG,USE_HYDE=False):
+        self.OPENAI_API_KEY = OPENAI_API_KEY
+        self.DATA_DIRECTORY = DATA_DIRECTORY
+        self.VECTOR_STORE_DIRECTORY = VECTOR_STORE_DIRECTORY
+        self.VECTOR_STORE_CHECK = VECTOR_STORE_CHECK
+        # self.DEBUG = DEBUG
+        self.vectorstore = None
+        self.bot = None
+
+    def create_vectorstore(self):
+
+        if  self.VECTOR_STORE_CHECK:
+            print("Loading Vectorstore")
+            self.load_vectorstore()
+        else:
+            print("Creating Vectorstore")
+            docs = DirectoryLoader(self.DATA_DIRECTORY).load()
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=10)
+            all_splits = text_splitter.split_documents(docs)
+            self.vectorstore = Chroma.from_documents(documents=docs, embedding=OpenAIEmbeddings(),
+                                                persist_directory=self.VECTOR_STORE_DIRECTORY)
+
+
+    def load_vectorstore(self):
+        self.vectorstore = Chroma(persist_directory=self.VECTOR_STORE_DIRECTORY, embedding_function=OpenAIEmbeddings())
+
+
+    def log(self, user_question, chatbot_reply):
+        # Log the user's question
+        logging.info(f"User: {user_question}")
+        # Log the chatbot's reply
+        logging.info(f"JIN-e: {chatbot_reply}")
+
+    def load_model(self):
+          self.create_vectorstore()
+          self.create_ensemble_retriever()
+
+    def chat(self, user_question):
+        result = self.bot({"query": user_question})
+        response = result["result"]
+        self.log(user_question, response)
+        return response
+
+    ### Adding Ensemble retriver 
+    def create_ensemble_retriever(self):
+        template = """
+        You are an Expert Policy Advisor.These Below are the Documents that are extracted from the different Policies.Your Job 
+                is to Provide the Answer to below question based on the text below. 
+                Here are few instructions for you to follow when answering a question.
+                - When you didnt find the relevant answers from below text Just Say "I dont know this,Please contact your HRBP for more details."
+                - These are policy Documents, When answering a question Do Not return in response that "This information is At Annex A/B".Provide a Complete response to request.
+                - Try to answer the questions in bullet format if possible.
+                - Use three sentences maximum to Answer the question in very concise manner
+                
+                {context}
+                Question: {question}
+                Helpful Answer:
+                """
+
+        QA_CHAIN_PROMPT = PromptTemplate.from_template(template)
+        print("====================="*10)
+        print("Loading Documents for Ensemble Retriver")
+        print("====================="*10)
+
+        docs = DirectoryLoader(self.DATA_DIRECTORY).load()
+        # text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=10)
+        # all_splits = text_splitter.split_documents(docs)
+
+        bm25_retriever = BM25Retriever.from_documents(docs)
+        # GEttting only two relevant documents 
+        bm25_retriever.k = 2
+        ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, 
+                                                self.vectorstore.as_retriever(search_kwargs={"k": 2})],
+                                                weights=[0.5, 0.5])
+        
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+        self.bot = RetrievalQA.from_chain_type(
+        llm,
+        retriever=ensemble_retriever,
+        chain_type_kwargs={"prompt": QA_CHAIN_PROMPT})
+
+
+
+if __name__ == "__main__":
+    # Set your configuration here
+    load_dotenv()
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    DATA_DIRECTORY = os.getenv("DATA_DIRECTORY")
+    VECTOR_STORE_DIRECTORY = os.getenv("VECTOR_STORE_DIRCTORY")
+    VECTOR_STORE_CHECK = os.getenv("VECTOR_STORE_CHECK")
+    
+    DEBUG = os.getenv("DEBUG")
+    USE_HYDE = os.getenv("USE_HYDE")
+    # Initialize Jine and start chatting
+    jine = Jine(OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG)
+    # print(jine.VECTOR_STORE_CHECK)
+    jine.load_model()
+    while True:
+        user_question = input("You: ")
+        if user_question.lower() in ["exit", "quit"]:
+            break
+        response = jine.chat(user_question)
+        print("JIN-e:", response)

jine_v1.py

@@ -0,0 +1,182 @@
+import logging
+import os
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import DirectoryLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.retrievers.multi_query import MultiQueryRetriever
+from langchain.prompts import PromptTemplate
+from langchain.chains import RetrievalQA
+from dotenv import load_dotenv
+
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
+from langchain.llms import OpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains import LLMChain, HypotheticalDocumentEmbedder
+
+## Setting up Log configuration
+logging.basicConfig(
+    filename='Logs/chatbot.log', # Name of the log file
+    level=logging.INFO,      # Logging level (you can use logging.DEBUG for more detailed logs)
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+
+class Jine:
+
+    def __init__(self, OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG,USE_HYDE=False):
+        self.OPENAI_API_KEY = OPENAI_API_KEY
+        self.DATA_DIRECTORY = DATA_DIRECTORY
+        self.VECTOR_STORE_DIRECTORY = VECTOR_STORE_DIRECTORY
+        self.VECTOR_STORE_CHECK = VECTOR_STORE_CHECK
+        self.DEBUG = DEBUG
+        self.vectorstore = None
+        self.bot = None
+        self.USE_HYDE = USE_HYDE
+        # creating this variable for BM25 Retriver.
+        # self.docs = None
+
+    def create_vectorstore(self):
+
+        if  self.VECTOR_STORE_CHECK:
+            print("Loading Vectorstore")
+            self.load_vectorstore()
+            print('im running')
+        else:
+            print("Creating Vectorstore")
+            docs = DirectoryLoader(self.DATA_DIRECTORY).load()
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=10)
+            all_splits = text_splitter.split_documents(docs)
+            if self.USE_HYDE:
+                base_embeddings = OpenAIEmbeddings()
+                llm = OpenAI()
+                embeddings_hyde = HypotheticalDocumentEmbedder.from_llm(llm, base_embeddings, "web_search")
+                self.vectorstore = Chroma.from_documents(documents=all_splits, embedding=embeddings_hyde,
+                                                    persist_directory=self.VECTOR_STORE_DIRECTORY)
+                
+            else:
+                self.vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings(),
+                                                    persist_directory=self.VECTOR_STORE_DIRECTORY)
+
+
+    def multi_query_retriever(self):
+        retriever_from_llm = MultiQueryRetriever.from_llm(retriever=self.vectorstore.as_retriever(),
+                                                        llm=ChatOpenAI(temperature=0))
+        template = """Use the following pieces of context to answer the question at the end. 
+        If you don't know the answer, just say that "i am unable to answer your query, for more information contact your HRBP", don't try to make up an answer. 
+        Use three sentences maximum and keep the answer as concise as possible.
+        {context}
+        Question: {question}
+        Helpful Answer:"""
+        QA_CHAIN_PROMPT = PromptTemplate.from_template(template)
+
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+        self.bot = RetrievalQA.from_chain_type(
+            llm,
+            retriever=retriever_from_llm,
+            chain_type_kwargs={"prompt": QA_CHAIN_PROMPT}
+        )
+
+    
+    def single_query_retriever(self):
+        template = """Use the following pieces of context to answer the question at the end. 
+        If you don't know the answer, just say that "i am unable to answer your query, for more information contact your HRBP", don't try to make up an answer. 
+        Use three sentences maximum and keep the answer as concise as possible.
+        {context}
+        Question: {question}
+        Helpful Answer:"""
+        QA_CHAIN_PROMPT = PromptTemplate.from_template(template)
+
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+        self.bot = RetrievalQA.from_chain_type(
+        llm,
+        retriever=self.vectorstore.as_retriever(),
+        chain_type_kwargs={"prompt": QA_CHAIN_PROMPT})
+
+    def load_vectorstore(self):
+        if self.USE_HYDE:
+            print("Using HYDE embeddings vectorstore")
+            base_embeddings = OpenAIEmbeddings()
+            llm = OpenAI()
+            embeddings_hyde = HypotheticalDocumentEmbedder.from_llm(llm, base_embeddings, "web_search")
+            self.vectorstore = Chroma(persist_directory=self.VECTOR_STORE_DIRECTORY, embedding_function=embeddings_hyde)
+        else:
+            print("Using Simple embeddings vectorstore")
+            self.vectorstore = Chroma(persist_directory=self.VECTOR_STORE_DIRECTORY, embedding_function=OpenAIEmbeddings())
+
+    def log(self, user_question, chatbot_reply):
+        # Log the user's question
+        logging.info(f"User: {user_question}")
+        # Log the chatbot's reply
+        logging.info(f"JIN-e: {chatbot_reply}")
+
+    def load_model(self):
+          self.create_vectorstore()
+        #   self.multi_query_retriever()
+        #   self.single_query_retriever()
+          self.create_ensemble_retriever()
+
+    def chat(self, user_question):
+        result = self.bot({"query": user_question})
+        response = result["result"]
+        self.log(user_question, response)
+        return response
+
+    ### Adding Ensemble retriver 
+    def create_ensemble_retriever(self):
+        template = """Use the following pieces of context to answer the question at the end. 
+        If you don't know the answer, just say that "i am unable to answer your query, for more information contact your HRBP", don't try to make up an answer. 
+        Use three sentences maximum and keep the answer as concise as possible.
+        {context}
+        Question: {question}
+        Helpful Answer:"""
+
+        QA_CHAIN_PROMPT = PromptTemplate.from_template(template)
+        print("====================="*10)
+        print("Loading Documents for Ensemble Retriver")
+        print("====================="*10)
+
+        docs = DirectoryLoader(self.DATA_DIRECTORY).load()
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=10)
+        all_splits = text_splitter.split_documents(docs)
+
+        bm25_retriever = BM25Retriever.from_documents(all_splits)
+        # GEttting only two relevant documents 
+        bm25_retriever.k = 2
+
+        ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, 
+                                                self.vectorstore.as_retriever(search_kwargs={"k": 2})],
+                                                weights=[0.5, 0.5])
+        
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo-16k", temperature=0)
+
+        self.bot = RetrievalQA.from_chain_type(
+        llm,
+        retriever=ensemble_retriever,
+        chain_type_kwargs={"prompt": QA_CHAIN_PROMPT})
+
+
+
+if __name__ == "__main__":
+    # Set your configuration here
+    load_dotenv()
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    DATA_DIRECTORY = os.getenv("DATA_DIRECTORY")
+    VECTOR_STORE_DIRECTORY = os.getenv("VECTOR_STORE_DIRCTORY")
+    VECTOR_STORE_CHECK = os.getenv("VECTOR_STORE_CHECK")
+    
+    DEBUG = os.getenv("DEBUG")
+    USE_HYDE = os.getenv("USE_HYDE")
+    # Initialize Jine and start chatting
+    jine = Jine(OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG)
+    # print(jine.VECTOR_STORE_CHECK)
+    jine.load_model()
+    while True:
+        user_question = input("You: ")
+        if user_question.lower() in ["exit", "quit"]:
+            break
+        response = jine.chat(user_question)
+        print("JIN-e:", response)

requirements

@@ -0,0 +1,6 @@
+langchain
+chromadb
+"unstructured[all-docs]"
+openai
+fastapi
+uvicorn

requirements.txt

@@ -0,0 +1,218 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: win-64
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+argilla==1.5.0
+asttokens==2.0.5
+async-timeout==4.0.2
+attrs==22.2.0
+backcall==0.2.0
+backoff==2.2.1
+beautifulsoup4==4.12.0
+bertopic==0.13.0
+blinker==1.6.2
+blis==0.7.9
+bs4==0.0.1
+cachetools==5.3.0
+catalogue==2.0.8
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+chromadb==0.3.11
+click==8.1.3
+clickhouse-connect==0.5.16
+colorama==0.4.6
+commonmark==0.9.1
+confection==0.0.3
+cryptography==40.0.0
+cymem==2.0.7
+cython==0.29.32
+dataclasses-json==0.5.7
+debugpy==1.5.1
+decorator==5.1.1
+deprecated==1.2.13
+docx2txt==0.8
+duckdb==0.7.1
+entrypoints==0.4
+et-xmlfile==1.1.0
+executing==0.8.3
+faker==17.6.0
+fastapi==0.95.0
+filelock==3.9.0
+flask==2.3.3
+flask-sqlalchemy==3.0.5
+flatbuffers==23.5.26
+frozenlist==1.3.3
+fst-pso==1.8.1
+funcy==1.17
+future==0.18.2
+fuzzytm==2.0.5
+gensim==4.3.0
+gitdb==4.0.10
+gitpython==3.1.31
+google-search-results==2.4.2
+greenlet==2.0.1
+h11==0.14.0
+hdbscan==0.8.29
+hnswlib==0.7.0
+httpcore==0.16.3
+httptools==0.5.0
+httpx==0.23.3
+huggingface-hub==0.11.1
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.1.0
+importlib-resources==6.0.1
+ipykernel==6.15.2
+ipython==8.7.0
+itsdangerous==2.1.2
+jedi==0.18.1
+jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter_client==7.4.8
+jupyter_core==5.1.1
+langchain==0.0.284
+langcodes==3.3.0
+langsmith==0.0.33
+llama-index==0.5.5
+llvmlite==0.39.1
+lxml==4.9.2
+lz4==4.3.2
+markdown==3.4.3
+markdown-it-py==2.2.0
+markupsafe==2.1.1
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+miniful==0.0.6
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+murmurhash==1.0.9
+mypy-extensions==1.0.0
+nest-asyncio==1.5.6
+nltk==3.8.1
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.23.5
+openai==0.27.2
+opencv-python==4.7.0.72
+openpyxl==3.1.1
+overrides==7.4.0
+packaging==22.0
+pandas==1.5.2
+parso==0.8.3
+pathy==0.10.1
+pdfminer-six==20221105
+pickleshare==0.7.5
+pillow==9.4.0
+pip==22.3.1
+platformdirs==2.5.2
+plotly==5.11.0
+preshed==3.0.8
+prompt-toolkit==3.0.36
+protobuf==3.20.3
+psutil==5.9.0
+pulsar-client==3.3.0
+pure_eval==0.2.2
+pyarrow==11.0.0
+pycparser==2.21
+pydantic==1.10.4
+pydeck==0.8.0
+pyfume==0.2.25
+pygments==2.14.0
+pyldavis==3.3.1
+pympler==1.0.1
+pynndescent==0.5.8
+pyodbc==4.0.35
+pypandoc==1.11
+pypdf2==3.0.1
+pypika==0.48.9
+pypyodbc==1.3.6
+pyreadline3==3.4.1
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==1.0.0
+python-magic==0.4.27
+python-pptx==0.6.21
+pytz==2022.7
+pytz-deprecation-shim==0.1.0.post0
+pywin32==305
+pyyaml==6.0
+pyzmq==23.2.0
+regex==2022.10.31
+requests==2.28.1
+rfc3986==1.5.0
+rich==13.0.1
+scikit-learn==1.2.0
+scipy==1.10.0
+semver==2.13.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.97
+setuptools==65.5.0
+simpful==2.9.0
+six==1.16.0
+sklearn==0.0.post1
+smart-open==6.3.0
+smmap==5.0.0
+sniffio==1.3.0
+soupsieve==2.4
+spacy==3.4.4
+spacy-legacy==3.0.11
+spacy-loggers==1.0.4
+sqlalchemy==2.0.20
+sqlite==3.40.0
+srsly==2.4.5
+stack_data==0.2.0
+starlette==0.26.1
+streamlit==1.20.0
+streamlit-chat==0.0.2.2
+sympy==1.12
+tenacity==8.2.2
+thinc==8.1.6
+threadpoolctl==3.1.0
+tiktoken==0.3.2
+tk==8.6.12
+tokenizers==0.13.2
+toml==0.10.2
+toolz==0.12.0
+torch==1.13.1
+torchvision==0.14.1
+tornado==6.2
+tqdm==4.66.1
+traitlets==5.7.1
+transformers==4.25.1
+typer==0.7.0
+typing-extensions==4.7.1
+typing-inspect==0.8.0
+tzdata==2023.3
+tzlocal==4.3
+umap-learn==0.5.3
+unstructured==0.5.7
+urllib3==1.26.13
+uvicorn==0.21.1
+validators==0.20.0
+vc==14.2
+vs2015_runtime==14.27.29016
+wasabi==0.10.1
+watchdog==3.0.0
+watchfiles==0.18.1
+wcwidth==0.2.5
+websockets==10.4
+werkzeug==2.3.7
+wheel==0.37.1
+wincertstore==0.2
+wrapt==1.14.1
+xlsxwriter==3.0.9
+xz==5.2.8
+yarl==1.8.2
+zeromq==4.3.4
+zipp==3.15.0
+zlib==1.2.13
+zstandard==0.20.0

streamlit_interface.py

@@ -0,0 +1,73 @@
+import streamlit as st
+from streamlit_chat import message
+import os 
+
+##### Importing JIN-e
+from jine import Jine
+from dotenv import load_dotenv
+import os
+
+
+load_dotenv()
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+DATA_DIRECTORY = os.getenv("DATA_DIRECTORY")
+VECTOR_STORE_DIRECTORY = os.getenv("VECTOR_STORE_DIRCTORY")
+VECTOR_STORE_CHECK = os.getenv("VECTOR_STORE_CHECK")
+DEBUG = os.getenv("DEBUG")
+USE_HYDE = os.getenv("USE_HYDE")
+
+# Initialize Jine
+
+
+@st.cache_resource()
+def load_model():
+    jine = Jine(OPENAI_API_KEY, VECTOR_STORE_DIRECTORY, VECTOR_STORE_CHECK, DATA_DIRECTORY, DEBUG,USE_HYDE)
+    jine.load_model()
+    return jine
+
+jine =load_model()
+
+import streamlit as st
+from streamlit_chat import message
+
+# st.set_page_config(
+#     page_title="JIN-e",
+#     page_icon=":robot:"
+# )
+# #
+
+st.header("JIN-e")
+st.markdown("Powered by People Analytics")
+
+if 'generated' not in st.session_state:
+    st.session_state['generated'] = []
+
+if 'past' not in st.session_state:
+    st.session_state['past'] = []
+
+# def query(payload):
+# 	response = requests.post(API_URL, headers=headers, json=payload)
+# 	return response.json()
+
+def get_text():
+    input_text = st.text_input("You: ","Hello, how are you?", key="input")
+    return input_text 
+
+
+user_input = get_text()
+
+if user_input:
+
+    response = jine.chat(user_input)
+
+    st.session_state.past.append(user_input)
+    st.session_state.generated.append(response)
+
+if st.session_state['generated']:
+
+    for i in range(len(st.session_state['generated'])-1, -1, -1):
+        message(st.session_state['past'][i], is_user=True, key=str(i) + '_user')
+        message(st.session_state["generated"][i], key=str(i))
+        
+        
+