|
|
|
|
|
import json |
|
import os |
|
import shutil |
|
import langchain |
|
import pinecone |
|
import streamlit as st |
|
from langchain.chat_models import ChatOpenAI |
|
from langchain.chains import ConversationalRetrievalChain, LLMChain, load_qa_chain |
|
from langchain.document_loaders import PyMuPDFLoader, UnstructuredFileLoader, UnstructuredWordDocumentLoader |
|
from langchain.embeddings import HuggingFaceEmbeddings, OpenAIEmbeddings |
|
from langchain.memory import ConversationBufferMemory |
|
from langchain.prompts import PromptTemplate |
|
from langchain.text_splitter import RecursiveCharacterTextSplitter |
|
from langchain.vectorstores import Chroma, Pinecone |
|
|
|
|
|
OPENAI_API_KEY = '' |
|
PINECONE_API_KEY = '' |
|
PINECONE_API_ENV = '' |
|
gpt3p5 = 'gpt-3.5-turbo-1106' |
|
gpt4 = 'gpt-4-1106-preview' |
|
local_model_tuples = [ |
|
|
|
] |
|
local_model_names = [t[1] for t in local_model_tuples] |
|
langchain.verbose = False |
|
|
|
|
|
@st.cache_data() |
|
def init(): |
|
pinecone_index_name = '' |
|
chroma_collection_name = '' |
|
persist_directory = '' |
|
docsearch_ready = False |
|
directory_name = 'tmp_docs' |
|
return pinecone_index_name, chroma_collection_name, persist_directory, docsearch_ready, directory_name |
|
|
|
|
|
@st.cache_data() |
|
def save_file(files, directory_name): |
|
|
|
|
|
|
|
def load_files(directory_name): |
|
|
|
|
|
|
|
@st.cache_resource() |
|
def ingest(_all_texts, use_pinecone, _embeddings, pinecone_index_name, chroma_collection_name, persist_directory): |
|
|
|
|
|
|
|
def setup_retriever(docsearch, k): |
|
|
|
|
|
|
|
def setup_docsearch(use_pinecone, pinecone_index_name, embeddings, chroma_collection_name, persist_directory): |
|
|
|
|
|
|
|
def get_response(query, chat_history, CRqa): |
|
|
|
|
|
|
|
@st.cache_resource() |
|
def use_local_llm(r_llm, local_llm_path, temperature): |
|
|
|
|
|
|
|
def setup_prompt(r_llm, usage): |
|
|
|
|
|
|
|
def setup_em_llm(OPENAI_API_KEY, temperature, r_llm, local_llm_path, usage): |
|
|
|
|
|
|
|
def load_chat_history(CHAT_HISTORY_FILENAME): |
|
|
|
|
|
|
|
def save_chat_history(chat_history, CHAT_HISTORY_FILENAME): |
|
|
|
|
|
|
|
pinecone_index_name, chroma_collection_name, persist_directory, docsearch_ready, directory_name = init() |
|
|
|
|
|
def main(pinecone_index_name, chroma_collection_name, persist_directory, docsearch_ready, directory_name): |
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
main(pinecone_index_name, chroma_collection_name, persist_directory, docsearch_ready, directory_name) |
|
|