|
from langchain.document_loaders import TextLoader, NotionDirectoryLoader |
|
from langchain.text_splitter import SpacyTextSplitter |
|
from langchain.embeddings import HuggingFaceEmbeddings |
|
from langchain.vectorstores import FAISS |
|
from langchain.chains import RetrievalQA |
|
from langchain.document_loaders import TextLoader, NotionDirectoryLoader |
|
from langchain.vectorstores.base import VectorStoreRetriever |
|
from langchain.llms import OpenAI |
|
|
|
|
|
class CustomEmbedding: |
|
notionDirectoryLoader = NotionDirectoryLoader( |
|
"documents/bussiness_context") |
|
embeddings = HuggingFaceEmbeddings() |
|
|
|
def calculateEmbedding(self): |
|
documents = self.notionDirectoryLoader.load() |
|
text_splitter = SpacyTextSplitter( |
|
chunk_size=256, pipeline="zh_core_web_sm", chunk_overlap=200) |
|
texts = text_splitter.split_documents(documents) |
|
|
|
docsearch = FAISS.from_documents(texts, self.embeddings) |
|
docsearch.save_local( |
|
folder_path="./documents/business_context.faiss") |
|
|
|
def getFAQChain(self, llm=OpenAI(temperature=0.7)): |
|
docsearch = FAISS.load_local( |
|
"./documents/business_context.faiss", self.embeddings) |
|
retriever = VectorStoreRetriever(vectorstore=docsearch) |
|
faq_chain = RetrievalQA.from_llm( |
|
llm=llm, retriever=retriever, verbose=True) |
|
return faq_chain |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
