File size: 1,617 Bytes
90729fc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
from langchain.document_loaders import TextLoader, NotionDirectoryLoader
from langchain.text_splitter import SpacyTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from langchain.chains import RetrievalQA
from langchain.document_loaders import TextLoader, NotionDirectoryLoader
from langchain.vectorstores.base import VectorStoreRetriever
from langchain.llms import OpenAI
class CustomEmbedding:
notionDirectoryLoader = NotionDirectoryLoader(
"documents/bussiness_context")
embeddings = HuggingFaceEmbeddings()
def calculateEmbedding(self):
documents = self.notionDirectoryLoader.load()
text_splitter = SpacyTextSplitter(
chunk_size=256, pipeline="zh_core_web_sm", chunk_overlap=200)
texts = text_splitter.split_documents(documents)
docsearch = FAISS.from_documents(texts, self.embeddings)
docsearch.save_local(
folder_path="./documents/business_context.faiss")
def getFAQChain(self, llm=OpenAI(temperature=0.7)):
docsearch = FAISS.load_local(
"./documents/business_context.faiss", self.embeddings)
retriever = VectorStoreRetriever(vectorstore=docsearch)
faq_chain = RetrievalQA.from_llm(
llm=llm, retriever=retriever, verbose=True)
return faq_chain
# customerEmbedding = CustomEmbedding()
# # customerEmbedding.calculateEmbedding()
# # customerEmbedding.calculateNotionEmbedding()
# faq_chain = customerEmbedding.getFAQChain()
# result = faq_chain.run(
# "Smart Domain 分层架构")
# print(result)
|