import os import pickle import langchain import faiss from langchain import HuggingFaceHub from langchain.chains import ConversationalRetrievalChain from langchain.chat_models import ChatOpenAI from langchain.document_loaders import DirectoryLoader, TextLoader, UnstructuredHTMLLoader from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings from langchain.memory import ConversationBufferWindowMemory from langchain.prompts.chat import ( ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate, ) from langchain.text_splitter import CharacterTextSplitter from langchain.vectorstores.faiss import FAISS from langchain.cache import InMemoryCache langchain.llm_cache = InMemoryCache() global model_name models = ["GPT-3.5", "Flan UL2", "GPT-4", "Flan T5"] pickle_file = "_vs.pkl" index_file = "_vs.index" models_folder = "models/" llm = ChatOpenAI(model_name="gpt-4", temperature=0.1) embeddings = OpenAIEmbeddings(model='text-embedding-ada-002') chat_history = [] memory = ConversationBufferWindowMemory(memory_key="chat_history", k=10) vectorstore_index = None file_url_mapping = { 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/03_design-thinking-in-action/02_multiply.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/1qm8i/multiply', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/03_design-thinking-in-action/01_utensil-grip-personalization.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/BYBmh/utensil-grip-personalization', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/03_design-thinking-in-action/03_city-x-project.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/0HDFr/city-x-project', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/01_module-4-information/02_an-intro-to-design-thinking.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/IdLQM/an-intro-to-design-thinking', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/02_an-introduction-to-design-thinking/01_what-is-design-thinking.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/npYkr/what-is-design-thinking', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/02_an-introduction-to-design-thinking/03_3d-printing-and-design-thinking.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/QJAGJ/3d-printing-and-design-thinking', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/02_an-introduction-to-design-thinking/02_understanding-user-needs.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/Jj03y/understanding-user-needs', 'docs/3d-printing-applications/01_course-orientation/01_about-the-courses/01_welcome-to-3d-printing-applications.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/xYJax/welcome-to-3d-printing-applications', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/02_whats-different-about-3d-printing/01_complexity-is-free.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/lA0z9/complexity-is-free', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/02_whats-different-about-3d-printing/02_3d-printing-a-paradigm-shift.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/I3NUA/3d-printing-a-paradigm-shift', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/01_webinar-3d-printing-with-soft-materials-bonus.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/PbM8Z/webinar-3d-printing-with-soft-materials-bonus', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/03_3d-printing-industry-trends/02_3d-printing-use-cases.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/5dSO8/3d-printing-use-cases', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/03_3d-printing-industry-trends/04_a-view-from-the-trenches.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/FAINi/a-view-from-the-trenches', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/03_3d-printing-industry-trends/03_a-venture-capitalists-view-of-the-industry.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/kcSMh/a-venture-capitalists-view-of-the-industry', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/03_3d-printing-industry-trends/01_a-look-into-the-past-and-the-future.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/IWgMe/a-look-into-the-past-and-the-future', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/04_3d-printing-on-the-edge/01_cutting-edge-applications.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/Cy1Ef/cutting-edge-applications', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/04_3d-printing-on-the-edge/02_bioprinting.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/V6WNO/bioprinting', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/02_a-new-way-of-making.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/jPkoW/a-new-way-of-making', 'docs/3d-printing-applications/06_course-wrap-up/01_course-wrap-up-whats-next/01_whats-next.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/ffuzj/whats-next', 'docs/3d-printing-applications/06_course-wrap-up/01_course-wrap-up-whats-next/03_gies-online-programs.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/gPv5h/gies-online-programs', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/03_3d-printing-in-education/01_3d-printing-in-education.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/rrgbB/3d-printing-in-education', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/03_3d-printing-in-education/02_girls-in-stem-makergirls.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/8JKL4/girls-in-stem-makergirls', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/01_module-3-information/02_3d-printing-in-development-and-education.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/oncn3/3d-printing-in-development-and-education', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/06_3d4d-with-techfortrade.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/aRmfA/3d4d-with-techfortrade', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/03_from-trash-to-objects.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/C5Yaq/from-trash-to-objects', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/04_enabling-the-future.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/PaC0r/enabling-the-future', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/05_a-step-up-with-bionic-hands.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/94W3o/a-step-up-with-bionic-hands', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/02_illinois-marketplace-and-maker-literacy-program.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/lE9Zj/illinois-marketplace-and-maker-literacy-program', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/02_3d-printing-for-development-3d4d/01_3d-printing-and-subsistence-marketplaces.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/MJPvM/3d-printing-and-subsistence-marketplaces', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/03_3d-printing-and-intellectual-property/01_3d-printing-and-the-future-or-demise-of-intellectual-property.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/6QRBO/3d-printing-and-the-future-or-demise-of-intellectual-property', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/01_module-2-information/02_on-demand-manufacturing.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/mwrCN/on-demand-manufacturing', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/01_a-market-of-one-3d-hubs.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/SKKXa/a-market-of-one-3d-hubs', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/06_careers-in-3d-printing.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/NBK4o/careers-in-3d-printing', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/04_learning-by-making-and-making-for-fun-at-the-fab-lab.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/PuC0A/learning-by-making-and-making-for-fun-at-the-fab-lab', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/05_for-businesses-and-entrepreneurs.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/I3WQW/for-businesses-and-entrepreneurs', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/03_on-demand-and-local-techshop.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/EMRez/on-demand-and-local-techshop', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/02_everyone-can-be-a-maker/02_a-market-of-a-few.en.txt':'https://www.coursera.org/learn/3d-printing-applications/lecture/4dB6p/a-market-of-a-few', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/01_module-4-information/03_module-4-readings_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/hnOry/module-4-readings', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/01_module-4-information/01_module-4-overview_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/zq4e8/module-4-overview', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/04_module-4-assignments/01_module-4-graded-quiz_exam.html':'https://www.coursera.org/learn/3d-printing-applications/exam/nRxIo/module-4-graded-quiz', 'docs/3d-printing-applications/05_module-4-from-ideas-to-objects/04_module-4-assignments/02_module-4-peer-review-assignment_peer_assignment_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/peer/EWHOn/module-4-peer-review-assignment', 'docs/3d-printing-applications/07_Resources/04_3d-printing-news-sources/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/LcUHI', 'docs/3d-printing-applications/07_Resources/03_3d-printing-applications/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/rLuyV', 'docs/3d-printing-applications/07_Resources/03_3d-printing-applications/01__scientists-create-new-bio-ink-for-3d-printing-with-stem-cells-322296.html':'http://tech.firstpost.com/news-analysis/scientists-create-new-bio-ink-for-3d-printing-with-stem-cells-322296.html', 'docs/3d-printing-applications/07_Resources/05_recommended-books/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/8hxKz', 'docs/3d-printing-applications/07_Resources/06_explore-the-imba/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/49U0k', 'docs/3d-printing-applications/07_Resources/01_about-our-team/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/62fHX', 'docs/3d-printing-applications/07_Resources/02_glossary/01__resources.html':'https://www.coursera.org/learn/3d-printing-applications/resources/dG0g4', 'docs/3d-printing-applications/01_course-orientation/01_about-the-courses/04_glossary_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/QQpUl/glossary', 'docs/3d-printing-applications/01_course-orientation/01_about-the-courses/03_about-the-discussion-forums_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/MRtAV/about-the-discussion-forums', 'docs/3d-printing-applications/01_course-orientation/01_about-the-courses/02_syllabus_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/9DenU/syllabus', 'docs/3d-printing-applications/01_course-orientation/02_about-your-classmates/01_updating-your-profile_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/DfAqY/updating-your-profile', 'docs/3d-printing-applications/01_course-orientation/02_about-your-classmates/02_social-media_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/Qf1Am/social-media', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/04_webinar-3d-printing-comes-of-age_3d-printing-comes-of-age.html':'http://www.3dprintingprofs.com/2016/07/3d-printing-comes-of-age-webinar/', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/05_bonus-webinar-mymini-factory-ceo-interview_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/2XNdg/bonus-webinar-mymini-factory-ceo-interview', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/03_bonus-webinar-whats-all-the-hype-around-3d-printing_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/jKR1S/bonus-webinar-whats-all-the-hype-around-3d-printing', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/02_bonus-webinar-audio-whats-the-hype-around-3dprinting_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/Wh3Lh/bonus-webinar-audio-whats-the-hype-around-3dprinting', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/06_bonus-webinar-the-maker-movement-in-education_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/hHU9F/bonus-webinar-the-maker-movement-in-education', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/06_optional-content-webinars-with-experts/04_webinar-3d-printing-comes-of-age_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/zNdWR/webinar-3d-printing-comes-of-age', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/05_module-1-assignments/02_module-1-peer-review-assignment_peer_assignment_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/peer/wW35G/module-1-peer-review-assignment', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/05_module-1-assignments/01_module-1-graded-quiz_exam.html':'https://www.coursera.org/learn/3d-printing-applications/exam/Mf63o/module-1-graded-quiz', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/01_module-1-overview_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/wEptY/module-1-overview', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/03_module-1-readings_3d-printing-complexity-is-free-may-be-costly-for-some.html':'https://www2.deloitte.com/us/en/insights/focus/3d-opportunity/3d-printing-complexity-is-free-may-be-costly-for-some.html', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/03_module-1-readings_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/SpZ2p/module-1-readings', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/03_module-1-readings_3d-printing-comes-of-age.html':'http://www.pwc.com/us/en/industrial-products/3d-printing-comes-of-age.html', 'docs/3d-printing-applications/02_module-1-3d-printing-a-new-way-of-making/01_module-1-information/03_module-1-readings_press71.html':'http://wohlersassociates.com/press71.html', 'docs/3d-printing-applications/06_course-wrap-up/01_course-wrap-up-whats-next/02_congratulations_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/oDj5t/congratulations', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/03_3d-printing-in-education/03_resources-for-educators_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/sdDp2/resources-for-educators', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/01_module-3-information/03_module-3-readings_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/5kR9o/module-3-readings', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/01_module-3-information/01_module-3-overview_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/OChGO/module-3-overview', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/01_module-3-information/03_module-3-readings_book.html':'http://sdu.ictp.it/3D/book.html', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/04_module-3-assignments/01_module-3-graded-quiz_exam.html':'https://www.coursera.org/learn/3d-printing-applications/exam/NMiTK/module-3-graded-quiz', 'docs/3d-printing-applications/04_module-3-3d-printing-for-development-and-education/04_module-3-assignments/02_module-3-peer-review-assignment_peer_assignment_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/peer/5xxgp/module-3-peer-review-assignment', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/04_module-2-assignment/01_module-2-graded-quiz_exam.html':'https://www.coursera.org/learn/3d-printing-applications/exam/6Z8Ef/module-2-graded-quiz', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/04_module-2-assignment/02_module-2-peer-review-assignment_peer_assignment_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/peer/Pt8hO/module-2-peer-review-assignment', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/01_module-2-information/01_module-2-overview_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/94W7x/module-2-overview', 'docs/3d-printing-applications/03_module-2-3d-printing-on-demand-manufacturing/01_module-2-information/03_module-2-readings_instructions.html':'https://www.coursera.org/learn/3d-printing-applications/supplement/3uaqE/module-2-readings' } system_template = """You are Coursera QA Bot. Have a conversation with a human, answering the following questions as best you can. You are a teaching assistant for a Coursera Course: 3D Printing Applications and can answer any question about that using vectorstore or context. Use the following pieces of context to answer the users question. ---------------- {context}""" messages = [ SystemMessagePromptTemplate.from_template(system_template), HumanMessagePromptTemplate.from_template("{question}"), ] CHAT_PROMPT = ChatPromptTemplate.from_messages(messages) def set_model_and_embeddings(model): global chat_history set_model(model) # set_embeddings(model) chat_history = [] def set_model(model): global llm print("Setting model to " + str(model)) if model == "GPT-3.5": print("Loading GPT-3.5") llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.1) elif model == "GPT-4": print("Loading GPT-4") llm = ChatOpenAI(model_name="gpt-4", temperature=0.1) elif model == "Flan UL2": print("Loading Flan-UL2") llm = HuggingFaceHub(repo_id="google/flan-ul2", model_kwargs={"temperature": 0.1, "max_new_tokens":500}) elif model == "Flan T5": print("Loading Flan T5") llm = HuggingFaceHub(repo_id="google/flan-t5-base", model_kwargs={"temperature": 0.1}) else: print("Loading GPT-3.5 from else") llm = ChatOpenAI(model_name="text-davinci-002", temperature=0.1) def set_embeddings(model): global embeddings if model == "GPT-3.5" or model == "GPT-4": print("Loading OpenAI embeddings") embeddings = OpenAIEmbeddings(model='text-embedding-ada-002') elif model == "Flan UL2" or model == "Flan T5": print("Loading Hugging Face embeddings") embeddings = HuggingFaceHubEmbeddings(repo_id="sentence-transformers/all-MiniLM-L6-v2") def get_search_index(model): global vectorstore_index if os.path.isfile(get_file_path(model, pickle_file)) and os.path.isfile( get_file_path(model, index_file)) and os.path.getsize(get_file_path(model, pickle_file)) > 0: # Load index from pickle file with open(get_file_path(model, pickle_file), "rb") as f: search_index = pickle.load(f) print("Loaded index") else: search_index = create_index(model) print("Created index") vectorstore_index = search_index return search_index def create_index(model): source_chunks = create_chunk_documents() search_index = search_index_from_docs(source_chunks) faiss.write_index(search_index.index, get_file_path(model, index_file)) # Save index to pickle file with open(get_file_path(model, pickle_file), "wb") as f: pickle.dump(search_index, f) return search_index def get_file_path(model, file): # If model is GPT3.5 or GPT4 return models_folder + openai + file else return models_folder + hf + file if model == "GPT-3.5" or model == "GPT-4": return models_folder + "openai" + file else: return models_folder + "hf" + file def search_index_from_docs(source_chunks): # print("source chunks: " + str(len(source_chunks))) # print("embeddings: " + str(embeddings)) search_index = FAISS.from_documents(source_chunks, embeddings) return search_index def get_html_files(): loader = DirectoryLoader('docs', glob="**/*.html", loader_cls=UnstructuredHTMLLoader, recursive=True) document_list = loader.load() return document_list def fetch_data_for_embeddings(): document_list = get_text_files() document_list.extend(get_html_files()) # use file_url_mapping to set metadata of document to url which has been set as the source for document in document_list: document.metadata["url"] = file_url_mapping.get(document.metadata["source"]) print("document list: " + str(len(document_list))) return document_list def get_text_files(): loader = DirectoryLoader('docs', glob="**/*.txt", loader_cls=TextLoader, recursive=True) document_list = loader.load() return document_list def create_chunk_documents(): sources = fetch_data_for_embeddings() splitter = CharacterTextSplitter(separator=" ", chunk_size=800, chunk_overlap=0) source_chunks = splitter.split_documents(sources) print("chunks: " + str(len(source_chunks))) return source_chunks def get_qa_chain(vectorstore_index): global llm, model_name print(llm) # embeddings_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76) # compression_retriever = ContextualCompressionRetriever(base_compressor=embeddings_filter, base_retriever=gpt_3_5_index.as_retriever()) retriever = vectorstore_index.as_retriever(search_type="similarity_score_threshold", search_kwargs={"score_threshold": .5}) chain = ConversationalRetrievalChain.from_llm(llm, retriever, return_source_documents=True, verbose=True, get_chat_history=get_chat_history, combine_docs_chain_kwargs={"prompt": CHAT_PROMPT}) return chain def get_chat_history(inputs) -> str: res = [] for human, ai in inputs: res.append(f"Human:{human}\nAI:{ai}") return "\n".join(res) def generate_answer(question) -> str: global chat_history, vectorstore_index chain = get_qa_chain(vectorstore_index) result = chain( {"question": question, "chat_history": chat_history, "vectordbkwargs": {"search_distance": 0.6}}) chat_history = [(question, result["answer"])] sources = [] print(result) for document in result['source_documents']: sources.append("\n" + document.metadata['url']) print(sources) source = ',\n'.join(set(sources)) return result['answer'] + 'SOURCES: ' + source