multiple models

app.py

@@ -7,19 +7,22 @@ from transformers import pipeline
 
 p = pipeline("automatic-speech-recognition")
 
-"""Use text to call chat method from main_old.py"""
-def add_text(history, text):
+"""Use text to call chat method from main.py"""
+
+models = ["GPT-3.5", "Flan UL2", "GPT-4", "Flan T5"]
+
+def add_text(history, text, model):
     print("Question asked: " + text)
-    response = run_model(text)
+    response = run_model(text, model)
     history = history + [(text, response)]
     print(history)
     return history, ""
 
 
-def run_model(text):
+def run_model(text, model):
     start_time = time.time()
     print("start time:" + str(start_time))
-    response = run(question=text)
+    response = run(text, model)
     end_time = time.time()
     # If response contains string `SOURCES:`, then add a \n before `SOURCES`
     if "SOURCES:" in response:
@@ -31,12 +34,12 @@ def run_model(text):
 
 
 
-def get_output(history, audio):
+def get_output(history, audio, model):
 
     txt = p(audio)["text"]
     # history.append(( (audio, ) , txt))
     audio_path = 'response.wav'
-    response = run_model(txt)
+    response = run_model(txt, model)
     # Remove all text from SOURCES: to the end of the string
     trimmed_response = response.split("SOURCES:")[0]
     myobj = gTTS(text=trimmed_response, lang='en', slow=False)
@@ -48,34 +51,54 @@ def get_output(history, audio):
     print(history)
     return history
 
+def set_model(history, model):
+    print("Model selected: " + model)
+    history = get_first_message(history)
+    index(model)
+    return history
+
+
+def get_first_message(history):
+    history = [(None,
+                'Learn about <a href="https://www.coursera.org/learn/3d-printing-revolution/home">3D printing Revolution</a> course with referred sources. Try out the new voice to voice Q&A on the course! ')]
+    return history
+
+
 def bot(history):
     return history
 
 with gr.Blocks() as demo:
-    index()
-    chatbot = gr.Chatbot([(None,'Learn about <a href="https://www.coursera.org/learn/3d-printing-revolution/home">3D printing Revolution</a> course with referred sources. Try out the new voice to voice Q&A on the course! ')], elem_id="chatbot").style(height=750)
 
+    chatbot = gr.Chatbot(get_first_message([]), elem_id="chatbot").style(height=600)
+
+    with gr.Row():
+        # Create radio button to select model
+        radio = gr.Radio(models, label="Choose a model", value="GPT-3.5", type="value")
     with gr.Row():
-        with gr.Column(scale=0.85):
+        with gr.Column(scale=0.75):
             txt = gr.Textbox(
                 label="Coursera Voice Q&A Bot",
                 placeholder="Enter text and press enter, or upload an image", lines=1
             ).style(container=False)
 
-        with gr.Column(scale=0.15):
-            audio = gr.Audio(source="microphone", type="filepath")
+        with gr.Column(scale=0.25):
+            audio = gr.Audio(source="microphone", type="filepath").style(container=False)
 
-    txt.submit(add_text, [chatbot, txt], [chatbot, txt], postprocess=False).then(
+    txt.submit(add_text, [chatbot, txt, radio], [chatbot, txt], postprocess=False).then(
         bot, chatbot, chatbot
     )
 
-
-    audio.change(fn=get_output, inputs=[chatbot, audio], outputs=[chatbot]).then(
+    audio.change(fn=get_output, inputs=[chatbot, audio, radio], outputs=[chatbot]).then(
         bot, chatbot, chatbot
     )
 
-    print(audio)
+    radio.change(fn=set_model, inputs=[chatbot, radio], outputs=[chatbot]).then(bot, chatbot, chatbot)
+
     audio.change(lambda:None, None, audio)
 
+    set_model(chatbot, radio.value)
+
 if __name__ == "__main__":
+    demo.queue()
+    demo.queue(concurrency_count=5)
     demo.launch(debug=True)

main.py

@@ -1,9 +1,10 @@
-from utils import get_search_index, get_qa_chain, generate_answer
+from utils import get_search_index, generate_answer, set_model_and_embeddings
 
-def index():
-    get_search_index()
+def index(model):
+    set_model_and_embeddings(model)
+    get_search_index(model)
     return True
 
-def run(question):
-    index()
+def run(question, model):
+    index(model)
     return generate_answer(question)

requirements.txt

@@ -1,4 +1,4 @@
-langchain==0.0.166
+langchain
 openai
 faiss-cpu==1.7.3
 unstructured==0.5.8
@@ -6,4 +6,5 @@ ffmpeg-python
 transformers
 gtts
 torch
-tiktoken
+tiktoken
+huggingface-hub

utils.py

@@ -2,10 +2,11 @@ import os
 import pickle
 
 import faiss
+from langchain import HuggingFaceHub
 from langchain.chains import ConversationalRetrievalChain
 from langchain.chat_models import ChatOpenAI
 from langchain.document_loaders import DirectoryLoader, TextLoader, UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings
 from langchain.memory import ConversationBufferWindowMemory
 from langchain.prompts.chat import (
     ChatPromptTemplate,
@@ -16,24 +17,26 @@ from langchain.text_splitter import CharacterTextSplitter
 from langchain.vectorstores.faiss import FAISS
 
 
-pickle_file = "open_ai.pkl"
-index_file = "open_ai.index"
+global model_name
 
+models = ["GPT-3.5", "Flan UL2", "GPT-4", "Flan T5"]
 
+pickle_file = "_vs.pkl"
+index_file = "_vs.index"
+models_folder = "models/"
 
-gpt_3_5 = ChatOpenAI(model_name='gpt-4',temperature=0.1)
+llm = ChatOpenAI(model_name="gpt-4", temperature=0.1)
 
 embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')
 
-
 chat_history = []
 
-memory = ConversationBufferWindowMemory(memory_key="chat_history")
+memory = ConversationBufferWindowMemory(memory_key="chat_history", k=10)
 
-gpt_3_5_index = None
+vectorstore_index = None
 
 system_template = """You are Coursera QA Bot. Have a conversation with a human, answering the following questions as best you can.
-You are a teaching assistant for a Coursera Course: The 3D Printing Evolution and can answer any question about that using vectorstore.
+You are a teaching assistant for a Coursera Course: The 3D Printing Evolution and can answer any question about that using vectorstore or context.
 Use the following pieces of context to answer the users question. 
 ----------------
 {context}"""
@@ -44,32 +47,82 @@ messages = [
 ]
 CHAT_PROMPT = ChatPromptTemplate.from_messages(messages)
 
-def get_search_index():
-    global gpt_3_5_index
-    if os.path.isfile(pickle_file) and os.path.isfile(index_file) and os.path.getsize(pickle_file) > 0:
+
+def set_model_and_embeddings(model):
+    global chat_history
+    set_model(model)
+    # set_embeddings(model)
+    chat_history = []
+
+
+def set_model(model):
+    global llm
+    print("Setting model to " + str(model))
+    if model == "GPT-3.5":
+        print("Loading GPT-3.5")
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.1)
+    elif model == "GPT-4":
+        print("Loading GPT-4")
+        llm = ChatOpenAI(model_name="gpt-4", temperature=0.1)
+    elif model == "Flan UL2":
+        print("Loading Flan-UL2")
+        llm = HuggingFaceHub(repo_id="google/flan-ul2", model_kwargs={"temperature": 0.1, "max_new_tokens":500})
+    elif model == "Flan T5":
+        print("Loading Flan T5")
+        llm = HuggingFaceHub(repo_id="google/flan-t5-base", model_kwargs={"temperature": 0.1})
+    else:
+        print("Loading GPT-3.5 from else")
+        llm = ChatOpenAI(model_name="text-davinci-002", temperature=0.1)
+
+
+def set_embeddings(model):
+    global embeddings
+    if model == "GPT-3.5" or model == "GPT-4":
+        print("Loading OpenAI embeddings")
+        embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')
+    elif model == "Flan UL2" or model == "Flan T5":
+        print("Loading Hugging Face embeddings")
+        embeddings = HuggingFaceHubEmbeddings(repo_id="sentence-transformers/all-MiniLM-L6-v2")
+
+
+def get_search_index(model):
+    global vectorstore_index
+    if os.path.isfile(get_file_path(model, pickle_file)) and os.path.isfile(
+            get_file_path(model, index_file)) and os.path.getsize(get_file_path(model, pickle_file)) > 0:
         # Load index from pickle file
-        with open(pickle_file, "rb") as f:
+        with open(get_file_path(model, pickle_file), "rb") as f:
             search_index = pickle.load(f)
+            print("Loaded index")
     else:
-        search_index = create_index()
+        search_index = create_index(model)
+        print("Created index")
 
-    gpt_3_5_index = search_index
+    vectorstore_index = search_index
     return search_index
 
 
-def create_index():
+def create_index(model):
     source_chunks = create_chunk_documents()
     search_index = search_index_from_docs(source_chunks)
-    faiss.write_index(search_index.index, index_file)
+    faiss.write_index(search_index.index, get_file_path(model, index_file))
     # Save index to pickle file
-    with open(pickle_file, "wb") as f:
+    with open(get_file_path(model, pickle_file), "wb") as f:
         pickle.dump(search_index, f)
     return search_index
 
 
+def get_file_path(model, file):
+    # If model is GPT3.5 or GPT4 return models_folder + openai + file else return models_folder + hf + file
+    if model == "GPT-3.5" or model == "GPT-4":
+        return models_folder + "openai" + file
+    else:
+        return models_folder + "hf" + file
+
+
 def search_index_from_docs(source_chunks):
     # print("source chunks: " + str(len(source_chunks)))
     # print("embeddings: " + str(embeddings))
+
     search_index = FAISS.from_documents(source_chunks, embeddings)
     return search_index
 
@@ -83,7 +136,7 @@ def get_html_files():
 def fetch_data_for_embeddings():
     document_list = get_text_files()
     document_list.extend(get_html_files())
-    print("document list" + str(len(document_list)))
+    print("document list: " + str(len(document_list)))
     return document_list
 
 
@@ -100,20 +153,26 @@ def create_chunk_documents():
 
     source_chunks = splitter.split_documents(sources)
 
-    print("sources" + str(len(source_chunks)))
+    print("chunks: " + str(len(source_chunks)))
 
     return source_chunks
 
 
-def get_qa_chain(gpt_3_5_index):
-    global gpt_3_5
+def get_qa_chain(vectorstore_index):
+    global llm, model_name
+    print(llm)
+
     # embeddings_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76)
     # compression_retriever = ContextualCompressionRetriever(base_compressor=embeddings_filter, base_retriever=gpt_3_5_index.as_retriever())
-    chain = ConversationalRetrievalChain.from_llm(gpt_3_5, gpt_3_5_index.as_retriever(), return_source_documents=True,
+    retriever = vectorstore_index.as_retriever(search_type="similarity_score_threshold",
+                                               search_kwargs={"score_threshold": .5})
+
+    chain = ConversationalRetrievalChain.from_llm(llm, retriever, return_source_documents=True,
                                                   verbose=True, get_chat_history=get_chat_history,
                                                   combine_docs_chain_kwargs={"prompt": CHAT_PROMPT})
     return chain
 
+
 def get_chat_history(inputs) -> str:
     res = []
     for human, ai in inputs:
@@ -122,18 +181,19 @@ def get_chat_history(inputs) -> str:
 
 
 def generate_answer(question) -> str:
-    global chat_history, gpt_3_5_index
-    gpt_3_5_chain = get_qa_chain(gpt_3_5_index)
+    global chat_history, vectorstore_index
+    chain = get_qa_chain(vectorstore_index)
 
-    result = gpt_3_5_chain(
+    result = chain(
         {"question": question, "chat_history": chat_history, "vectordbkwargs": {"search_distance": 0.6}})
     chat_history = [(question, result["answer"])]
     sources = []
-    print(result['answer'])
+    print(result)
 
     for document in result['source_documents']:
         source = document.metadata['source']
         sources.append(source.split('/')[-1].split('.')[0])
+        print(sources)
 
     source = ',\n'.join(set(sources))
-    return result['answer'] + '\nSOURCES: ' + source
+    return result['answer'] + '\nSOURCES: ' + source