base model

app.py

@@ -1,16 +1,30 @@
 import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from inference import get_bot_response
+from rag import get_context
+from config import config
 from huggingface_hub import InferenceClient
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 model_name = "mistralai/Mistral-7B-Instruct-v0.2"
-client = InferenceClient(model_name)
 
 
-print("test")
+client = InferenceClient(model_name)
+print("tokenizer start loading")
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+print("tokenizer loaded")
+print("model start loading")
+model = AutoModelForCausalLM.from_pretrained(model_name,
+                                             device_map="auto",
+                                             trust_remote_code=False,
+                                             revision="main")
+print("model loaded")
+
+# model = AutoModelForCausalLM.from_pretrained(config["model_checkpoint"],
+#                                                 device_map="auto",
+#                                                 trust_remote_code=False,
+#                                                 revision="main")
+
 
 def respond(
     message,
@@ -22,31 +36,12 @@ def respond(
 ):
     messages = [{"role": "system", "content": system_message}]
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+    request = message
+    context = get_context(request, config["top_k"])
+    response = get_bot_response(request, context, model, tokenizer)
 
-        response += token
-        yield response
+    return response
 
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[

app_old.py

@@ -1,30 +1,16 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from inference import get_bot_response
-from rag import get_context
-from config import config
 from huggingface_hub import InferenceClient
 
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 model_name = "mistralai/Mistral-7B-Instruct-v0.2"
-
-
 client = InferenceClient(model_name)
 
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-
-print("model start loading")
-model = AutoModelForCausalLM.from_pretrained(model_name,
-                                             device_map="auto",
-                                             trust_remote_code=False,
-                                             revision="main")
-print("model loaded")
-
-# model = AutoModelForCausalLM.from_pretrained(config["model_checkpoint"],
-#                                                 device_map="auto",
-#                                                 trust_remote_code=False,
-#                                                 revision="main")
 
+print("test")
 
 def respond(
     message,
@@ -36,12 +22,31 @@ def respond(
 ):
     messages = [{"role": "system", "content": system_message}]
 
-    request = message
-    context = get_context(request, config["top_k"])
-    response = get_bot_response(request, context, model, tokenizer)
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
 
-    return response
+        response += token
+        yield response
 
+"""
+For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+"""
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[

rag.py

@@ -5,7 +5,7 @@ from llama_index.core.query_engine import RetrieverQueryEngine
 from llama_index.core.postprocessor import SimilarityPostprocessor
 
 from config import config
-
+print("rag start loading")
 Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
 
 Settings.llm = None
@@ -14,7 +14,7 @@ Settings.chunk_overlap = config["chunk_overlap"]
 
 documents = SimpleDirectoryReader("rag_data").load_data()
 index = VectorStoreIndex.from_documents(documents)
-
+print("rag loaded")
 def get_context(query, top_k = 3):
     retriever = VectorIndexRetriever(
         index=index,