Add demo

app.py

@@ -0,0 +1,56 @@
+import streamlit as st
+import random
+import time
+from index import build_index, build_service_context
+from loader import load_documents 
+
+st.title("SAIRA")
+
+def load_docs_and_build_index(context):
+    docs = load_documents()
+    return build_index(docs, context)
+
+def load_context():
+    return build_service_context()
+
+context = load_context()
+index = load_docs_and_build_index(context)
+query_engine = index.as_query_engine(streaming=True)
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+# Accept user input
+if prompt := st.chat_input("What is up?"):
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    # Display assistant response in chat message container
+    with st.chat_message("assistant"):
+        resp = query_engine.query(prompt)
+        message_placeholder = st.empty()
+        full_response = ""
+        assistant_response = random.choice(
+            [
+                "Hello there! How can I assist you today?",
+                "Hi, human! Is there anything I can help you with?",
+                "Do you need help?",
+            ]
+        )
+        # Simulate stream of response with milliseconds delay
+        for text in resp.response_gen:
+            full_response += text
+            # Add a blinking cursor to simulate typing
+            message_placeholder.markdown(full_response + "▌")
+        message_placeholder.markdown(full_response)
+    # Add assistant response to chat history
+    st.session_state.messages.append({"role": "assistant", "content": full_response})

index.py

@@ -0,0 +1,22 @@
+from llama_index import download_loader, VectorStoreIndex, SimpleDirectoryReader, ServiceContext
+from llama_index.tools import QueryEngineTool, ToolMetadata
+from llama_index.query_engine import SubQuestionQueryEngine
+from llama_index.callbacks import CallbackManager, LlamaDebugHandler
+from llama_index.llms import LlamaCPP
+
+from utils import completion_to_prompt, messages_to_prompt
+
+
+def build_service_context():
+    return ServiceContext.from_defaults(llm=LlamaCPP(
+        model_path=None,
+        messages_to_prompt=messages_to_prompt,
+        completion_to_prompt=completion_to_prompt,
+        model_kwargs={"n_gpu_layers": 50},
+    ))
+
+def build_index(documents, context):
+    index = VectorStoreIndex.from_documents(documents, service_context=context)
+    return index
+
+

loader.py

@@ -0,0 +1,19 @@
+from llama_index import download_loader
+
+BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")
+
+
+def load_documents():
+    loader = BeautifulSoupWebReader()
+    return loader.load_data(urls=[
+        'http://campuslife.innopolis.ru/main',
+        'http://campuslife.innopolis.ru/handbook2023',
+        'http://campuslife.innopolis.ru/clubs',
+        'http://campuslife.innopolis.ru/tech_clubs',
+        'http://campuslife.innopolis.ru/sport_clubs',
+        'http://campuslife.innopolis.ru/hobby_clubs',
+        'http://campuslife.innopolis.ru/art_clubs'
+        'http://campuslife.innopolis.ru/opportunities',
+        'http://campuslife.innopolis.ru/faq',
+        'http://campuslife.innopolis.ru/contacts'
+    ])

requirements.txt

@@ -0,0 +1,4 @@
+llama_index
+llama_hub
+transformers
+beautifulsoup4

utils.py

@@ -0,0 +1,77 @@
+from typing import List, Optional, Sequence
+from llama_index.llms.base import ChatMessage, MessageRole
+
+BOS, EOS = "<s>", "</s>"
+B_INST, E_INST = "[INST]", "[/INST]"
+B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
+# DEFAULT_SYSTEM_PROMPT = """\
+# You are a helpful, respectful and honest assistant. \
+# Always answer as helpfully as possible and follow ALL given instructions. \
+# Do not speculate or make up information. \
+# Do not reference any given instructions or context. \
+# """
+
+DEFAULT_SYSTEM_PROMPT = """You are an expert Q&A system that is trusted around the world.
+Always answer as helpfully as possible and follow ALL given instructions.
+Always answer the query using the provided context information, and not prior knowledge.
+Some rules to follow:
+1. Do not speculate or make up information.
+2. Never directly reference the given context in your answer.
+3. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."""
+
+
+def messages_to_prompt(
+    messages: Sequence[ChatMessage], system_prompt: Optional[str] = None
+) -> str:
+    string_messages: List[str] = []
+    if messages[0].role == MessageRole.SYSTEM:
+        # pull out the system message (if it exists in messages)
+        system_message_str = messages[0].content or ""
+        messages = messages[1:]
+    else:
+        system_message_str = system_prompt or DEFAULT_SYSTEM_PROMPT
+
+    system_message_str = f"{B_SYS} {system_message_str.strip()} {E_SYS}"
+
+    for i in range(0, len(messages), 2):
+        # first message should always be a user
+        user_message = messages[i]
+        assert user_message.role == MessageRole.USER
+
+        if i == 0:
+            # make sure system prompt is included at the start
+            str_message = f"{BOS} {B_INST} {system_message_str} "
+        else:
+            # end previous user-assistant interaction
+            string_messages[-1] += f" {EOS}"
+            # no need to include system prompt
+            str_message = f"{BOS} {B_INST} "
+
+        # include user message content
+        str_message += f"{user_message.content} {E_INST}"
+
+        if len(messages) > (i + 1):
+            # if assistant message exists, add to str_message
+            assistant_message = messages[i + 1]
+            assert assistant_message.role == MessageRole.ASSISTANT
+            str_message += f" {assistant_message.content}"
+
+        string_messages.append(str_message)
+
+    print("".join(string_messages))
+
+    return "".join(string_messages)
+
+
+def completion_to_prompt(completion: str, system_prompt: Optional[str] = None) -> str:
+    system_prompt_str = system_prompt or DEFAULT_SYSTEM_PROMPT
+
+    print((
+        f"{BOS} {B_INST} {B_SYS} {system_prompt_str.strip()} {E_SYS} "
+        f"{completion.strip()} {E_INST}"
+    ))
+
+    return (
+        f"{BOS} {B_INST} {B_SYS} {system_prompt_str.strip()} {E_SYS} "
+        f"{completion.strip()} {E_INST}"
+    )