attempting some control

app.py

@@ -5,6 +5,9 @@ from llama_index.core import (
     StorageContext,
     load_index_from_storage, Settings,
 )
+from llama_index.core.indices.vector_store import VectorIndexRetriever
+from llama_index.core.query_engine import RetrieverQueryEngine
+from llama_index.core.postprocessor import SentenceTransformerRerank, SimilarityPostprocessor
 from llama_index.llms.huggingface import HuggingFaceLLM
 import torch
 PERSIST_DIR = './storage'
@@ -18,14 +21,38 @@ Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-base-en-v1.5",
 Settings.llm = HuggingFaceLLM(
     model_name="meta-llama/Meta-Llama-3-8B-Instruct",
     tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct",
-    context_window=2048,
-    max_new_tokens=256,
-    generate_kwargs={"temperature": 0.7, "top_k": 50, "top_p": 0.95},
+    context_window=4096,
+    max_new_tokens=512,
+    generate_kwargs={"temperature": 0.3, "top_k": 50, "top_p": 0.85},
     device_map="auto",
 )
 
 storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
 index = load_index_from_storage(storage_context)
+
+prompt_helper = PromptHelper(
+    context_window=4096,
+    num_output=512,
+    chunk_overlap_ratio=0.1,
+    chunk_size_limit=None
+)
+
+retriever = VectorIndexRetriever(
+    index=index,
+    similarity_top_k=5,
+)
+
+query_engine = RetrieverQueryEngine.from_args(
+    retriever,
+    node_postprocessors=[SimilarityPostprocessor(similarity_cutoff=0.7)],
+    prompt_helper=prompt_helper
+)
+
+def chatbot_response(message, history):
+    # Add a custom prompt template
+    prompt = f"Based on the Elder Scrolls lore, please answer the following question:\n\n{message}\n\nAnswer:"
+    response = query_engine.query(prompt)
+    return str(response)
 query_engine = index.as_query_engine()