Update app.py

app.py

@@ -15,6 +15,7 @@ from langchain_core.messages import (
 )
 from langchain_huggingface import ChatHuggingFace
 from langchain_core.output_parsers import StrOutputParser
+from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain_huggingface import HuggingFaceEndpoint
 from qdrant_client.http import models as rest
 #from qdrant_client import QdrantClient
@@ -140,7 +141,7 @@ async def chat(query,history,sources,reports,subtype,year):
     question_lst= [query]
     for question in question_lst:
         retriever = vectorstore.as_retriever(
-          search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.6, "k": 5, "filter":filter})
+          search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.6, "k": 3, "filter":filter})
         
         context_retrieved = retriever.invoke(question)
         print(len(context_retrieved))
@@ -184,6 +185,7 @@ async def chat(query,history,sources,reports,subtype,year):
 
     # llama-3_1 endpoint = https://howaqfw0lpap12sg.us-east-1.aws.endpoints.huggingface.cloud
     # llama-3 endpoint = https://nhe9phsr2zhs0e36.eu-west-1.aws.endpoints.huggingface.cloud
+    callbacks = [StreamingStdOutCallbackHandler()]
     llm_qa = HuggingFaceEndpoint(
         endpoint_url="https://howaqfw0lpap12sg.us-east-1.aws.endpoints.huggingface.cloud",
         max_new_tokens=512*3,
@@ -191,6 +193,8 @@ async def chat(query,history,sources,reports,subtype,year):
         top_p=0.95,
         typical_p=0.95,
         temperature=0.01,
+        callbacks=callbacks,
+        streaming=True,
         repetition_penalty=1.03,)
 
     # create rag chain