Update rag.py

app/main.py

@@ -77,7 +77,7 @@ def upload(files: list[UploadFile]):
 @app.get("/clear")
 def ping():
     session_assistant.clear()
-    message = "All files have been cleared."
+    message = "All files have been cleared. The first query may take a little longer."
     generator = re.split(r'(\s)', message)
     return StreamingResponse(astreamer(generator), media_type='text/event-stream')
 

app/rag.py

@@ -47,8 +47,10 @@ class ChatPDF:
             model_url="https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-GGUF/resolve/main/qwen2-0_5b-instruct-fp16.gguf",
             temperature=0.1,
             max_new_tokens=256,
+            n_ctx=3900,
+            n_batch=256,
             context_window=3900, #32k
-            # generate_kwargs={},
+            generate_kwargs={"max_tokens": 256, "temperature": 0.1, "top_k": 3},
             # model_kwargs={"n_gpu_layers": -1},
             # messages_to_prompt=self.messages_to_prompt,
             # completion_to_prompt=self.completion_to_prompt,