Update main.py

main.py

@@ -11,8 +11,8 @@ class InferenceRequest(BaseModel):
 
 app = FastAPI()
 
-llm = Llama(model_path="./models/vicuna-7b-v1.5.Q4_K_M.gguf",
-            verbose=False, n_ctx=4096)
+llm = Llama(model_path="./models/mistral-7b-openorca.Q4_K_S.gguf",
+            verbose=False, n_ctx=8192)
 
 
 @app.get("/")
@@ -32,7 +32,7 @@ async def inference(request: InferenceRequest):
     # process request
     try:
         result = llm(input_text, temperature=0.2,
-                     top_k=5, max_tokens=max_tokens)
+                     top_k=5, max_tokens=max_tokens, stop=["<|im_end|>"])
         return result
     except:
         pass