mistral-7b-openorca

Runtime error

vasilee commited on Oct 3, 2023

Commit

12dc7cc

•

1 Parent(s): 729e996

Update main.py

Files changed (1) hide show

main.py CHANGED Viewed

@@ -12,7 +12,7 @@ class InferenceRequest(BaseModel):
 app = FastAPI()
 llm = Llama(model_path="./models/mistral-7b-openorca.Q4_K_S.gguf",
-            verbose=False, n_ctx=8192)
 @app.get("/")

 app = FastAPI()
 llm = Llama(model_path="./models/mistral-7b-openorca.Q4_K_S.gguf",
+            verbose=False, n_ctx=4096)
 @app.get("/")