Spaces:

muryshev
/

saiga-api-cuda

Paused

muryshev commited on Oct 2, 2023

Commit

4b0ae5d

•

1 Parent(s): 47b6c1e

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -16,7 +16,7 @@ ROLE_TOKENS = {
     "system": SYSTEM_TOKEN
 }
-CONTEXT_SIZE = 4096
 # Create a lock object
 lock = threading.Lock()
@@ -47,7 +47,7 @@ model = Llama(
     logits_all=True,
     #n_threads=12,
     verbose=True,
-    n_gpu_layers=30,
     n_gqa=8       #must be set for 70b models
 )

     "system": SYSTEM_TOKEN
 }
+CONTEXT_SIZE = 3000
 # Create a lock object
 lock = threading.Lock()
     logits_all=True,
     #n_threads=12,
     verbose=True,
+    n_gpu_layers=35,
     n_gqa=8       #must be set for 70b models
 )