Update app.py

app.py

@@ -27,8 +27,13 @@ app.logger.setLevel(logging.DEBUG)  # Set the desired logging level
 #model_path = "../models/model-q4_K.gguf"  # Replace with the actual model path
 #model_name = "model/ggml-model-q4_K.gguf"
 
-repo_name = "IlyaGusev/saiga2_13b_gguf"
-model_name = "model-q4_K.gguf"
+#repo_name = "IlyaGusev/saiga2_13b_gguf"
+#model_name = "model-q4_K.gguf"
+
+repo_name = "IlyaGusev/saiga2_70b_gguf"
+model_name = "ggml-model-q4_1.gguf"
+
+
 
 snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
 
@@ -39,7 +44,8 @@ model = Llama(
     #n_batch=100,
     logits_all=True,
     #n_threads=12,
-    verbose=True
+    verbose=True,
+    n_gqa=8       #must be set for 70b models
 )