Update app.py

app.py

@@ -71,20 +71,20 @@ stop_generation = False
 def generate_tokens(model, generator):
     global stop_generation
     app.logger.info('generate_tokens started')
-    #with lock:
-    try:
-        for token in generator:            
-            if token == model.token_eos() or stop_generation:
-                stop_generation = False
-                app.logger.info('Abort generating')
-                yield b''  # End of chunk
-                break
-                
-            token_str = model.detokenize([token])#.decode("utf-8", errors="ignore")
-            yield token_str 
-    except Exception as e:
-        app.logger.info('generator exception')
-        yield b''  # End of chunk
+    with lock:
+        try:
+            for token in generator:            
+                if token == model.token_eos() or stop_generation:
+                    stop_generation = False
+                    app.logger.info('Abort generating')
+                    yield b''  # End of chunk
+                    break
+                    
+                token_str = model.detokenize([token])#.decode("utf-8", errors="ignore")
+                yield token_str 
+        except Exception as e:
+            app.logger.info('generator exception')
+            yield b''  # End of chunk
 
 @app.route('/stop_generation', methods=['GET'])
 def handler_stop_generation():
@@ -133,7 +133,7 @@ def generate_search_request():
         logits_all=True,
         #n_threads=12,
         verbose=True,
-        n_gpu_layers=40,
+        n_gpu_layers=30,
         n_gqa=8       #must be set for 70b models
     )
     
@@ -183,7 +183,7 @@ def generate_response():
         logits_all=True,
         #n_threads=12,
         verbose=True,
-        n_gpu_layers=40,
+        n_gpu_layers=30,
         n_gqa=8       #must be set for 70b models
     )
     
@@ -239,4 +239,4 @@ def generate_response():
     return Response(generate_tokens(model, generator), content_type='text/plain', status=200, direct_passthrough=True)
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860, debug=False, threaded=True)
+    app.run(host="0.0.0.0", port=7860, debug=False, threaded=False)