Spaces:
Sleeping
Sleeping
File size: 2,062 Bytes
7794643 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
from flask import Flask, render_template
from flask_socketio import SocketIO, emit
import subprocess
import threading
app = Flask(__name__)
socketio = SocketIO(app, cors_allowed_origins='*')
# Global variable to manage thread control
stop_event = threading.Event()
# Global variable to store the process reference
current_process = None
def stream_process_output(command):
"""Execute a command and emit stdout line by line, with thread control."""
global current_process
# Start the subprocess
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
current_process = process # Store the reference to the current process
for stdout_line in process.stdout:
if stop_event.is_set(): # Stop if the event is triggered
break
socketio.emit('response', {'word': stdout_line})
socketio.sleep(0.1) # Yield to allow other threads to run
process.stdout.close()
process.wait()
@socketio.on('query')
def start_stream(data=None):
"""Start the process and stream its stdout to the client, ensuring thread control."""
global stop_event, current_process
if data is None:
return
query = data['query']
command = ['python3', 'run_inference.py', '-m', 'Llama3-8B-1.58-100B-tokens-TQ2_0.gguf', '-p', query]
if 'args' in data and data.get('args'):
additional_args = data['args'].strip().split()
command.extend(additional_args)
print(f"command- {command}")
# If there is an existing running task, terminate it
if current_process and current_process.poll() is None:
current_process.terminate()
stop_event.set() # Signal the current thread to stop
stop_event.clear() # Reset the stop event for the new thread
# Start a new background task
socketio.start_background_task(target=stream_process_output, command=command)
@app.route('/')
def index():
return render_template('index.html')
if __name__ == '__main__':
socketio.run(app, host='0.0.0.0', port=7860)
|