File size: 6,512 Bytes
d58f539 2558f9d f7de418 d58f539 f7de418 d6247a0 dec22aa 4ff6b18 eedb394 4ff6b18 dec22aa 645d699 7fa63ee f8bd65e 2558f9d 645d699 1ee1757 694882d d6247a0 1ee1757 d6247a0 a0f34aa d6247a0 645d699 d6247a0 1ee1757 d6247a0 694882d 913a139 d6247a0 7fa63ee 1ee1757 7fa63ee 1ee1757 f8bd65e 7fa63ee 1ee1757 7fa63ee 1ee1757 7fa63ee 694882d 7fa63ee 1ee1757 7fa63ee 694882d 7fa63ee 1ee1757 7fa63ee e9633ca 1ee1757 e9633ca 1ee1757 7fa63ee e9633ca 1ee1757 e9633ca 1ee1757 d6247a0 f8bd65e d58f539 2558f9d 1ee1757 2558f9d f8bd65e 2558f9d f8bd65e 1ee1757 ef46ff0 1ee1757 a0f34aa 694882d 7fa63ee 694882d 1ee1757 7fa63ee 694882d 1ee1757 e9633ca 1ee1757 e9633ca 694882d 84a59c3 1ee1757 f8bd65e a0f34aa 645d699 f8bd65e 1ee1757 4ff6b18 1ee1757 84a59c3 645d699 1ee1757 cc76c21 1ee1757 645d699 d778dbd 1ee1757 645d699 a0f34aa 1ee1757 a0f34aa d778dbd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 |
import gradio as gr
from gradio_webrtc import WebRTC, ReplyOnPause, AdditionalOutputs
import numpy as np
import io
from pydub import AudioSegment
import openai
import time
import base64
import os
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
if account_sid and auth_token:
from twilio.rest import Client
client = Client(account_sid, auth_token)
token = client.tokens.create()
rtc_configuration = {
"iceServers": token.ice_servers,
"iceTransportPolicy": "relay",
}
else:
rtc_configuration = None
def update_or_append_conversation(conversation, id, role, content):
# Find if there's an existing message with the given id
for message in conversation:
if message.get("id") == id and message.get("role") == role:
message["content"] = content
return
# If not found, append a new message
conversation.append({"id": id, "role": role, "content": content})
def generate_response_and_audio(audio_bytes: bytes, lepton_conversation: list[dict],
client: openai.OpenAI):
if client is None:
raise gr.Error("Please enter a valid API key first.")
# mp3 bitrate
bitrate = 128
audio_data = base64.b64encode(audio_bytes).decode()
try:
stream = client.chat.completions.create(
extra_body={
"require_audio": True,
"tts_preset_id": "jessica",
"tts_audio_format": "mp3",
"tts_audio_bitrate": bitrate
},
model="llama3.1-8b",
messages=lepton_conversation + [{"role": "user", "content": [{"type": "audio", "data": audio_data}]}],
temperature=0.7,
max_tokens=256,
stream=True,
)
id = str(time.time())
full_response = ""
asr_result = ""
all_audio = b""
for i, chunk in enumerate(stream):
if not chunk.choices:
continue
delta = chunk.choices[0].delta
content = delta.content
audio = getattr(chunk.choices[0], "audio", [])
asr_results = getattr(chunk.choices[0], "asr_results", [])
if asr_results:
print(i, "asr_results")
asr_result += "".join(asr_results)
yield id, None, asr_result, None
if content:
print(i, "content")
full_response += content
yield id, full_response, None, None
if audio:
print(i, "audio")
# Accumulate audio bytes and yield them
audio_bytes_accumulated = b''.join([base64.b64decode(a) for a in audio])
all_audio += audio_bytes_accumulated
audio = AudioSegment.from_file(io.BytesIO(audio_bytes_accumulated), format="mp3")
audio_array = np.array(audio.get_array_of_samples(), dtype=np.int16).reshape(1, -1)
print("audio.frame_rate", audio.frame_rate)
yield id, None, None, (audio.frame_rate, audio_array)
if all_audio:
all_audio = AudioSegment.from_file(io.BytesIO(all_audio), format="mp3")
all_audio.export("all_audio.mp3", format="mp3")
yield id, full_response, asr_result, None
print("finishing loop")
except Exception as e:
raise gr.Error(f"Error during audio streaming: {e}")
def response(audio: tuple[int, np.ndarray], lepton_conversation: list[dict],
gradio_conversation: list[dict], client: openai.OpenAI):
audio_buffer = io.BytesIO()
segment = AudioSegment(
audio[1].tobytes(),
frame_rate=audio[0],
sample_width=audio[1].dtype.itemsize,
channels=1,
)
segment.export(audio_buffer, format="mp3")
generator = generate_response_and_audio(audio_buffer.getvalue(), lepton_conversation, client)
for id, text, asr, audio in generator:
if asr:
update_or_append_conversation(lepton_conversation, id, "user", asr)
update_or_append_conversation(gradio_conversation, id, "user", asr)
yield AdditionalOutputs(lepton_conversation, gradio_conversation)
if text:
update_or_append_conversation(lepton_conversation, id, "assistant", text)
update_or_append_conversation(gradio_conversation, id, "assistant", text)
yield AdditionalOutputs(lepton_conversation, gradio_conversation)
if audio:
yield audio
else:
yield AdditionalOutputs(lepton_conversation, gradio_conversation)
def set_api_key(lepton_api_key):
try:
client = openai.OpenAI(
base_url="https://llama3-1-8b.lepton.run/api/v1/",
api_key=lepton_api_key
)
except:
raise gr.Error("Invalid API keys. Please try again.")
gr.Info("Successfully set API keys.", duration=3)
return client, gr.update(visible=True), gr.update(visible=False)
with gr.Blocks() as demo:
with gr.Group():
with gr.Row():
chatbot = gr.Chatbot(label="Conversation", type="messages")
with gr.Row(visible=False) as mic_row:
audio = WebRTC(modality="audio", mode="send-receive",
label="Audio Stream",
rtc_configuration=rtc_configuration)
with gr.Row(equal_height=True) as api_row:
api_key_input = gr.Textbox(type="password", value=os.getenv("LEPTONAI_API_KEY"),
label="Enter Your Lepton AI Key")
client_state = gr.State(None)
lepton_conversation = gr.State([{"role": "system",
"content": "You are a knowledgeable assistant who will engage in spoken conversations with users. "
"Keep your answers short and natural as they will be read aloud."}])
api_key_input.submit(set_api_key, inputs=[api_key_input],
outputs=[client_state, mic_row, api_row])
audio.stream(
ReplyOnPause(response, output_sample_rate=44100, output_frame_size=882),
inputs=[audio, lepton_conversation, chatbot, client_state],
outputs=[audio]
)
audio.on_additional_outputs(lambda l, g: (l, g), outputs=[lepton_conversation, chatbot],
queue=False, show_progress="hidden")
demo.launch() |