|
@app.function( |
|
class InteractiveChat: |
|
def __init__(self, model_name="openai/whisper-large", tts_choice="OpenVoice", **kwargs): |
|
self.whisper_processor = WhisperProcessor.from_pretrained(model_name) |
|
self.whisper_model = WhisperForConditionalGeneration.from_pretrained(model_name) |
|
self.zephyr_tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta") |
|
self.zephyr_model = AutoModelForCausalLM.from_pretrained("HuggingFaceH4/zephyr-7b-beta", device_map="auto") |
|
self.zephyr_pipeline = pipeline("text-generation", model=self.zephyr_model, tokenizer=self.zephyr_tokenizer) |
|
self.tts_choice = tts_choice |
|
|
|
def generate_response(self, input_data): |
|
input_features = self.whisper_processor(input_data, sampling_rate=16_000, return_tensors="pt").input_features |
|
predicted_ids = self.whisper_model.generate(input_features) |
|
transcription = self.whisper_processor.batch_decode(predicted_ids, skip_special_tokens=True)[0] |
|
|
|
|
|
response = self.zephyr_pipeline(transcription, max_length=1000)[0]["generated_text"] |
|
return transcription, response |
|
|
|
def speak(self, text): |
|
try: |
|
if self.tts_choice == "OpenVoice": |
|
model_path = snapshot_download("facebook/mms-tts-eng") |
|
pipe = pipeline("text-to-speech", model=model_path) |
|
audio_array = pipe(text).audio |
|
pygame.mixer.init() |
|
sound = pygame.sndarray.make_sound(audio_array) |
|
sound.play() |
|
pygame.time.delay(int(sound.get_length() * 1000)) |
|
else: |
|
tts = gTTS(text=text, lang='en') |
|
tts.save("response.mp3") |
|
pygame.mixer.init() |
|
pygame.mixer.music.load("response.mp3") |
|
pygame.mixer.music.play() |
|
while pygame.mixer.music.get_busy(): |
|
pygame.time.Clock().tick(10) |
|
except Exception as e: |
|
print("Error occurred during speech generation:", e) |
|
|
|
|
|
with gr.Blocks() as demo: |
|
model_choice = gr.Dropdown(["openai/whisper-large"], label="Whisper Model", value="openai/whisper-large") |
|
tts_choice = gr.Radio(["OpenVoice", "gTTS"], label="TTS Engine", value="OpenVoice") |
|
input_data = gr.Audio(source="microphone", type="numpy", label="Speak Your Message") |
|
output_text = gr.Textbox(label="Transcription and Response") |
|
|
|
model_choice.change(lambda x, y: InteractiveChat(x, y), inputs=[model_choice, tts_choice], outputs=None) |
|
tts_choice.change(lambda x, y: InteractiveChat(y, x), inputs=[tts_choice, model_choice], outputs=None) |
|
input_data.change(lambda x, model: model.generate_response(x), inputs=[input_data, model_choice], |
|
outputs=output_text) |
|
input_data.change(lambda x, model: model.speak(x[1]), inputs=[output_text, model_choice], |
|
outputs=None)) |
