File size: 3,336 Bytes
d347764 cc6d9dc ff3f816 d347764 6ab6711 32e9053 d347764 32e9053 d347764 32e9053 d347764 6ab6711 d347764 cc6d9dc 32e9053 d347764 6ab6711 d347764 ff3f816 d347764 f805e49 8bebf7d f805e49 c737803 d347764 226ec3a d347764 f805e49 d347764 c737803 3946ba6 c737803 49c1298 d347764 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
import gradio as gr
import librosa
import logging
import numpy as np
import torch
from datasets import load_dataset
from transformers import VitsModel, VitsTokenizer
from transformers import WhisperForConditionalGeneration, WhisperProcessor
device = "cuda:0" if torch.cuda.is_available() else "cpu"
target_language = "french"
# load speech translation checkpoint
whisper_model_name = "openai/whisper-base"
whisper_processor = WhisperProcessor.from_pretrained(whisper_model_name)
whisper_model = WhisperForConditionalGeneration.from_pretrained(whisper_model_name)
decoder_ids = whisper_processor.get_decoder_prompt_ids(language=target_language, task="transcribe")
# load text-to-speech checkpoint and speaker embeddings
model = VitsModel.from_pretrained("facebook/mms-tts-fra")
tokenizer = VitsTokenizer.from_pretrained("facebook/mms-tts-fra")
def translate(audio):
if isinstance(audio, str):
# Account for recorded audio
audio = {
"path": audio,
"sampling_rate": 16_000,
"array": librosa.load(audio, sr=16_000)[0]
}
elif audio["sampling_rate"] != 16_000:
audio["array"] = librosa.resample(audio["array"], audio["sampling_rate"], 16_000)
input_features = whisper_processor(audio["array"], sampling_rate=16000, return_tensors="pt").input_features
predicted_ids = whisper_model.generate(input_features, forced_decoder_ids=decoder_ids)
translated_text = whisper_processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]
return translated_text
def synthesise(text):
inputs = tokenizer(text, return_tensors="pt")
with torch.no_grad():
outputs = model(inputs["input_ids"])
speech = outputs["waveform"]
return speech.cpu()
def speech_to_speech_translation(audio):
translated_text = translate(audio)
logging.info(f"Translated Text: {translated_text}")
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return 16000, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in French. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
[SpeechT5 TTS](https://huggingface.co/preetam8/speecht5_finetuned_voxpopuli_fr) model for text-to-speech finetuned for french:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
logging.getLogger().setLevel(logging.INFO)
demo.launch()
|