Update app.py

app.py

@@ -2,9 +2,9 @@ import gradio as gr
 import numpy as np
 import torch
 from transformers import pipeline, VitsModel
-from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor
+#from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor
 from transformers import WhisperTokenizer, GenerationConfig
-
+from transformers import pipeline, VitsModel, AutoTokenizer, AutoTokenizer
 
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
@@ -22,8 +22,8 @@ asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-medium
 
 # ---------------- Speech generator mms-tts-spa --------------------------#
 
-#vist_model = VitsModel.from_pretrained("facebook/mms-tts-spa")
-#vist_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-spa")
+vist_model = VitsModel.from_pretrained("facebook/mms-tts-spa")
+vist_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-spa")
 
 # ---------------- Speech generator  specht5_tts --------------------------#
 
@@ -39,30 +39,43 @@ speaker_embeddings2 = torch.tensor(speaker_embeddings2)
 print(speaker_embeddings2)
 
 
-def language_detector(text):
-    resultado = lang_detector(text)
-    idioma_detectado = resultado[0]['label']
-    print(idioma_detectado)
-    return idioma_detectado
+# def language_detector(text):
+#     resultado = lang_detector(text)
+#     idioma_detectado = resultado[0]['label']
+#     print(idioma_detectado)
+#     return idioma_detectado
 
 def translate(audio):
     outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "es"})
     print(outputs["text"])
     return outputs["text"]
 
+# def synthesise(text):
+#     inputs = processor(text=text, return_tensors="pt")
+#     output = model.generate_speech(inputs["input_ids"], speaker_embeddings2, vocoder=vocoder)
+#     return output
+
+# def speech_to_speech_translation(audio):
+#     translated_text = translate(audio)
+#     synthesised_speech = synthesise(translated_text)
+#     audio_data = synthesised_speech.cpu().numpy()
+#     #audio_data = np.squeeze(audio_data)
+#     #audio_data = audio_data / np.max(np.abs(audio_data))
+#     sample_rate = 16000
+#     return (sample_rate, audio_data)
+
 def synthesise(text):
-    inputs = processor(text=text, return_tensors="pt")
-    output = model.generate_speech(inputs["input_ids"], speaker_embeddings2, vocoder=vocoder)
+    print(text)
+    inputs = vist_tokenizer(text, return_tensors="pt")
+    with torch.no_grad():
+        output = vist_model(**inputs).waveform[0]
     return output
 
 def speech_to_speech_translation(audio):
     translated_text = translate(audio)
     synthesised_speech = synthesise(translated_text)
-    audio_data = synthesised_speech.cpu().numpy()
-    #audio_data = np.squeeze(audio_data)
-    #audio_data = audio_data / np.max(np.abs(audio_data))
-    sample_rate = 16000
-    return (sample_rate, audio_data)
+    synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
+    return 16000, synthesised_speech
 
 title = "Cascaded STST"
 description = """