add fastspeech2 TTS, fastspeech2 TTS + HifiGAN, transformer TTS models

app.py

@@ -12,6 +12,7 @@ import torchaudio
 from transformers import pipeline
 from pathlib import Path
 
+import pdb
 # local import
 import sys
 from espnet2.bin.tts_inference import Text2Speech
@@ -48,9 +49,15 @@ transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/whipser_
 
 # @title English multi-speaker pretrained model { run: "auto" }
 lang = "English"
-tag = "kan-bayashi/libritts_xvector_vits"
+# tag = "kan-bayashi/libritts_xvector_vits"
+ft2_tag = "kan-bayashi/libritts_xvector_vits" #@param ["kan-bayashi/vctk_gst_tacotron2", "kan-bayashi/vctk_gst_transformer", "kan-bayashi/vctk_xvector_tacotron2", "kan-bayashi/vctk_xvector_transformer", "kan-bayashi/vctk_xvector_conformer_fastspeech2", "kan-bayashi/vctk_gst+xvector_tacotron2", "kan-bayashi/vctk_gst+xvector_transformer", "kan-bayashi/vctk_gst+xvector_conformer_fastspeech2", "kan-bayashi/vctk_multi_spk_vits", "kan-bayashi/vctk_full_band_multi_spk_vits", "kan-bayashi/libritts_xvector_transformer"
+transformer_tag = "kan-bayashi/libritts_xvector_transformer"
+# ft2_tag = "kan-bayashi/libritts_xvector_conformer_fastspeech2"
 # vits needs no vocoder
+
+# Vocoders
 vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+hifigan_vocoder_tag = "parallel_wavegan/parallel_wavegan/libritts_hifigan.v1"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
 
 from espnet2.bin.tts_inference import Text2Speech
 from espnet2.utils.types import str_or_none
@@ -67,6 +74,42 @@ text2speech = Text2Speech.from_pretrained(
     speed_control_alpha=1.0,
 )
 
+# Fastspeech2 
+ft2_text2speech = Text2Speech.from_pretrained(
+    model_tag=ft2_tag,
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+# Fastspeech2 + hifigan
+ft2_text2speech_hifi = Text2Speech.from_pretrained(
+    model_tag=ft2_tag,
+    vocoder_tag=str_or_none(hifigan_vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+# transformer tag 
+transformer_text2speech = Text2Speech.from_pretrained(
+    model_tag=transformer_tag,
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+# from google.cloud import texttospeech
+# Google_TTS_client = texttospeech.TextToSpeechClient()
+
 import glob
 import os
 import numpy as np
@@ -95,14 +138,12 @@ male_spks = {
     "Male2": "1320_122612",
     "Male3": "672_122797"
 }
-    # "M3": "1188_133604",
-    # "M4": "61_70970",
+
 female_spks = {"Female1": "5683_32865", "Female2": "121_121726", "Female3": "8463_287645"}
 # "F3": "121_121726"
 spks = dict(male_spks, **female_spks)
 spk_names = sorted(spks.keys())
 
-
 def ASRTTS(audio_file, spk_name, ref_text=""):
     spk = spks[spk_name]
     spembs = xvectors[spk]
@@ -156,6 +197,105 @@ def ASRTTS_clean(audio_file, spk_name):
     return save_id
 
 
+def ft2_ASRTTS_clean(audio_file, spk_name):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+
+    reg_text = transcriber(audio_file)["text"]
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = ft2_text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_fs2_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+    return save_id
+
+def ft2_ASRTTS_clean_hifi(audio_file, spk_name):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+
+    reg_text = transcriber(audio_file)["text"]
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = ft2_text2speech_hifi(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_fs2_hifi_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+    return save_id
+
+def transformer_ASRTTS_clean(audio_file, spk_name):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+
+    reg_text = transcriber(audio_file)["text"]
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = transformer_text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_transformer_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+    return save_id
+
+# def google_ASRTTS_clean(audio_file, spk_name):
+#     spk = spks[spk_name]
+#     spembs = xvectors[spk]
+
+#     reg_text = transcriber(audio_file)["text"]
+#     # pdb.set_trace()
+#     synthesis_input = texttospeech.SynthesisInput(text=reg_text)
+#     voice = texttospeech.VoiceSelectionParams(
+#         language_code="en-US", ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL
+#     )
+#     audio_config = texttospeech.AudioConfig(
+#         audio_encoding=texttospeech.AudioEncoding.MP3
+#     )
+#     response = Google_TTS_client.synthesize_speech(
+#         input=synthesis_input, voice=voice, audio_config=audio_config
+#     )
+    
+#     save_id = (
+#         "./wav/" + Path(audio_file).stem + "_google_" + spk_name + "_spkembs.wav"
+
+#     )
+#     with open(save_id, "wb") as out_file:
+#         out_file.write(response.audio_content)
+    
+#     return save_id
+
+
 reference_textbox = gr.Textbox(
     value="",
     placeholder="Input reference here",
@@ -198,10 +338,14 @@ def show_icon(choice):
         spk_icon = gr.Image.update(value="speaker_icons/male1.png", visible=True)
     elif choice == "Male2":
         spk_icon = gr.Image.update(value="speaker_icons/male2.png", visible=True)
+    elif choice == "Male3":
+        spk_icon = gr.Image.update(value="speaker_icons/male3.png", visible=True)
     elif choice == "Female1":
         spk_icon = gr.Image.update(value="speaker_icons/female1.png", visible=True)
     elif choice == "Female2":
         spk_icon = gr.Image.update(value="speaker_icons/female2.png", visible=True)
+    elif choice == "Female3":
+        spk_icon = gr.Image.update(value="speaker_icons/female3.png", visible=True)
     return spk_icon
 
 def get_download_file(audio_file=None):
@@ -246,18 +390,76 @@ with gr.Blocks(
         speaker_option.change(
             fn=show_icon, inputs=speaker_option, outputs=spk_icon
         )
-
-    b2 = gr.Button("Convert")
+    with gr.Column():
+        with gr.Row():
+            b2 = gr.Button("Convert")
+            
+            output_audio = gr.Audio(
+                source="upload", file="filepath", label="Converted Audio", interactive=False
+            )
+            
+            b2.click(
+                ASRTTS_clean,
+                inputs=[input_audio, speaker_option],
+                outputs=output_audio,
+                api_name="convert"
+            )
+        with gr.Row():
+            # Fastspeech2 + PWG [under construction]
+            b_ft2 = gr.Button("Convert_fastspeech2")
+            
+            output_audio_ft2= gr.Audio(
+                source="upload", file="filepath", label="Converted Audio", interactive=False
+            )
+            
+            b_ft2.click(
+                ft2_ASRTTS_clean,
+                inputs=[input_audio, speaker_option],
+                outputs=output_audio_ft2,
+                api_name="convert_ft2"
+            )
+        with gr.Row():
+            # Fastspeech2 + hifigan [under construction]
+            b_ft2_hifi = gr.Button("Convert_fastspeech2+HifiGAN")
+            
+            output_audio_ft2_hifi= gr.Audio(
+                source="upload", file="filepath", label="Converted Audio", interactive=False
+            )
+            
+            b_ft2_hifi.click(
+                ft2_ASRTTS_clean_hifi,
+                inputs=[input_audio, speaker_option],
+                outputs=output_audio_ft2_hifi,
+                api_name="convert_ft2_hifi"
+            )
+        with gr.Row():
+            # transformer [TODO]
+            b_transformer = gr.Button("Convert_transformer")
+
+            output_audio_transformer= gr.Audio(
+                source="upload", file="filepath", label="Converted Audio", interactive=False
+            )
+            
+            b_transformer.click(
+                transformer_ASRTTS_clean,
+                inputs=[input_audio, speaker_option],
+                outputs=output_audio_transformer,
+                api_name="convert_trans"
+            )
+        
+    # google tts [TODO]
+    # b_google = gr.Button("Convert_googleTTS")
     
-    output_audio = gr.Audio(
-        source="upload", file="filepath", label="Converted Audio", interactive=False
-    )
+    # output_audio_google= gr.Audio(
+    #     source="upload", file="filepath", label="Converted Audio", interactive=False
+    # )
     
-    b2.click(
-        ASRTTS_clean,
-        inputs=[input_audio, speaker_option],
-        outputs=output_audio,
-        api_name="convert"
-    )
-
-demo.launch(share=False)
+    # b_google.click(
+    #     google_ASRTTS_clean,
+    #     inputs=[input_audio, speaker_option],
+    #     outputs=output_audio_google,
+    #     api_name="convert"
+    # )
+    
+    
+demo.launch(share=True)