support local vocoder

.gitattributes

@@ -34,4 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 xvector filter=lfs diff=lfs merge=lfs -text
 TTS_models filter=lfs diff=lfs merge=lfs -text
-vocoders filter=lfs diff=lfs merge=lfs -text
+parallel_wavegan filter=lfs diff=lfs merge=lfs -text

app.py

@@ -47,26 +47,42 @@ model = AutoModelForSpeechSeq2Seq.from_pretrained("KevinGeng/whipser_medium_en_P
 
 transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/whipser_medium_en_PAL300_step25")
 
+# Text2Mel models
 # @title English multi-speaker pretrained model { run: "auto" }
 lang = "English"
-# tag = "kan-bayashi/libritts_xvector_vits"
+vits_tag = "kan-bayashi/libritts_xvector_vits"
 ft2_tag = "kan-bayashi/libritts_xvector_vits" #@param ["kan-bayashi/vctk_gst_tacotron2", "kan-bayashi/vctk_gst_transformer", "kan-bayashi/vctk_xvector_tacotron2", "kan-bayashi/vctk_xvector_transformer", "kan-bayashi/vctk_xvector_conformer_fastspeech2", "kan-bayashi/vctk_gst+xvector_tacotron2", "kan-bayashi/vctk_gst+xvector_transformer", "kan-bayashi/vctk_gst+xvector_conformer_fastspeech2", "kan-bayashi/vctk_multi_spk_vits", "kan-bayashi/vctk_full_band_multi_spk_vits", "kan-bayashi/libritts_xvector_transformer"
-transformer_tag = "kan-bayashi/libritts_xvector_transformer"
 # ft2_tag = "kan-bayashi/libritts_xvector_conformer_fastspeech2"
-# vits needs no vocoder
+transformer_tag = "kan-bayashi/libritts_xvector_transformer"
+
+# !!! vits needs no vocoder !!!
+# Local Text2Mel models
+
+vits_config_local = "TTS_models/libritts_xvector_vits/config.yaml"
+vits_model_local = "TTS_models/libritts_xvector_vits/train.total_count.ave_10best.pth"
+
+# TODO
+ft2_config_local = ""
+ft2_model_local= ""
+transformer_config_local = ""
+transformer_config_local = ""
 
 # Vocoders
 vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
 hifigan_vocoder_tag = "parallel_wavegan/parallel_wavegan/libritts_hifigan.v1"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
 
+# Local Vocoders
+## Make sure the use parallel_wavegan as prefix (PWG feature)
+vocoder_tag_local = "parallel_wavegan/vctk_parallel_wavegan.v1.long"
+hifigan_vocoder_tag_local = "parallel_wavegan/libritts_hifigan.v1"
+
 from espnet2.bin.tts_inference import Text2Speech
 from espnet2.utils.types import str_or_none
 
 # local import
 text2speech = Text2Speech.from_pretrained(
-    train_config = "TTS_models/libritts_xvector_vits/config.yaml",
-    model_file="TTS_models/libritts_xvector_vits/train.total_count.ave_10best.pth",
-    vocoder_tag=str_or_none(vocoder_tag),
+    train_config = vits_config_local,
+    model_file=vits_model_local,
     device="cuda",
     use_att_constraint=False,
     backward_window=1,
@@ -77,7 +93,7 @@ text2speech = Text2Speech.from_pretrained(
 # Fastspeech2 
 ft2_text2speech = Text2Speech.from_pretrained(
     model_tag=ft2_tag,
-    vocoder_tag=str_or_none(vocoder_tag),
+    vocoder_tag=str_or_none(vocoder_tag_local),
     device="cuda",
     use_att_constraint=False,
     backward_window=1,
@@ -88,7 +104,7 @@ ft2_text2speech = Text2Speech.from_pretrained(
 # Fastspeech2 + hifigan
 ft2_text2speech_hifi = Text2Speech.from_pretrained(
     model_tag=ft2_tag,
-    vocoder_tag=str_or_none(hifigan_vocoder_tag),
+    vocoder_tag=str_or_none(hifigan_vocoder_tag_local),
     device="cuda",
     use_att_constraint=False,
     backward_window=1,
@@ -99,16 +115,13 @@ ft2_text2speech_hifi = Text2Speech.from_pretrained(
 # transformer tag 
 transformer_text2speech = Text2Speech.from_pretrained(
     model_tag=transformer_tag,
-    vocoder_tag=str_or_none(vocoder_tag),
+    vocoder_tag=str_or_none(vocoder_tag_local),
     device="cuda",
     use_att_constraint=False,
     backward_window=1,
     forward_window=3,
     speed_control_alpha=1.0,
 )
-pdb.set_trace()
-# from google.cloud import texttospeech
-# Google_TTS_client = texttospeech.TextToSpeechClient()
 
 import glob
 import os
@@ -139,8 +152,10 @@ male_spks = {
     "Male3": "672_122797"
 }
 
-female_spks = {"Female1": "5683_32865", "Female2": "121_121726", "Female3": "8463_287645"}
-# "F3": "121_121726"
+female_spks = {"Female1": "5683_32865",
+               "Female2": "121_121726",
+               "Female3": "8463_287645"}
+
 spks = dict(male_spks, **female_spks)
 spk_names = sorted(spks.keys())
 

vocoders/vctk_parallel_wavegan.v1.long/._checkpoint-1000000steps.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ccb8cfc739515054284e6ac7c75afdab0c771eba7d132c4e19efff528147a1a1
-size 223

vocoders/vctk_parallel_wavegan.v1.long/._stats.h5

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f688626155bff6853a6045408be9bb248828abc482e2218ee0d93183cede5062
-size 223