multiaccent inference, cleaners and models

Dockerfile

@@ -40,11 +40,14 @@ COPY --chown=user requirements.txt $HOME/app/
 
 RUN pip install -r requirements.txt
 
-# download from inference script
-#RUN huggingface-cli download BSC-LT/matcha-tts-cat-multispeaker matcha_multispeaker_cat_opset_15_10_steps_2399.onnx --local-dir $HOME/app/
-#RUN huggingface-cli download BSC-LT/matcha-tts-cat-multispeaker config.yaml --local-dir $HOME/app/
-#RUN huggingface-cli download BSC-LT/vocos-mel-22khz-cat  mel_spec_22khz_cat.onnx --local-dir $HOME/app/
-#RUN huggingface-cli download BSC-LT/vocos-mel-22khz-cat  config.yaml --local-dir $HOME/app/
+# download from hf hub
+RUN huggingface-cli download BSC-LT/matcha-tts-cat-multiaccent matcha_multispeaker_cat_bal_opset_15_10_steps.onnx --local-dir  $HOME/app/
+RUN huggingface-cli download BSC-LT/matcha-tts-cat-multiaccent matcha_multispeaker_cat_occ_opset_15_10_steps.onnx --local-dir  $HOME/app/
+RUN huggingface-cli download BSC-LT/matcha-tts-cat-multiaccent matcha_multispeaker_cat_val_opset_15_10_steps.onnx --local-dir  $HOME/app/
+RUN huggingface-cli download BSC-LT/matcha-tts-cat-multiaccent config.yaml--local-dir  $HOME/app/
+
+RUN huggingface-cli download BSC-LT/vocos-mel-22khz-cat  mel_spec_22khz_cat.onnx --local-dir $HOME/app/
+RUN huggingface-cli download BSC-LT/vocos-mel-22khz-cat  config.yaml --local-dir $HOME/app/
 
 COPY --chown=user . $HOME/app/
 

infer_onnx.py

@@ -10,10 +10,11 @@ import yaml
 import json
 import os
 
-from huggingface_hub import hf_hub_download
 from time import perf_counter
+import random
 
-DEFAULT_SPEAKER_ID = os.environ.get("DEFAULT_SPEAKER_ID", default="caf_08106")
+DEFAULT_SPEAKER_ID = os.environ.get("DEFAULT_SPEAKER_ID", default="quim")
+DEFAULT_ACCENT= os.environ.get("DEFAULT_ACCENT", default="balear")
 
 def intersperse(lst, item):
     result = [item] * (len(lst) * 2 + 1)
@@ -21,10 +22,10 @@ def intersperse(lst, item):
     return result
 
 
-def process_text(i: int, text: str, device: torch.device):
+def process_text(i: int, text: str, device: torch.device, cleaner:str):
     print(f"[{i}] - Input text: {text}")
     x = torch.tensor(
-        intersperse(text_to_sequence(text, ["catalan_cleaners"]), 0),
+        intersperse(text_to_sequence(text, [cleaner]), 0),
         dtype=torch.long,
         device=device,
     )[None]
@@ -33,20 +34,36 @@ def process_text(i: int, text: str, device: torch.device):
     print(x_phones)
     return x.numpy(), x_lengths.numpy()
 
-MODEL_PATH_MATCHA_MEL=hf_hub_download(repo_id="BSC-LT/matcha-tts-cat-multispeaker", filename="matcha_multispeaker_cat_opset_15_10_steps_2399.onnx")
-MODEL_PATH_MATCHA="matcha_hifigan_multispeaker_cat.onnx"
-MODEL_PATH_VOCOS=hf_hub_download(repo_id="BSC-LT/vocos-mel-22khz-cat", filename="mel_spec_22khz_cat.onnx")
-CONFIG_PATH=hf_hub_download(repo_id="BSC-LT/vocos-mel-22khz-cat", filename="config.yaml")
-SPEAKER_ID_DICT="spk_to_id.json"
+# paths
+MODEL_PATH_MATCHA_MEL_BAL="matcha_multispeaker_cat_bal_opset_15_10_steps.onnx"
+#MODEL_PATH_MATCHA_MEL_CAT=hf_hub_download(repo_id="BSC-LT/matcha-tts-cat-multispeaker", filename="matcha_multispeaker_cat_opset_15_10_steps_2399.onnx")
+MODEL_PATH_MATCHA_MEL_OCC="matcha_multispeaker_cat_occ_opset_15_10_steps.onnx"
+MODEL_PATH_MATCHA_MEL_VAL="matcha_multispeaker_cat_val_opset_15_10_steps.onnx"
+MODEL_PATH_VOCOS="mel_spec_22khz_cat.onnx"
+CONFIG_PATH="config.yaml"
+SPEAKER_ID_DICT="spk_to_id_2.json"
 
+# Load models
 sess_options = onnxruntime.SessionOptions()
-model_matcha_mel= onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA_MEL), sess_options=sess_options, providers=["CPUExecutionProvider"])
+model_matcha_mel_bal = onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA_MEL_BAL), sess_options=sess_options, providers=["CPUExecutionProvider"])
+#model_matcha_mel_cat = onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA_MEL_CAT), sess_options=sess_options, providers=["CPUExecutionProvider"])
+model_matcha_mel_occ = onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA_MEL_OCC), sess_options=sess_options, providers=["CPUExecutionProvider"])
+model_matcha_mel_val = onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA_MEL_VAL), sess_options=sess_options, providers=["CPUExecutionProvider"])
 model_vocos = onnxruntime.InferenceSession(str(MODEL_PATH_VOCOS), sess_options=sess_options, providers=["CPUExecutionProvider"])
-#model_matcha = onnxruntime.InferenceSession(str(MODEL_PATH_MATCHA), sess_options=sess_options, providers=["CPUExecutionProvider"])
 
 speaker_id_dict = json.load(open(SPEAKER_ID_DICT))
-speakers = [sp for sp in speaker_id_dict.keys()]
-speakers.sort()
+accents = [e for e in speaker_id_dict.keys()]
+
+models={"balear":model_matcha_mel_bal,
+        "occidental": model_matcha_mel_occ,
+        "valencia": model_matcha_mel_val}
+
+cleaners={"balear": "catalan_balear_cleaners",
+        "occidental": "catalan_occidental_cleaners",
+        "valencia": "catalan_valencia_cleaners"}
+
+
+speakers = [sp for sp in speaker_id_dict[DEFAULT_ACCENT].keys()]
 
 def vocos_inference(mel,denoise):
 
@@ -131,10 +148,11 @@ def vocos_inference(mel,denoise):
     return y
 
 
-def tts(text:str, spk_name:str, temperature:float, length_scale:float, denoise:bool):
-    spk_id = speaker_id_dict[spk_name]
+def tts(text:str, accent:str, spk_name:str, temperature:float, length_scale:float, denoise:bool):
+    spk_id = speaker_id_dict[accent][spk_name]
     sid = np.array([int(spk_id)]) if spk_id is not None else None
-    text_matcha , text_lengths = process_text(0,text,"cpu") 
+    text_matcha , text_lengths = process_text(0,text,"cpu",cleaner=cleaners[accent])
+    model_matcha_mel = models[accent]
 
     # MATCHA VOCOS
     inputs = {
@@ -158,15 +176,6 @@ def tts(text:str, spk_name:str, temperature:float, length_scale:float, denoise:b
     with tempfile.NamedTemporaryFile(suffix=".wav", delete=False, dir="/home/user/app") as fp_matcha_vocos:
         sf.write(fp_matcha_vocos.name, wavs_vocos.squeeze(0), 22050, "PCM_24")
 
-    #MATCHA HIFIGAN
-
-    inputs = {
-        "x": text_matcha,
-        "x_lengths": text_lengths,
-        "scales": np.array([temperature, length_scale], dtype=np.float32),
-        "spks": sid
-    }
-    hifigan_t0 = perf_counter()
     print(f"RTF matcha + vocos { (mel_infer_secs + vocos_infer_secs) / (wavs_vocos.shape[1]/22050) }")
     return fp_matcha_vocos.name
 
@@ -197,7 +206,27 @@ with open("about.md", "r", encoding="utf-8") as f:
 
 article = "Training and demo by The Language Technologies Unit from Barcelona Supercomputing Center."
 
-vits2_inference = gr.Interface(
+
+def rs_change(accent):
+    rnd_idx = random.randint(0, 1)
+    return gr.Dropdown(choices=speaker_id_dict[accent], interactive=True,value=list(speaker_id_dict[accent].keys())[rnd_idx])
+
+accent_dropdown = gr.Dropdown(
+            choices=accents,
+            label="Accent",
+            value=DEFAULT_ACCENT,
+            info=f"Models are trained on 4 accents"
+        )
+
+speaker_dropdown = gr.Dropdown(
+            choices=speaker_id_dict[DEFAULT_ACCENT],
+            label="Speaker id",
+            value=DEFAULT_SPEAKER_ID,
+            info=f"Models are trained on 2 speakers. You can prompt the model using one of these speaker ids.",  
+            interactive=True   
+        )
+
+matcha_inference = gr.Interface(
     fn=tts,
     inputs=[
         gr.Textbox(
@@ -205,12 +234,8 @@ vits2_inference = gr.Interface(
             max_lines=1,
             label="Input text",
         ),
-        gr.Dropdown(
-            choices=speakers,
-            label="Speaker id",
-            value=DEFAULT_SPEAKER_ID,
-            info=f"Models are trained on 47 speakers. You can prompt the model using one of these speaker ids."
-        ),
+        accent_dropdown,
+        speaker_dropdown,
         gr.Slider(
             0.1,
             2.0,
@@ -239,7 +264,8 @@ demo = gr.Blocks()
 with demo:
     gr.Markdown(title)
     gr.Markdown(description)
-    gr.TabbedInterface([vits2_inference, about_article], ["Demo", "About"])
+    gr.TabbedInterface([matcha_inference, about_article], ["Demo", "About"])
+    accent_dropdown.select(fn=rs_change, inputs=accent_dropdown, outputs=speaker_dropdown)
     gr.Markdown(article)
 
 demo.queue(max_size=10)

spk_to_id_2.json

@@ -0,0 +1,18 @@
+{
+  "balear":{
+    "quim": 0,
+    "olga": 1
+  },
+  "central":{
+    "grau": 0,
+    "elia": 1
+  },
+  "occidental":{
+    "pere": 0,
+    "emma": 1
+  },
+  "valencia":{  
+    "lluc": 0,
+    "gina": 1
+  }
+}

text/cleaners.py

@@ -17,8 +17,10 @@ from unidecode import unidecode
 from phonemizer import phonemize
 from phonemizer.backend import EspeakBackend
 
-backend = EspeakBackend("ca", preserve_punctuation=True, with_stress=True)
-backend_en = EspeakBackend("en-us", preserve_punctuation=True, with_stress=True)
+backend_cat = EspeakBackend("ca", preserve_punctuation=True, with_stress=True)
+backend_bal = EspeakBackend("ca-ba", preserve_punctuation=True, with_stress=True)
+backend_val = EspeakBackend("ca-va", preserve_punctuation=True, with_stress=True)
+backend_occ = EspeakBackend("ca-nw", preserve_punctuation=True, with_stress=True)
 
 # Regular expression matching whitespace:
 _whitespace_re = re.compile(r"\s+")
@@ -113,21 +115,43 @@ def english_cleaners2(text):
     return phonemes
 
 
-def english_cleaners3(text):
-    """Pipeline for English text, including abbreviation expansion. + punctuation + stress"""
-    text = convert_to_ascii(text)
+def catalan_cleaners(text):
+    """Pipeline for catalan text, including punctuation + stress"""
+    #text = convert_to_ascii(text)
     text = lowercase(text)
-    text = expand_abbreviations(text)
-    phonemes = backend_en.phonemize([text], strip=True)[0]
+    #text = expand_abbreviations(text)
+    phonemes = backend_cat.phonemize([text], strip=True)[0]
+    phonemes = collapse_whitespace(phonemes)
+    return phonemes
+
+def catalan_balear_cleaners(text):
+    """Pipeline for Catalan text, including abbreviation expansion. + punctuation + stress"""
+    # text = convert_to_ascii(text)
+    text = lowercase(text)
+    # text = expand_abbreviations(text)
+    phonemes = backend_bal.phonemize([text], strip=True, njobs=1)[0]
     phonemes = collapse_whitespace(phonemes)
+    # print(phonemes)  # check punctuations!!
     return phonemes
 
+def catalan_occidental_cleaners(text):
+    """Pipeline for Catalan text, including abbreviation expansion. + punctuation + stress"""
+    # text = convert_to_ascii(text)
+    text = lowercase(text)
+    # text = expand_abbreviations(text)
+    phonemes = backend_occ.phonemize([text], strip=True, njobs=1)[0]
+    phonemes = collapse_whitespace(phonemes)
+    # print(phonemes)  # check punctuations!!
+    return phonemes
 
-def catalan_cleaners(text):
-    """Pipeline for catalan text, including punctuation + stress"""
-    #text = convert_to_ascii(text)
+def catalan_valencia_cleaners(text):
+    """Pipeline for Catalan text, including abbreviation expansion. + punctuation + stress"""
+    # text = convert_to_ascii(text)
     text = lowercase(text)
-    #text = expand_abbreviations(text)
-    phonemes = backend.phonemize([text], strip=True)[0]
+    # text = expand_abbreviations(text)
+    phonemes = backend_val.phonemize([text], strip=True, njobs=1)[0]
     phonemes = collapse_whitespace(phonemes)
+    # print(phonemes)  # check punctuations!!
     return phonemes
+
+