multilingual model!

Data/TalkFlower_CNzh/config.json

@@ -1,96 +0,0 @@
-{
-    "train": {
-      "log_interval": 200,
-      "eval_interval": 1000,
-      "seed": 42,
-      "epochs": 1000,
-      "learning_rate": 0.0002,
-      "betas": [
-        0.8,
-        0.99
-      ],
-      "eps": 1e-09,
-      "batch_size": 12,
-      "fp16_run": false,
-      "lr_decay": 0.99995,
-      "segment_size": 16384,
-      "init_lr_ratio": 1,
-      "warmup_epochs": 0,
-      "c_mel": 45,
-      "c_kl": 1.0,
-      "skip_optimizer": true
-    },
-    "data": {
-      "training_files": "filelists/train.list",
-      "validation_files": "filelists/val.list",
-      "max_wav_value": 32768.0,
-      "sampling_rate": 44100,
-      "filter_length": 2048,
-      "hop_length": 512,
-      "win_length": 2048,
-      "n_mel_channels": 128,
-      "mel_fmin": 0.0,
-      "mel_fmax": null,
-      "add_blank": true,
-      "n_speakers": 700,
-      "cleaned_text": true,
-      "spk2id": {
-        "TalkFlower_CNzh": 0
-      }
-    },
-    "model": {
-      "use_spk_conditioned_encoder": true,
-      "use_noise_scaled_mas": true,
-      "use_mel_posterior_encoder": false,
-      "use_duration_discriminator": true,
-      "inter_channels": 192,
-      "hidden_channels": 192,
-      "filter_channels": 768,
-      "n_heads": 2,
-      "n_layers": 6,
-      "kernel_size": 3,
-      "p_dropout": 0.1,
-      "resblock": "1",
-      "resblock_kernel_sizes": [
-        3,
-        7,
-        11
-      ],
-      "resblock_dilation_sizes": [
-        [
-          1,
-          3,
-          5
-        ],
-        [
-          1,
-          3,
-          5
-        ],
-        [
-          1,
-          3,
-          5
-        ]
-      ],
-      "upsample_rates": [
-        8,
-        8,
-        2,
-        2,
-        2
-      ],
-      "upsample_initial_channel": 512,
-      "upsample_kernel_sizes": [
-        16,
-        16,
-        8,
-        2,
-        2
-      ],
-      "n_layers_q": 3,
-      "use_spectral_norm": false,
-      "gin_channels": 256
-    },
-    "version": "2.0"
-  }

app.py

@@ -7,7 +7,7 @@ from presets import *
 with gr.Blocks(css=customCSS) as demo:
     exceed_flag = gr.State(value=False)
     tmp_string = gr.Textbox(value="", visible=False)
-    character_area = gr.HTML(get_character_html("你好呀！"), elem_id="character_area")
+    character_area = gr.HTML(get_character_html("你好呀！我现在支持多语言了呢！"), elem_id="character_area")
     with gr.Tab("Speak", elem_id="tab-speak"):
         speak_input = gr.Textbox(lines=1, label="Talking Flower will say:", elem_classes="wonder-card input_text", elem_id="speak_input")
         speak_button = gr.Button("Speak!", elem_id="speak_button", elem_classes="main-button wonder-card")

config.yml

@@ -4,7 +4,7 @@
 # 拟提供通用路径配置，统一存放数据，避免数据放得很乱
 # 每个数据集与其对应的模型存放至统一路径下，后续所有的路径配置均为相对于datasetPath的路径
 # 不填或者填空则路径为相对于项目根目录的路径
-dataset_path: "Data/TalkFlower_CNzh"
+dataset_path: ""
 
 # 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
 mirror: ""
@@ -34,7 +34,7 @@ preprocess_text:
   # 验证集路径
   val_path: "filelists/val.list"
   # 配置文件路径
-  config_path: "Data/TalkFlower_CNzh/config.json"
+  config_path: "Data/config.json"
   # 每个speaker的验证集条数
   val_per_spk: 5
   # 验证集最大条数，多于的会被截断并放到训练集中
@@ -47,12 +47,12 @@ preprocess_text:
 # 注意， “:” 后需要加空格
 bert_gen:
   # 训练数据集配置文件路径
-  config_path: "Data/TalkFlower_CNzh/config.json"
+  config_path: "Data/config.json"
   # 并行数
   num_processes: 8
   # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
   # 该选项同时决定了get_bert_feature的默认设备
-  device: "cuda"
+  device: "cpu"
   # 使用多卡推理
   use_multi_device: false
 
@@ -60,11 +60,11 @@ bert_gen:
 # 注意， “:” 后需要加空格
 emo_gen:
   # 训练数据集配置文件路径
-  config_path: "Data/TalkFlower_CNzh/config.json"
+  config_path: "Data/config.json"
   # 并行数
   num_processes: 2
   # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
-  device: "cuda"
+  device: "cpu"
 
 # train 训练配置
 # 注意， “:” 后需要加空格
@@ -85,7 +85,7 @@ train_ms:
   # 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
   model: "models"
   # 配置文件路径
-  config_path: "config.json"
+  config_path: "Data/config.json"
   # 训练使用的worker，不建议超过CPU核心数
   num_workers: 16
   # 关闭此项可以节约接近50%的磁盘空间，但是可能导致实际训练速度变慢和更高的CPU使用率。
@@ -100,9 +100,9 @@ webui:
   # 推理设备
   device: "cpu"
   # 模型路径
-  model: "../../models/G_48000.pth"
+  model: "models/G_multilingual.pth"
   # 配置文件路径
-  config_path: "config.json"
+  config_path: "Data/config.json"
   # 端口号
   port: 7860
   # 是否公开部署，对外网开放
@@ -120,16 +120,16 @@ server:
   # 端口号
   port: 5000
   # 模型默认使用设备：但是当前并没有实现这个配置。
-  device: "cuda"
+  device: "cpu"
   # 需要加载的所有模型的配置
   # 注意，所有模型都必须正确配置model与config的路径，空路径会导致加载错误。
   models:
     - # 模型的路径
-      model: "models/G_48000.pth"
+      model: "models/G_multilingual.pth"
       # 模型config.json的路径
-      config: "TalkFlower_CNzh/config.json"
+      config: "Data/config.json"
       # 模型使用设备，若填写则会覆盖默认配置
-      device: "cuda"
+      device: "cpu"
       # 模型默认使用的语言
       language: "ZH"
       # 模型人物默认参数

emo_gen.py

@@ -1,19 +1,21 @@
+import argparse
+import os
+from pathlib import Path
+
+import librosa
+import numpy as np
 import torch
 import torch.nn as nn
-from torch.utils.data import Dataset
-from torch.utils.data import DataLoader
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
 from transformers import Wav2Vec2Processor
 from transformers.models.wav2vec2.modeling_wav2vec2 import (
     Wav2Vec2Model,
     Wav2Vec2PreTrainedModel,
 )
-import librosa
-import numpy as np
-import argparse
-from config import config
+
 import utils
-import os
-from tqdm import tqdm
+from config import config
 
 
 class RegressionHead(nn.Module):
@@ -78,11 +80,6 @@ class AudioDataset(Dataset):
         return torch.from_numpy(processed_data)
 
 
-model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
-processor = Wav2Vec2Processor.from_pretrained(model_name)
-model = EmotionModel.from_pretrained(model_name)
-
-
 def process_func(
     x: np.ndarray,
     sampling_rate: int,
@@ -135,16 +132,12 @@ if __name__ == "__main__":
     device = config.bert_gen_config.device
 
     model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
-    processor = (
-        Wav2Vec2Processor.from_pretrained(model_name)
-        if processor is None
-        else processor
-    )
-    model = (
-        EmotionModel.from_pretrained(model_name).to(device)
-        if model is None
-        else model.to(device)
-    )
+    REPO_ID = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+    if not Path(model_name).joinpath("pytorch_model.bin").exists():
+        utils.download_emo_models(config.mirror, model_name, REPO_ID)
+
+    processor = Wav2Vec2Processor.from_pretrained(model_name)
+    model = EmotionModel.from_pretrained(model_name).to(device)
 
     lines = []
     with open(hps.data.training_files, encoding="utf-8") as f:

infer.py

@@ -29,7 +29,7 @@ from oldVersion.V101.text import symbols as V101symbols
 from oldVersion import V111, V110, V101, V200
 
 # 当前版本信息
-latest_version = "2.0"
+latest_version = "2.1"
 
 # 版本兼容
 SynthesizerTrnMap = {
@@ -82,7 +82,7 @@ def get_net_g(model_path: str, version: str, device: str, hps):
     return net_g
 
 
-def get_text(text, reference_audio, emotion, language_str, hps, device):
+def get_text(text, language_str, hps, device):
     # 在此处实现当前版本的get_text
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -113,12 +113,6 @@ def get_text(text, reference_audio, emotion, language_str, hps, device):
     else:
         raise ValueError("language_str should be ZH, JP or EN")
 
-    emo = (
-        torch.from_numpy(get_emo(reference_audio))
-        if reference_audio
-        else torch.Tensor([emotion])
-    )
-
     assert bert.shape[-1] == len(
         phone
     ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
@@ -126,7 +120,16 @@ def get_text(text, reference_audio, emotion, language_str, hps, device):
     phone = torch.LongTensor(phone)
     tone = torch.LongTensor(tone)
     language = torch.LongTensor(language)
-    return bert, ja_bert, en_bert, emo, phone, tone, language
+    return bert, ja_bert, en_bert, phone, tone, language
+
+
+def get_emo_(reference_audio, emotion):
+    emo = (
+        torch.from_numpy(get_emo(reference_audio))
+        if reference_audio
+        else torch.Tensor([emotion])
+    )
+    return emo
 
 
 def infer(
@@ -191,9 +194,10 @@ def infer(
                 device,
             )
     # 在此处实现当前版本的推理
-    bert, ja_bert, en_bert, emo, phones, tones, lang_ids = get_text(
-        text, reference_audio, emotion, language, hps, device
+    bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+        text, language, hps, device
     )
+    emo = get_emo_(reference_audio, emotion)
     if skip_start:
         phones = phones[1:]
         tones = tones[1:]
@@ -261,10 +265,8 @@ def infer_multilang(
     skip_start=False,
     skip_end=False,
 ):
-    bert, ja_bert, en_bert, emo, phones, tones, lang_ids  = [], [], [], [], [], [], []
-    # bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
-    #     text, language, hps, device
-    # )
+    bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
+    emo = get_emo_(reference_audio, emotion)
     for idx, (txt, lang) in enumerate(zip(text, language)):
         skip_start = (idx != 0) or (skip_start and idx == 0)
         skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
@@ -272,16 +274,14 @@ def infer_multilang(
             temp_bert,
             temp_ja_bert,
             temp_en_bert,
-            temp_emo,
             temp_phones,
             temp_tones,
             temp_lang_ids,
-        ) = get_text(txt, ref, emotion, language, hps, device)
+        ) = get_text(txt, lang, hps, device)
         if skip_start:
             temp_bert = temp_bert[:, 1:]
             temp_ja_bert = temp_ja_bert[:, 1:]
             temp_en_bert = temp_en_bert[:, 1:]
-            temp_emo = temp_emo[:, 1:]
             temp_phones = temp_phones[1:]
             temp_tones = temp_tones[1:]
             temp_lang_ids = temp_lang_ids[1:]
@@ -289,21 +289,18 @@ def infer_multilang(
             temp_bert = temp_bert[:, :-1]
             temp_ja_bert = temp_ja_bert[:, :-1]
             temp_en_bert = temp_en_bert[:, :-1]
-            temp_emo = temp_emo[:, :-1]
             temp_phones = temp_phones[:-1]
             temp_tones = temp_tones[:-1]
             temp_lang_ids = temp_lang_ids[:-1]
         bert.append(temp_bert)
         ja_bert.append(temp_ja_bert)
         en_bert.append(temp_en_bert)
-        emo.append(temp_emo)
         phones.append(temp_phones)
         tones.append(temp_tones)
         lang_ids.append(temp_lang_ids)
     bert = torch.concatenate(bert, dim=1)
     ja_bert = torch.concatenate(ja_bert, dim=1)
     en_bert = torch.concatenate(en_bert, dim=1)
-    emo = torch.concatenate(emo, dim=1)
     phones = torch.concatenate(phones, dim=0)
     tones = torch.concatenate(tones, dim=0)
     lang_ids = torch.concatenate(lang_ids, dim=0)

presets.py

@@ -4,10 +4,11 @@ import numpy as np
 import torch
 import re_matching
 import utils
-from infer import infer, latest_version, get_net_g
+from infer import infer, latest_version, get_net_g, infer_multilang
 import gradio as gr
 from config import config
 from tools.webui import reload_javascript, get_character_html
+from tools.sentence import split_by_language
 
 logging.basicConfig(
     level=logging.INFO,
@@ -42,6 +43,7 @@ def speak_fn(
         interval_between_para=0.2,      # 段间间隔
         interval_between_sent=1,        # 句间间隔
     ):
+    audio_list = []
     while text.find("\n\n") != -1:
         text = text.replace("\n\n", "\n")
     if len(text) > 100:
@@ -54,58 +56,113 @@ def speak_fn(
             audio_value = "./assets/audios/overlength.wav"
         exceed_flag = not exceed_flag
     else:
-        audio_list = []
-        if len(text) > 42:
-            logging.info(f"Long Text: {text}")
-            para_list = re_matching.cut_para(text)
-            for p in para_list:
-                audio_list_sent = []
-                sent_list = re_matching.cut_sent(p)
-                for s in sent_list:
-                    audio = infer(
-                        s,
-                        sdp_ratio=sdp_ratio,
-                        noise_scale=noise_scale,
-                        noise_scale_w=noise_scale_w,
-                        length_scale=length_scale,
-                        sid=speaker,
-                        language=language,
-                        hps=hps,
-                        net_g=net_g,
-                        device=device,
-                        reference_audio=reference_audio,
-                        emotion=emotion,
-                    )
-                    audio_list_sent.append(audio)
-                    silence = np.zeros((int)(44100 * interval_between_sent))
-                    audio_list_sent.append(silence)
-                if (interval_between_para - interval_between_sent) > 0:
-                    silence = np.zeros((int)(44100 * (interval_between_para - interval_between_sent)))
-                    audio_list_sent.append(silence)
-                audio16bit = gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_list_sent))  # 对完整句子做音量归一
-                audio_list.append(audio16bit)
-        else:
-            logging.info(f"Short Text: {text}")
-            silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
-            with torch.no_grad():
-                for piece in text.split("|"):
-                    audio = infer(
-                        piece,
-                        sdp_ratio=sdp_ratio,
-                        noise_scale=noise_scale,
-                        noise_scale_w=noise_scale_w,
-                        length_scale=length_scale,
-                        sid=speaker,
-                        language=language,
-                        hps=hps,
-                        net_g=net_g,
-                        device=device,
-                        reference_audio=reference_audio,
-                        emotion=emotion,
-                    )
-                    audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
-                    audio_list.append(audio16bit)
-                    audio_list.append(silence)  # 将静音添加到列表中
+        for idx, slice in enumerate(text.split("|")):
+            if slice == "":
+                continue
+            skip_start = idx != 0
+            skip_end = idx != len(text.split("|")) - 1
+            sentences_list = split_by_language(
+                slice, target_languages=["zh", "ja", "en"]
+            )
+            idx = 0
+            while idx < len(sentences_list):
+                text_to_generate = []
+                lang_to_generate = []
+                while True:
+                    content, lang = sentences_list[idx]
+                    temp_text = [content]
+                    lang = lang.upper()
+                    if lang == "JA":
+                        lang = "JP"
+                    if len(text_to_generate) > 0:
+                        text_to_generate[-1] += [temp_text.pop(0)]
+                        lang_to_generate[-1] += [lang]
+                    if len(temp_text) > 0:
+                        text_to_generate += [[i] for i in temp_text]
+                        lang_to_generate += [[lang]] * len(temp_text)
+                    if idx + 1 < len(sentences_list):
+                        idx += 1
+                    else:
+                        break
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(sentences_list) - 1) and skip_end
+                print(text_to_generate, lang_to_generate)
+                
+                with torch.no_grad():
+                    for i, piece in enumerate(text_to_generate):
+                        skip_start = (i != 0) and skip_start
+                        skip_end = (i != len(text_to_generate) - 1) and skip_end
+                        audio = infer_multilang(
+                            piece,
+                            reference_audio=reference_audio,
+                            emotion=emotion,
+                            sdp_ratio=sdp_ratio,
+                            noise_scale=noise_scale,
+                            noise_scale_w=noise_scale_w,
+                            length_scale=length_scale,
+                            sid=speaker,
+                            language=lang_to_generate[i],
+                            hps=hps,
+                            net_g=net_g,
+                            device=device,
+                            skip_start=skip_start,
+                            skip_end=skip_end,
+                        )
+                        audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+                        audio_list.append(audio16bit)
+                idx += 1
+        # 单一语言推理
+        # if len(text) > 42:
+        #     logging.info(f"Long Text: {text}")
+        #     para_list = re_matching.cut_para(text)
+        #     for p in para_list:
+        #         audio_list_sent = []
+        #         sent_list = re_matching.cut_sent(p)
+        #         for s in sent_list:
+        #             audio = infer(
+        #                 s,
+        #                 sdp_ratio=sdp_ratio,
+        #                 noise_scale=noise_scale,
+        #                 noise_scale_w=noise_scale_w,
+        #                 length_scale=length_scale,
+        #                 sid=speaker,
+        #                 language=language,
+        #                 hps=hps,
+        #                 net_g=net_g,
+        #                 device=device,
+        #                 reference_audio=reference_audio,
+        #                 emotion=emotion,
+        #             )
+        #             audio_list_sent.append(audio)
+        #             silence = np.zeros((int)(44100 * interval_between_sent))
+        #             audio_list_sent.append(silence)
+        #         if (interval_between_para - interval_between_sent) > 0:
+        #             silence = np.zeros((int)(44100 * (interval_between_para - interval_between_sent)))
+        #             audio_list_sent.append(silence)
+        #         audio16bit = gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_list_sent))  # 对完整句子做音量归一
+        #         audio_list.append(audio16bit)
+        # else:
+        #     logging.info(f"Short Text: {text}")
+        #     silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+        #     with torch.no_grad():
+        #         for piece in text.split("|"):
+        #             audio = infer(
+        #                 piece,
+        #                 sdp_ratio=sdp_ratio,
+        #                 noise_scale=noise_scale,
+        #                 noise_scale_w=noise_scale_w,
+        #                 length_scale=length_scale,
+        #                 sid=speaker,
+        #                 language=language,
+        #                 hps=hps,
+        #                 net_g=net_g,
+        #                 device=device,
+        #                 reference_audio=reference_audio,
+        #                 emotion=emotion,
+        #             )
+        #             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+        #             audio_list.append(audio16bit)
+        #             audio_list.append(silence)  # 将静音添加到列表中
         
         audio_concat = np.concatenate(audio_list)
         audio_value = (hps.data.sampling_rate, audio_concat)
@@ -113,13 +170,15 @@ def speak_fn(
     return gr.update(value=audio_value, autoplay=True), get_character_html(text), exceed_flag, gr.update(interactive=True)
 
 
+
 def submit_lock_fn():    
     return gr.update(interactive=False)
 
 
 def init_fn():
-    gr.Info("2023-11-24: 优化长句生成效果；增加示例；更新了一些小彩蛋；画了一些大饼）")
-    gr.Info("Only support Chinese now. Trying to train a mutilingual model. 欢迎在 Community 中提建议~")
+    gr.Info("2023-11-28: 支持多语言啦！闲聊花花现在能说中、英、日语啦！")
+    # gr.Info("2023-11-24: 优化长句生成效果；增加示例；更新了一些小彩蛋；画了一些大饼）")
+    gr.Info("Support languages: ZH|EN|JA. 欢迎在 Community 中提建议~")
     
     index = random.randint(1,7)
     welcome_text = get_sentence("Welcome", index)

utils.py

@@ -9,12 +9,31 @@ import numpy as np
 from huggingface_hub import hf_hub_download
 from scipy.io.wavfile import read
 import torch
+import re
 
 MATPLOTLIB_FLAG = False
 
 logger = logging.getLogger(__name__)
 
 
+def download_emo_models(mirror, repo_id, model_name):
+    if mirror == "openi":
+        import openi
+
+        openi.model.download_model(
+            "Stardust_minus/Bert-VITS2",
+            repo_id.split("/")[-1],
+            "./emotional",
+        )
+    else:
+        hf_hub_download(
+            repo_id,
+            "pytorch_model.bin",
+            local_dir=model_name,
+            local_dir_use_symlinks=False,
+        )
+
+
 def download_checkpoint(
     dir_path, repo_config, token=None, regex="G_*.pth", mirror="openi"
 ):
@@ -385,3 +404,54 @@ class HParams:
 
     def __repr__(self):
         return self.__dict__.__repr__()
+
+
+def load_model(model_path, config_path):
+    hps = get_hparams_from_file(config_path)
+    net = SynthesizerTrn(
+        # len(symbols),
+        108,
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=hps.data.n_speakers,
+        **hps.model,
+    ).to("cpu")
+    _ = net.eval()
+    _ = load_checkpoint(model_path, net, None, skip_optimizer=True)
+    return net
+
+
+def mix_model(
+    network1, network2, output_path, voice_ratio=(0.5, 0.5), tone_ratio=(0.5, 0.5)
+):
+    if hasattr(network1, "module"):
+        state_dict1 = network1.module.state_dict()
+        state_dict2 = network2.module.state_dict()
+    else:
+        state_dict1 = network1.state_dict()
+        state_dict2 = network2.state_dict()
+    for k in state_dict1.keys():
+        if k not in state_dict2.keys():
+            continue
+        if "enc_p" in k:
+            state_dict1[k] = (
+                state_dict1[k].clone() * tone_ratio[0]
+                + state_dict2[k].clone() * tone_ratio[1]
+            )
+        else:
+            state_dict1[k] = (
+                state_dict1[k].clone() * voice_ratio[0]
+                + state_dict2[k].clone() * voice_ratio[1]
+            )
+    for k in state_dict2.keys():
+        if k not in state_dict1.keys():
+            state_dict1[k] = state_dict2[k].clone()
+    torch.save(
+        {"model": state_dict1, "iteration": 0, "optimizer": None, "learning_rate": 0},
+        output_path,
+    )
+
+
+def get_steps(model_path):
+    matches = re.findall(r"\d+", model_path)
+    return matches[-1] if matches else None