update

.gitignore

@@ -165,4 +165,6 @@ cython_debug/
 filelists/*
 !/filelists/esd.list
 data/*
-/infer_save
+/infer_save
+
+.idea

README.md

@@ -1,5 +1,5 @@
 ---
-title: Umamusume Bert Vits2
+title: Bert Vits2
 emoji: 📊
 colorFrom: red
 colorTo: green

app.py

@@ -1,12 +1,17 @@
-# flake8: noqa: E402
-
-import sys, os
+import sys
 import logging
 import os
-import time
-import numpy as np  # 假设你使用NumPy来处理音频数据
-import shutil  # 用于删除文件夹和文件
-from scipy.io import wavfile
+import json
+import torch
+import argparse
+import commons
+import utils
+import gradio as gr
+
+from models import SynthesizerTrn
+from text.symbols import symbols
+from text import cleaned_text_to_sequence, get_bert
+from text.cleaner import clean_text
 
 logging.getLogger("numba").setLevel(logging.WARNING)
 logging.getLogger("markdown_it").setLevel(logging.WARNING)
@@ -18,29 +23,11 @@ logging.basicConfig(
 )
 
 logger = logging.getLogger(__name__)
+limitation = os.getenv("SYSTEM") == "spaces"  # limit text and audio length in huggingface spaces
 
-import torch
-import argparse
-import commons
-import utils
-from models import SynthesizerTrn
-from text.symbols import symbols
-from text import cleaned_text_to_sequence, get_bert
-from text.cleaner import clean_text
-import gradio as gr
-import webbrowser
-import numpy as np
-
-net_g = None
-
-if sys.platform == "darwin" and torch.backends.mps.is_available():
-    device = "mps"
-    os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
-else:
-    device = "cuda"
 
-
-def get_text(text, language_str, hps):
+def get_text(text, hps):
+    language_str = "JP"
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
 
@@ -55,15 +42,8 @@ def get_text(text, language_str, hps):
     del word2ph
     assert bert.shape[-1] == len(phone), phone
 
-    if language_str == "ZH":
-        bert = bert
-        ja_bert = torch.zeros(768, len(phone))
-    elif language_str == "JP":
-        ja_bert = bert
-        bert = torch.zeros(1024, len(phone))
-    else:
-        bert = torch.zeros(1024, len(phone))
-        ja_bert = torch.zeros(768, len(phone))
+    ja_bert = bert
+    bert = torch.zeros(1024, len(phone))
 
     assert bert.shape[-1] == len(
         phone
@@ -75,9 +55,8 @@ def get_text(text, language_str, hps):
     return bert, ja_bert, phone, tone, language
 
 
-def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
-    global net_g
-    bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
+def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, net_g_ms, hps):
+    bert, ja_bert, phones, tones, lang_ids = get_text(text, hps)
     with torch.no_grad():
         x_tst = phones.to(device).unsqueeze(0)
         tones = tones.to(device).unsqueeze(0)
@@ -85,14 +64,13 @@ def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, langua
         bert = bert.to(device).unsqueeze(0)
         ja_bert = ja_bert.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
-        #print(x_tst.type(), tones.type(), lang_ids.type(), bert.type(), ja_bert.type(), x_tst_lengths.type())
         del phones
-        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        sid = torch.LongTensor([sid]).to(device)
         audio = (
-            net_g.infer(
+            net_g_ms.infer(
                 x_tst,
                 x_tst_lengths,
-                speakers,
+                sid,
                 tones,
                 lang_ids,
                 bert,
@@ -106,108 +84,25 @@ def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, langua
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
-        torch.cuda.empty_cache()
-        return audio
-    
-def infer_2(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
-    global net_g_2
-    bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
-    with torch.no_grad():
-        x_tst = phones.to(device).unsqueeze(0)
-        tones = tones.to(device).unsqueeze(0)
-        lang_ids = lang_ids.to(device).unsqueeze(0)
-        bert = bert.to(device).unsqueeze(0)
-        ja_bert = ja_bert.to(device).unsqueeze(0)
-        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
-        #print(x_tst.type(), tones.type(), lang_ids.type(), bert.type(), ja_bert.type(), x_tst_lengths.type())
-        del phones
-        speakers = torch.LongTensor([hps_2.data.spk2id[sid]]).to(device)
-        audio = (
-            net_g_2.infer(
-                x_tst,
-                x_tst_lengths,
-                speakers,
-                tones,
-                lang_ids,
-                bert,
-                ja_bert,
-                sdp_ratio=sdp_ratio,
-                noise_scale=noise_scale,
-                noise_scale_w=noise_scale_w,
-                length_scale=length_scale,
-            )[0][0, 0]
-            .data.cpu()
-            .float()
-            .numpy()
-        )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, sid
         torch.cuda.empty_cache()
         return audio
 
-__LOG__ = "./generation_logs.txt"
-def tts_fn(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language,from_model=0):
-    # 清空 ./infer_save 文件夹
-    if os.path.exists('./infer_save'):
-        shutil.rmtree('./infer_save')
-    os.makedirs('./infer_save')
-
-    slices = text.split("\n")
-    slices = [slice for slice in slices if slice.strip() != ""]
-    audio_list = []
-    with torch.no_grad():
-        with open(__LOG__,"a",encoding="UTF-8") as f:
-            for slice in slices:
-                assert len(slice) < 150 # 限制输入的文本长度
-                if from_model == 0:
-                    audio = infer(slice, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale, sid=speaker, language=language)
-                else:
-                    audio = infer_2(slice, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale, sid=speaker, language=language)
-                audio_list.append(audio)
-                
-                # 创建唯一的文件名
-                timestamp = str(int(time.time() * 1000))
-                audio_file_path = f'./infer_save/audio_{timestamp}.wav'
-                
-                # 保存音频数据到.wav文件
-                wavfile.write(audio_file_path, hps.data.sampling_rate, audio)
-                
-                silence = np.zeros(hps.data.sampling_rate, dtype=np.int16)  # 生成1秒的静音
-                audio_list.append(silence)  # 将静音添加到列表中
-                
-                f.write(f"{slice} | {speaker}\n")
-                print(f"{slice} | {speaker}")
-
-    audio_concat = np.concatenate(audio_list)
-    return "Success", (hps.data.sampling_rate, audio_concat)
-def tts_fn_2(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language,from_model=1):
-    return  tts_fn(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language,from_model)
+def create_tts_fn(net_g_ms, hps):
+    def tts_fn(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale):
+        print(f"{text} | {speaker}")
+        sid = hps.data.spk2id[speaker]
+        text = text.replace('\n', ' ').replace('\r', '').replace(" ", "")
+        if limitation:
+            max_len = 100
+            if len(text) > max_len:
+                return "Error: Text is too long", None
+        audio = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w,
+                      length_scale=length_scale, sid=sid, net_g_ms=net_g_ms, hps=hps)
+        return "Success", (hps.data.sampling_rate, audio)
+    return tts_fn
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-m", "--model", default="./logs/natuki/G_72000.pth", help="path of your model"
-    )
-    parser.add_argument(
-        "-c",
-        "--config",
-        default="./configs/config.json",
-        help="path of your config file",
-    )
-    parser.add_argument(
-        "--share", default=False, help="make link public", action="store_true"
-    )
-    parser.add_argument(
-        "-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log"
-    )
-
-    args = parser.parse_args()
-    if args.debug:
-        logger.info("Enable DEBUG-LEVEL log")
-        logging.basicConfig(level=logging.DEBUG)
-    hps = utils.get_hparams_from_file("./logs/umamusume/config.json")
-    hps_2 = utils.get_hparams_from_file("./logs/natuki/config.json")
-
     device = (
         "cuda:0"
         if torch.cuda.is_available()
@@ -217,128 +112,57 @@ if __name__ == "__main__":
             else "cpu"
         )
     )
-    net_g = SynthesizerTrn(
-        len(symbols),
-        hps.data.filter_length // 2 + 1,
-        hps.train.segment_size // hps.data.hop_length,
-        n_speakers=hps.data.n_speakers,
-        **hps.model,
-    ).to(device)
-    _ = net_g.eval()
-
-    net_g_2 = SynthesizerTrn(
-        len(symbols),
-        hps.data.filter_length // 2 + 1,
-        hps.train.segment_size // hps.data.hop_length,
-        n_speakers=hps.data.n_speakers,
-        **hps.model,
-    ).to(device)
 
-    _ = utils.load_checkpoint("./logs/clara/G_4400.pth", net_g, None, skip_optimizer=True)
-    _ = utils.load_checkpoint("./logs/kafka/G_4000.pth", net_g_2, None, skip_optimizer=True)
-
-    speaker_ids = hps.data.spk2id
-    speakers = list(speaker_ids.keys())
-    speaker_ids_2 = hps_2.data.spk2id
-    speakers_2 = list(speaker_ids_2.keys())
-
-
-    languages = ["ZH", "JP"]
-    with gr.Blocks() as app:
-        with gr.Tab(label="umamusume"):
-            with gr.Row():
-                with gr.Column():
-                    text = gr.TextArea(
-                        label="Text",
-                        placeholder="Input Text Here",
-                        value="はりきっていこう！",
-                    )
-                    speaker = gr.Dropdown(
-                        choices=speakers, value=speakers[0], label="Speaker"
-                    )
-                    sdp_ratio = gr.Slider(
-                        minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
-                    )
-                    noise_scale = gr.Slider(
-                        minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise Scale"
-                    )
-                    noise_scale_w = gr.Slider(
-                        minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise Scale W"
-                    )
-                    length_scale = gr.Slider(
-                        minimum=0.1, maximum=2, value=1, step=0.1, label="Length Scale"
-                    )
-                    language = gr.Dropdown(
-                        choices=languages, value=languages[1], label="Language"
-                    )
-                    btn = gr.Button("Generate!", variant="primary")
-                with gr.Column():
-                    text_output = gr.Textbox(label="Message")
-                    audio_output = gr.Audio(label="Output Audio")
-                    gr.Markdown("# 赛马娘 Bert-VITS2 语音合成\n"
-                        "Project page：[GitHub](https://github.com/fishaudio/Bert-VITS2)\n"
-                        "- 本项目在日语方面有所欠缺，特别是音调的设计上，需要帮助。\n"
-                        "- このプロジェクトは、日本語の方面で不足しています。特に、音調の設計に関して助けが欲しいです。")
-
-            btn.click(
-                tts_fn,
-                inputs=[
-                    text,
-                    speaker,
-                    sdp_ratio,
-                    noise_scale,
-                    noise_scale_w,
-                    length_scale,
-                    language,
-                ],
-                outputs=[text_output, audio_output],
-            )
-        with gr.Tab(label="natuki"):
-            with gr.Row():
-                with gr.Column():
-                    text2 = gr.TextArea(
-                        label="Text",
-                        placeholder="Input Text Here",
-                        value="はりきっていこう！",
-                    )
-                    speaker2 = gr.Dropdown(
-                        choices=speakers_2, value=speakers_2[0], label="Speaker"
-                    )
-                    sdp_ratio2 = gr.Slider(
-                        minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
-                    )
-                    noise_scale2 = gr.Slider(
-                        minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise Scale"
-                    )
-                    noise_scale_w2 = gr.Slider(
-                        minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise Scale W"
-                    )
-                    length_scale2 = gr.Slider(
-                        minimum=0.1, maximum=2, value=1, step=0.1, label="Length Scale"
-                    )
-                    language2 = gr.Dropdown(
-                        choices=languages, value=languages[1], label="Language"
-                    )
-                    btn2 = gr.Button("Generate!", variant="primary")
-                with gr.Column():
-                    text_output2 = gr.Textbox(label="Message")
-                    audio_output2 = gr.Audio(label="Output Audio")
-                    gr.Markdown("# 赛马娘 Bert-VITS2 语音合成\n"
-                        "Project page：[GitHub](https://github.com/fishaudio/Bert-VITS2)\n"
-                        "- 本项目在日语方面有所欠缺，特别是音调的设计上，需要帮助。\n"
-                        "- このプロジェクトは、日本語の方面で不足しています。特に、音調の設計に関して助けが欲しいです。")
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--share", default=False, help="make link public", action="store_true")
+    parser.add_argument("-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log")
+    args = parser.parse_args()
+    if args.debug:
+        logger.info("Enable DEBUG-LEVEL log")
+        logging.basicConfig(level=logging.DEBUG)
 
-            btn2.click(
-                tts_fn_2,
-                inputs=[
-                    text2,
-                    speaker2,
-                    sdp_ratio2,
-                    noise_scale2,
-                    noise_scale_w2,
-                    length_scale2,
-                    language2,
-                ],
-                outputs=[text_output2, audio_output2],
-            )
-    app.launch()
+    models = []
+    with open("pretrained_models/info.json", "r", encoding="utf-8") as f:
+        models_info = json.load(f)
+    for i, info in models_info.items():
+        if not info['enable']:
+            continue
+        name = info['name']
+        title = info['title']
+        example = info['example']
+        hps = utils.get_hparams_from_file(f"./pretrained_models/{name}/config.json")
+        net_g_ms = SynthesizerTrn(
+            len(symbols),
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            n_speakers=hps.data.n_speakers,
+            **hps.model)
+        utils.load_checkpoint(f'pretrained_models/{i}/{i}.pth', net_g_ms, None, skip_optimizer=True)
+        _ = net_g_ms.eval().to(device)
+        models.append((name, title, example, list(hps.data.spk2id.keys()), net_g_ms, create_tts_fn(net_g_ms, hps)))
+    with gr.Blocks(theme='NoCrypt/miku') as app:
+        with gr.Tabs():
+            for (name, title, example, speakers, net_g_ms, tts_fn) in models:
+                with gr.TabItem(name):
+                    with gr.Row():
+                        gr.Markdown(
+                            '<div align="center">'
+                            f'<a><strong>{title}</strong></a>'
+                            f'</div>'
+                        )
+                    with gr.Row():
+                        with gr.Column():
+                            input_text = gr.Textbox(label="Text (100 words limitation)" if limitation else "Text", lines=5, value=example)
+                            btn = gr.Button(value="Generate", variant="primary")
+                            with gr.Row():
+                                sp = gr.Dropdown(choices=speakers, value=speakers[0], label="Speaker")
+                            with gr.Row():
+                                sdpr = gr.Slider(label="SDP Ratio", minimum=0, maximum=1, step=0.1, value=0.2)
+                                ns = gr.Slider(label="noise_scale", minimum=0.1, maximum=1.0, step=0.1, value=0.6)
+                                nsw = gr.Slider(label="noise_scale_w", minimum=0.1, maximum=1.0, step=0.1, value=0.8)
+                                ls = gr.Slider(label="length_scale", minimum=0.1, maximum=2.0, step=0.1, value=1)
+                        with gr.Column():
+                            o1 = gr.Textbox(label="Output Message")
+                            o2 = gr.Audio(label="Output Audio")
+                        btn.click(tts_fn, inputs=[input_text, sp, sdpr, ns, nsw, ls], outputs=[o1, o2])
+        app.queue(concurrency_count=1).launch(share=args.share)

pretrained_models/info.json

@@ -0,0 +1,14 @@
+{
+	"kafka": {
+		"enable": true,
+		"name": "kafka",
+        "title": "Honkai: Star Rail-カフカ",
+		"example": "嗅ぎます？この子は、特に香りもいいんです。艶があるっていうのかなぁ。とにかく、絶対に嗅いだ方がいい。ほら、どうです？"
+    },
+	"clara": {
+		"enable": true,
+		"name": "clara",
+        "title": "Honkai: Star Rail-クラーラ",
+		"example": "ーーーチャンスって何の？誰？どこから話してる？"
+    }
+}

server.py

@@ -1,170 +0,0 @@
-from flask import Flask, request, Response
-from io import BytesIO
-import torch
-from av import open as avopen
-
-import commons
-import utils
-from models import SynthesizerTrn
-from text.symbols import symbols
-from text import cleaned_text_to_sequence, get_bert
-from text.cleaner import clean_text
-from scipy.io import wavfile
-
-# Flask Init
-app = Flask(__name__)
-app.config["JSON_AS_ASCII"] = False
-
-
-def get_text(text, language_str, hps):
-    norm_text, phone, tone, word2ph = clean_text(text, language_str)
-    phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
-
-    if hps.data.add_blank:
-        phone = commons.intersperse(phone, 0)
-        tone = commons.intersperse(tone, 0)
-        language = commons.intersperse(language, 0)
-        for i in range(len(word2ph)):
-            word2ph[i] = word2ph[i] * 2
-        word2ph[0] += 1
-    bert = get_bert(norm_text, word2ph, language_str)
-    del word2ph
-    assert bert.shape[-1] == len(phone), phone
-
-    if language_str == "ZH":
-        bert = bert
-        ja_bert = torch.zeros(768, len(phone))
-    elif language_str == "JA":
-        ja_bert = bert
-        bert = torch.zeros(1024, len(phone))
-    else:
-        bert = torch.zeros(1024, len(phone))
-        ja_bert = torch.zeros(768, len(phone))
-    assert bert.shape[-1] == len(
-        phone
-    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
-    phone = torch.LongTensor(phone)
-    tone = torch.LongTensor(tone)
-    language = torch.LongTensor(language)
-    return bert, ja_bert, phone, tone, language
-
-
-def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
-    bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
-    with torch.no_grad():
-        x_tst = phones.to(dev).unsqueeze(0)
-        tones = tones.to(dev).unsqueeze(0)
-        lang_ids = lang_ids.to(dev).unsqueeze(0)
-        bert = bert.to(dev).unsqueeze(0)
-        ja_bert = ja_bert.to(device).unsqueeze(0)
-        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(dev)
-        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(dev)
-        audio = (
-            net_g.infer(
-                x_tst,
-                x_tst_lengths,
-                speakers,
-                tones,
-                lang_ids,
-                bert,
-                ja_bert,
-                sdp_ratio=sdp_ratio,
-                noise_scale=noise_scale,
-                noise_scale_w=noise_scale_w,
-                length_scale=length_scale,
-            )[0][0, 0]
-            .data.cpu()
-            .float()
-            .numpy()
-        )
-        return audio
-
-
-def replace_punctuation(text, i=2):
-    punctuation = "，。？！"
-    for char in punctuation:
-        text = text.replace(char, char * i)
-    return text
-
-
-def wav2(i, o, format):
-    inp = avopen(i, "rb")
-    out = avopen(o, "wb", format=format)
-    if format == "ogg":
-        format = "libvorbis"
-
-    ostream = out.add_stream(format)
-
-    for frame in inp.decode(audio=0):
-        for p in ostream.encode(frame):
-            out.mux(p)
-
-    for p in ostream.encode(None):
-        out.mux(p)
-
-    out.close()
-    inp.close()
-
-
-# Load Generator
-hps = utils.get_hparams_from_file("./configs/config.json")
-
-dev = "cuda"
-net_g = SynthesizerTrn(
-    len(symbols),
-    hps.data.filter_length // 2 + 1,
-    hps.train.segment_size // hps.data.hop_length,
-    n_speakers=hps.data.n_speakers,
-    **hps.model,
-).to(dev)
-_ = net_g.eval()
-
-_ = utils.load_checkpoint("logs/G_649000.pth", net_g, None, skip_optimizer=True)
-
-
-@app.route("/")
-def main():
-    try:
-        speaker = request.args.get("speaker")
-        text = request.args.get("text").replace("/n", "")
-        sdp_ratio = float(request.args.get("sdp_ratio", 0.2))
-        noise = float(request.args.get("noise", 0.5))
-        noisew = float(request.args.get("noisew", 0.6))
-        length = float(request.args.get("length", 1.2))
-        language = request.args.get("language")
-        if length >= 2:
-            return "Too big length"
-        if len(text) >= 250:
-            return "Too long text"
-        fmt = request.args.get("format", "wav")
-        if None in (speaker, text):
-            return "Missing Parameter"
-        if fmt not in ("mp3", "wav", "ogg"):
-            return "Invalid Format"
-        if language not in ("JA", "ZH"):
-            return "Invalid language"
-    except:
-        return "Invalid Parameter"
-
-    with torch.no_grad():
-        audio = infer(
-            text,
-            sdp_ratio=sdp_ratio,
-            noise_scale=noise,
-            noise_scale_w=noisew,
-            length_scale=length,
-            sid=speaker,
-            language=language,
-        )
-
-    with BytesIO() as wav:
-        wavfile.write(wav, hps.data.sampling_rate, audio)
-        torch.cuda.empty_cache()
-        if fmt == "wav":
-            return Response(wav.getvalue(), mimetype="audio/wav")
-        wav.seek(0, 0)
-        with BytesIO() as ofp:
-            wav2(wav, ofp, fmt)
-            return Response(
-                ofp.getvalue(), mimetype="audio/mpeg" if fmt == "mp3" else "audio/ogg"
-            )

text/__init__.py

@@ -19,10 +19,8 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
 
 
 def get_bert(norm_text, word2ph, language, device):
-    from .chinese_bert import get_bert_feature as zh_bert
-    from .english_bert_mock import get_bert_feature as en_bert
     from .japanese_bert import get_bert_feature as jp_bert
 
-    lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
+    lang_bert_func_map = {"JP": jp_bert}
     bert = lang_bert_func_map[language](norm_text, word2ph, device)
     return bert