diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..e11490ce780dd062e795006d5979badad0e8b739 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,40 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +*.index filter=lfs diff=lfs merge=lfs -text +model/macmiller/macmiller.png filter=lfs diff=lfs merge=lfs -text +model/hamza/Hamza.png filter=lfs diff=lfs merge=lfs -text +model/gambino/Hamza.png filter=lfs diff=lfs merge=lfs -text +model/angele/Angele.png filter=lfs diff=lfs merge=lfs -text +model/leto/Leto.png filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..8bbcbdad1f797051187c5b916a79e9cdc253e0f8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,51 @@ +MIT License + +Copyright (c) 2023 liujing04 +Copyright (c) 2023 源文雨 +Copyright (c) 2023 on9.moe Webslaves + + 本软件及其相关代码以MIT协议开源，作者不对软件具备任何控制力，使用软件者、传播软件导出的声音者自负全责。 + 如不认可该条款，则不能使用或引用软件包内任何代码和文件。 + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +特此授予任何获得本软件和相关文档文件（以下简称“软件”）副本的人免费使用、复制、修改、合并、出版、分发、再授权和/或销售本软件的权利，以及授予本软件所提供的人使用本软件的权利，但须符合以下条件： +上述版权声明和本许可声明应包含在软件的所有副本或实质部分中。 +软件是“按原样”提供的，没有任何明示或暗示的保证，包括但不限于适销性、适用于特定目的和不侵权的保证。在任何情况下，作者或版权持有人均不承担因软件或软件的使用或其他交易而产生、产生或与之相关的任何索赔、损害赔偿或其他责任，无论是在合同诉讼、侵权诉讼还是其他诉讼中。 + +相关引用库协议如下： +################# +ContentVec +https://github.com/auspicious3000/contentvec/blob/main/LICENSE +MIT License +################# +VITS +https://github.com/jaywalnut310/vits/blob/main/LICENSE +MIT License +################# +HIFIGAN +https://github.com/jik876/hifi-gan/blob/master/LICENSE +MIT License +################# +gradio +https://github.com/gradio-app/gradio/blob/main/LICENSE +Apache License 2.0 +################# +ffmpeg +https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 +https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2021-02-28-12-32/ffmpeg-n4.3.2-160-gfbb9368226-win64-lgpl-4.3.zip +LPGLv3 License +MIT License +################# +ultimatevocalremovergui +https://github.com/Anjok07/ultimatevocalremovergui/blob/master/LICENSE +https://github.com/yang123qwe/vocal_separation_by_uvr5 +MIT License +################# +audio-slicer +https://github.com/openvpi/audio-slicer/blob/main/LICENSE +MIT License diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fa0b037d0b2a2673a6b6aef214a4c4cc0740afc8 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +title: RubinAudiov2 +emoji: 🗣 +colorFrom: red +colorTo: purple +sdk: gradio +sdk_version: 3.28.3 +app_file: app_multi.py +pinned: true +license: mit +duplicated_from: RubinAudio/RubinAudiov2 +--- diff --git a/app_multi MODELS LOAD.py b/app_multi MODELS LOAD.py new file mode 100644 index 0000000000000000000000000000000000000000..13b49358ea51729e41429b2b529255bbfabcf087 --- /dev/null +++ b/app_multi MODELS LOAD.py @@ -0,0 +1,368 @@ +from typing import Union + +from argparse import ArgumentParser + +import faiss +import asyncio +import json +import hashlib +from os import path, getenv + +import gradio as gr + +import torch + +import numpy as np +import librosa + +import config # Import the whole config module +import util +from infer_pack.models import ( + SynthesizerTrnMs768NSFsid, + SynthesizerTrnMs768NSFsid_nono +) +from infer_pack.models import ( + SynthesizerTrnMs256NSFsid, + SynthesizerTrnMs256NSFsid_nono +) +from vc_infer_pipeline import VC + +# Function to check if the script is running on Hugging Face Spaces +def is_huggingface_spaces() -> bool: + return getenv('SYSTEM') == 'spaces' +in_hf_space = getenv('SYSTEM') == 'spaces' + + +# Argument parsing +arg_parser = ArgumentParser() +arg_parser.add_argument( + '--hubert', + default=getenv('RVC_HUBERT', 'hubert_base.pt'), + help='path to hubert base model (default: hubert_base.pt)' +) +arg_parser.add_argument( + '--config', + default=getenv('RVC_MULTI_CFG', 'multi_config.json'), + help='path to config file (default: multi_config.json)' +) +arg_parser.add_argument( + '--api', + action='store_true', + help='enable api endpoint' +) +arg_parser.add_argument( + '--cache-examples', + action='store_true', + help='enable example caching, please remember delete gradio_cached_examples folder when example config has been modified' # noqa +) +args = arg_parser.parse_args() + +app_css = ''' +#model_info img { + max-width: 100px; + max-height: 100px; + float: right; +} +#model_info p { + margin: unset; +} +''' + +app = gr.Blocks( + theme=gr.themes.Monochrome(primary_hue="blue", secondary_hue="slate"), + css=app_css, + analytics_enabled=False +) + +# Load hubert model +hubert_model = util.load_hubert_model(config.device, args.hubert) +hubert_model.eval() + +# Load models +multi_cfg = json.load(open(args.config, 'r')) +loaded_models = [] + +for model_name in multi_cfg.get('models'): + print(f'Loading model: {model_name}') + + # Load model info + model_info = json.load( + open(path.join('model', model_name, 'config.json'), 'r') + ) + + # Load RVC checkpoint + cpt = torch.load( + path.join('model', model_name, model_info['model']), + map_location='cpu' + ) + tgt_sr = cpt['config'][-1] + cpt['config'][-3] = cpt['weight']['emb_g.weight'].shape[0] # n_spk + + if_f0 = cpt.get('f0', 1) + # Check the dimension of the 'enc_p.emb_phone.weight' tensor + emb_phone_weight_size = cpt['weight']['enc_p.emb_phone.weight'].shape[1] + if emb_phone_weight_size == 768: + if if_f0 == 1: + net_g = SynthesizerTrnMs768NSFsid( + *cpt['config'], + is_half=util.is_half(config.device) + ) + else: + net_g = SynthesizerTrnMs768NSFsid_nono(*cpt['config']) + elif emb_phone_weight_size == 256: + if if_f0 == 1: + net_g = SynthesizerTrnMs256NSFsid( + *cpt['config'], + is_half=util.is_half(config.device) + ) + else: + net_g = SynthesizerTrnMs256NSFsid_nono(*cpt['config']) + else: + raise ValueError(f"Unexpected emb_phone_weight_size: {emb_phone_weight_size}") + + del net_g.enc_q + + # According to original code, this thing seems necessary. + print(net_g.load_state_dict(cpt['weight'], strict=False)) + + net_g.eval().to(config.device) + net_g = net_g.half() if util.is_half(config.device) else net_g.float() + + vc = VC(tgt_sr, config) + + loaded_models.append(dict( + name=model_name, + metadata=model_info, + vc=vc, + net_g=net_g, + if_f0=if_f0, + target_sr=tgt_sr + )) + +print(f'Models loaded: {len(loaded_models)}') + +# https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI/blob/main/infer-web.py#L118 # noqa +def vc_func( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option +): + if input_audio is None: + return (None, 'Please provide input audio.') + + if model_index is None: + return (None, 'Please select a model.') + + model = loaded_models[model_index] + + # Reference: so-vits + (audio_samp, audio_npy) = input_audio + + # https://huggingface.co/spaces/zomehwh/rvc-models/blob/main/app.py#L49 + # Can be change well, we will see + if (audio_npy.shape[0] / audio_samp) > 5000 and in_hf_space: + return (None, 'Input audio is longer than 5 mins.') + + # Bloody hell: https://stackoverflow.com/questions/26921836/ + if audio_npy.dtype != np.float32: # :thonk: + audio_npy = ( + audio_npy / np.iinfo(audio_npy.dtype).max + ).astype(np.float32) + + if len(audio_npy.shape) > 1: + audio_npy = librosa.to_mono(audio_npy.transpose(1, 0)) + + if audio_samp != 16000: + audio_npy = librosa.resample( + audio_npy, + orig_sr=audio_samp, + target_sr=16000 + ) + + pitch_int = int(pitch_adjust) + + resample = ( + 0 if resample_option == 'Disable resampling' + else int(resample_option) + ) + + times = [0, 0, 0] + + checksum = hashlib.sha512() + checksum.update(audio_npy.tobytes()) + + output_audio = model['vc'].pipeline( + hubert_model, + model['net_g'], + model['metadata'].get('speaker_id', 0), + audio_npy, + checksum.hexdigest(), + times, + pitch_int, + f0_method, + path.join('model', model['name'], model['metadata']['feat_index']), + feat_ratio, + model['if_f0'], + filter_radius, + model['target_sr'], + resample, + rms_mix_rate, + 'v2' + ) + + out_sr = ( + resample if resample >= 16000 and model['target_sr'] != resample + else model['target_sr'] + ) + + print(f'npy: {times[0]}s, f0: {times[1]}s, infer: {times[2]}s') + return ((out_sr, output_audio), 'Success') + +def update_model_info(model_index): + if model_index is None: + return str( + '### Model info\n' + 'Please select a model from dropdown above.' + ) + + model = loaded_models[model_index] + model_icon = model['metadata'].get('icon', '') + + return str( + '### Model info\n' + '![model icon]({icon})' + '**{name}**\n\n' + 'Author: {author}\n\n' + 'Source: {source}\n\n' + '{note}' + ).format( + name=model['metadata'].get('name'), + author=model['metadata'].get('author', 'Anonymous'), + source=model['metadata'].get('source', 'Unknown'), + note=model['metadata'].get('note', ''), + icon=( + model_icon + if model_icon.startswith(('http://', 'https://')) + else '/file/model/%s/%s' % (model['name'], model_icon) + ) + ) + + +def _example_vc( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option +): + (audio, message) = vc_func( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option + ) + return ( + audio, + message, + update_model_info(model_index) + ) + +with app: + gr.HTML('
') + gr.HTML('

Rubin.Audio webUI

') + gr.Markdown( + '
Please visit Rubin.audio for more information
' + 'Upload clean audio with no effects or layered vocals.
' + ) + with gr.Row(): + with gr.Column(): + input_audio = gr.Audio(label='Input audio') + output_audio = gr.Audio(label='Output audio') + with gr.Box(): + model_info = gr.Markdown( + '### Model info\n' + 'Please select a model from dropdown.', + elem_id='model_info' + ) + model_index = gr.Dropdown( + [ + '%s - %s' % ( + m['metadata'].get('source', 'Unknown'), + m['metadata'].get('name') + ) + for m in loaded_models + ], + label='Model', + type='index' + ) + pitch_adjust = gr.Slider( + label='Pitch', + #elem_id="slider1", + minimum=-24, + maximum=24, + step=1, + value=0 + ) + with gr.Accordion('Advanced options', open=False): + f0_method = gr.Radio( + label='Render Algo', + choices=['harvest', 'crepe', 'pm'], + value='harvest', + interactive=True + ) + feat_ratio = gr.Slider( + label='Feature ratio', + minimum=0, + maximum=1, + step=0.1, + value=0.6 + ) + filter_radius = gr.Slider( + label='Filter radius', + minimum=0, + maximum=7, + step=1, + value=3 + ) + rms_mix_rate = gr.Slider( + label='Volume envelope mix rate', + minimum=0, + maximum=1, + step=0.1, + value=1 + ) + resample_rate = gr.Dropdown( + [ + 'Disable resampling', + '16000', + '22050', + '44100', + '48000' + ], + label='Resample rate', + value='Disable resampling' + ) + vc_convert_btn = gr.Button('Convert', variant='primary') + output_msg = gr.Textbox(label='Output message') + + vc_convert_btn.click( + vc_func, + [ + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_rate + ], + [output_audio, output_msg], + api_name='audio_conversion' + ) + + + model_index.change( + update_model_info, + inputs=[model_index], + outputs=[model_info], + show_progress=False, + queue=False + ) + +if is_huggingface_spaces(): + app.launch() +else: + app.queue( + concurrency_count=1, + max_size=20, + api_open=False + ).launch(share=True) diff --git a/app_multi.py b/app_multi.py new file mode 100644 index 0000000000000000000000000000000000000000..7bab7072021cc9e78531eb965027f62bfe694fde --- /dev/null +++ b/app_multi.py @@ -0,0 +1,389 @@ +from typing import Union + +from argparse import ArgumentParser + +import faiss +import asyncio +import json +import hashlib +from os import path, getenv + +import gradio as gr + +import torch + +import numpy as np +import librosa + +import config # Import the whole config module +import util +from infer_pack.models import ( + SynthesizerTrnMs768NSFsid, + SynthesizerTrnMs768NSFsid_nono +) +from infer_pack.models import ( + SynthesizerTrnMs256NSFsid, + SynthesizerTrnMs256NSFsid_nono +) +from vc_infer_pipeline import VC + +# Function to check if the script is running on Hugging Face Spaces +def is_huggingface_spaces() -> bool: + return getenv('SYSTEM') == 'spaces' +in_hf_space = getenv('SYSTEM') == 'spaces' + + +# Argument parsing +arg_parser = ArgumentParser() +arg_parser.add_argument( + '--hubert', + default=getenv('RVC_HUBERT', 'hubert_base.pt'), + help='path to hubert base model (default: hubert_base.pt)' +) +arg_parser.add_argument( + '--config', + default=getenv('RVC_MULTI_CFG', 'multi_config.json'), + help='path to config file (default: multi_config.json)' +) +arg_parser.add_argument( + '--api', + action='store_true', + help='enable api endpoint' +) +arg_parser.add_argument( + '--cache-examples', + action='store_true', + help='enable example caching, please remember delete gradio_cached_examples folder when example config has been modified' # noqa +) +args = arg_parser.parse_args() + +app_css = ''' +#model_info img { + max-width: 100px; + max-height: 100px; + float: right; +} +#model_info p { + margin: unset; +} +''' + +app = gr.Blocks( + theme=gr.themes.Monochrome(primary_hue="blue", secondary_hue="slate"), + css=app_css, + analytics_enabled=False +) + +# Load hubert model +hubert_model = util.load_hubert_model(config.device, args.hubert) +hubert_model.eval() + +# Load models +multi_cfg = json.load(open(args.config, 'r')) +loaded_models = [] + +for model_name in multi_cfg.get('models'): + print(f'Loading model: {model_name}') + + # Load model info + model_info = json.load( + open(path.join('model', model_name, 'config.json'), 'r') + ) + + # Load RVC checkpoint + cpt = torch.load( + path.join('model', model_name, model_info['model']), + map_location='cpu' + ) + + print(cpt.keys()) + + # If your model checkpoint is not too large, this line can be useful for understanding the structure of your model. + # If the output is too large to be useful, you might want to remove or comment it out. + for key, value in cpt.items(): + print(f"{key}: {type(value)}") # This will print out the type of each item in the cpt dictionary. + + # Check if the 'config' key exists before trying to access it + if 'config' in cpt: + tgt_sr = cpt['config'][-1] + cpt['config'][-3] = cpt['weight']['emb_g.weight'].shape[0] # n_spk + else: + print(f"Warning: Model {model_name} does not have a 'config' key. Skipping this model.") + continue + + if_f0 = cpt.get('f0', 1) + # Check the dimension of the 'enc_p.emb_phone.weight' tensor + emb_phone_weight_size = cpt['weight']['enc_p.emb_phone.weight'].shape[1] + if emb_phone_weight_size == 768: + if if_f0 == 1: + net_g = SynthesizerTrnMs768NSFsid( + *cpt['config'], + is_half=util.is_half(config.device) + ) + else: + net_g = SynthesizerTrnMs768NSFsid_nono(*cpt['config']) + elif emb_phone_weight_size == 256: + if if_f0 == 1: + net_g = SynthesizerTrnMs256NSFsid( + *cpt['config'], + is_half=util.is_half(config.device) + ) + else: + net_g = SynthesizerTrnMs256NSFsid_nono(*cpt['config']) + else: + raise ValueError(f"Unexpected emb_phone_weight_size: {emb_phone_weight_size}") + + del net_g.enc_q + + # According to original code, this thing seems necessary. + print(net_g.load_state_dict(cpt['weight'], strict=False)) + + net_g.eval().to(config.device) + net_g = net_g.half() if util.is_half(config.device) else net_g.float() + + vc = VC(tgt_sr, config) + + loaded_models.append(dict( + name=model_name, + metadata=model_info, + vc=vc, + net_g=net_g, + if_f0=if_f0, + target_sr=tgt_sr + )) + +print(f'Models loaded: {len(loaded_models)}') + +# https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI/blob/main/infer-web.py#L118 # noqa +def vc_func( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option +): + if input_audio is None: + return (None, 'Please provide input audio.') + + if model_index is None: + return (None, 'Please select a model.') + + model = loaded_models[model_index] + + # Reference: so-vits + (audio_samp, audio_npy) = input_audio + + # https://huggingface.co/spaces/zomehwh/rvc-models/blob/main/app.py#L49 + # Can be change well, we will see + if (audio_npy.shape[0] / audio_samp) > 5000 and in_hf_space: + return (None, 'Input audio is longer than 5 mins.') + + # Bloody hell: https://stackoverflow.com/questions/26921836/ + if audio_npy.dtype != np.float32: # :thonk: + audio_npy = ( + audio_npy / np.iinfo(audio_npy.dtype).max + ).astype(np.float32) + + if len(audio_npy.shape) > 1: + audio_npy = librosa.to_mono(audio_npy.transpose(1, 0)) + + if audio_samp != 16000: + audio_npy = librosa.resample( + audio_npy, + orig_sr=audio_samp, + target_sr=16000 + ) + + pitch_int = int(pitch_adjust) + + resample = ( + 0 if resample_option == 'Disable resampling' + else int(resample_option) + ) + + times = [0, 0, 0] + + checksum = hashlib.sha512() + checksum.update(audio_npy.tobytes()) + + output_audio = model['vc'].pipeline( + hubert_model, + model['net_g'], + model['metadata'].get('speaker_id', 0), + audio_npy, + checksum.hexdigest(), + times, + pitch_int, + f0_method, + path.join('model', model['name'], model['metadata']['feat_index']), + feat_ratio, + model['if_f0'], + filter_radius, + model['target_sr'], + resample, + rms_mix_rate, + 'v2' + ) + + out_sr = ( + resample if resample >= 16000 and model['target_sr'] != resample + else model['target_sr'] + ) + + print(f'npy: {times[0]}s, f0: {times[1]}s, infer: {times[2]}s') + return ((out_sr, output_audio), 'Success') + +def update_model_info(model_index): + if model_index is None: + return str( + '### Model info\n' + 'Please select a model from dropdown above.' + ) + + model = loaded_models[model_index] + model_icon = model['metadata'].get('icon', '') + + return str( + '### Model info\n' + '![model icon]({icon})' + '**{name}**\n\n' + 'Author: {author}\n\n' + 'Source: {source}\n\n' + '{note}' + ).format( + name=model['metadata'].get('name'), + author=model['metadata'].get('author', 'Anonymous'), + source=model['metadata'].get('source', 'Unknown'), + note=model['metadata'].get('note', ''), + icon=( + model_icon + if model_icon.startswith(('http://', 'https://')) + else '/file/model/%s/%s' % (model['name'], model_icon) + ) + ) + + +def _example_vc( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option +): + (audio, message) = vc_func( + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_option + ) + return ( + audio, + message, + update_model_info(model_index) + ) + +with app: + gr.HTML('
') + gr.HTML('

Rubin.Audio EXPERIMENTAL!!

') + gr.Markdown( + '
Please visit Rubin.audio for more information
' + 'Upload clean audio with no effects or layered vocals.
' + ) + with gr.Accordion('Instructions', open=False): + gr.Markdown( + '- For faster results upload a vocal that is under 1 minute in length.\n' + '- Use dry, effect-free vocals for optimal results.\n' + '- On mobile, utilize your iPhone\'s video feature for quick audio recording and voice trials.\n' + '- For transitioning from male to female voice, adjust +12 semitones. For female to male, adjust -12 semitones.\n' + '- Harvest algo is better sounding, PM is faster\n' + ) + with gr.Row(): + with gr.Column(): + input_audio = gr.Audio(label='Input audio') + output_audio = gr.Audio(label='Output audio') + with gr.Box(): + model_info = gr.Markdown( + '### Model info\n' + 'Please select a model from dropdown.', + elem_id='model_info' + ) + model_index = gr.Dropdown( + [ + '%s - %s' % ( + m['metadata'].get('source', 'Unknown'), + m['metadata'].get('name') + ) + for m in loaded_models + ], + label='Model', + type='index' + ) + pitch_adjust = gr.Slider( + label='Pitch', + #elem_id="slider1", + minimum=-24, + maximum=24, + step=1, + value=0 + ) + with gr.Accordion('Advanced options', open=False): + f0_method = gr.Radio( + label='Render Algo', + choices=['harvest', 'crepe', 'pm'], + value='harvest', + interactive=True + ) + feat_ratio = gr.Slider( + label='Feature ratio', + minimum=0, + maximum=1, + step=0.1, + value=0.6 + ) + filter_radius = gr.Slider( + label='Filter radius', + minimum=0, + maximum=7, + step=1, + value=3 + ) + rms_mix_rate = gr.Slider( + label='Volume envelope mix rate', + minimum=0, + maximum=1, + step=0.1, + value=1 + ) + resample_rate = gr.Dropdown( + [ + 'Disable resampling', + '16000', + '22050', + '44100', + '48000' + ], + label='Resample rate', + value='Disable resampling' + ) + vc_convert_btn = gr.Button('Convert', variant='primary') + output_msg = gr.Textbox(label='Output message') + + vc_convert_btn.click( + vc_func, + [ + input_audio, model_index, pitch_adjust, f0_method, feat_ratio, + filter_radius, rms_mix_rate, resample_rate + ], + [output_audio, output_msg], + api_name='audio_conversion' + ) + + + model_index.change( + update_model_info, + inputs=[model_index], + outputs=[model_info], + show_progress=False, + queue=False + ) + +if is_huggingface_spaces(): + app.launch() +else: + app.queue( + concurrency_count=1, + max_size=20, + api_open=False + ).launch(share=True) diff --git a/config.py b/config.py new file mode 100644 index 0000000000000000000000000000000000000000..e07d93cf81ea0d72ffe318cc37bc1064bc94533b --- /dev/null +++ b/config.py @@ -0,0 +1,17 @@ +import torch + +import util + +device = ( + 'cuda:0' if torch.cuda.is_available() + else ( + 'mps' if util.has_mps() + else 'cpu' + ) +) +is_half = util.is_half(device) + +x_pad = 3 if is_half else 1 +x_query = 10 if is_half else 6 +x_center = 60 if is_half else 38 +x_max = 65 if is_half else 41 diff --git a/hubert_base.pt b/hubert_base.pt new file mode 100644 index 0000000000000000000000000000000000000000..72f47ab58564f01d5cc8b05c63bdf96d944551ff --- /dev/null +++ b/hubert_base.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96 +size 189507909 diff --git a/infer_pack/attentions.py b/infer_pack/attentions.py new file mode 100644 index 0000000000000000000000000000000000000000..77cb63ffccf3e33badf22d50862a64ba517b487f --- /dev/null +++ b/infer_pack/attentions.py @@ -0,0 +1,417 @@ +import copy +import math +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from infer_pack import commons +from infer_pack import modules +from infer_pack.modules import LayerNorm + + +class Encoder(nn.Module): + def __init__( + self, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size=1, + p_dropout=0.0, + window_size=10, + **kwargs + ): + super().__init__() + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.window_size = window_size + + self.drop = nn.Dropout(p_dropout) + self.attn_layers = nn.ModuleList() + self.norm_layers_1 = nn.ModuleList() + self.ffn_layers = nn.ModuleList() + self.norm_layers_2 = nn.ModuleList() + for i in range(self.n_layers): + self.attn_layers.append( + MultiHeadAttention( + hidden_channels, + hidden_channels, + n_heads, + p_dropout=p_dropout, + window_size=window_size, + ) + ) + self.norm_layers_1.append(LayerNorm(hidden_channels)) + self.ffn_layers.append( + FFN( + hidden_channels, + hidden_channels, + filter_channels, + kernel_size, + p_dropout=p_dropout, + ) + ) + self.norm_layers_2.append(LayerNorm(hidden_channels)) + + def forward(self, x, x_mask): + attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) + x = x * x_mask + for i in range(self.n_layers): + y = self.attn_layers[i](x, x, attn_mask) + y = self.drop(y) + x = self.norm_layers_1[i](x + y) + + y = self.ffn_layers[i](x, x_mask) + y = self.drop(y) + x = self.norm_layers_2[i](x + y) + x = x * x_mask + return x + + +class Decoder(nn.Module): + def __init__( + self, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size=1, + p_dropout=0.0, + proximal_bias=False, + proximal_init=True, + **kwargs + ): + super().__init__() + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.proximal_bias = proximal_bias + self.proximal_init = proximal_init + + self.drop = nn.Dropout(p_dropout) + self.self_attn_layers = nn.ModuleList() + self.norm_layers_0 = nn.ModuleList() + self.encdec_attn_layers = nn.ModuleList() + self.norm_layers_1 = nn.ModuleList() + self.ffn_layers = nn.ModuleList() + self.norm_layers_2 = nn.ModuleList() + for i in range(self.n_layers): + self.self_attn_layers.append( + MultiHeadAttention( + hidden_channels, + hidden_channels, + n_heads, + p_dropout=p_dropout, + proximal_bias=proximal_bias, + proximal_init=proximal_init, + ) + ) + self.norm_layers_0.append(LayerNorm(hidden_channels)) + self.encdec_attn_layers.append( + MultiHeadAttention( + hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout + ) + ) + self.norm_layers_1.append(LayerNorm(hidden_channels)) + self.ffn_layers.append( + FFN( + hidden_channels, + hidden_channels, + filter_channels, + kernel_size, + p_dropout=p_dropout, + causal=True, + ) + ) + self.norm_layers_2.append(LayerNorm(hidden_channels)) + + def forward(self, x, x_mask, h, h_mask): + """ + x: decoder input + h: encoder output + """ + self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to( + device=x.device, dtype=x.dtype + ) + encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1) + x = x * x_mask + for i in range(self.n_layers): + y = self.self_attn_layers[i](x, x, self_attn_mask) + y = self.drop(y) + x = self.norm_layers_0[i](x + y) + + y = self.encdec_attn_layers[i](x, h, encdec_attn_mask) + y = self.drop(y) + x = self.norm_layers_1[i](x + y) + + y = self.ffn_layers[i](x, x_mask) + y = self.drop(y) + x = self.norm_layers_2[i](x + y) + x = x * x_mask + return x + + +class MultiHeadAttention(nn.Module): + def __init__( + self, + channels, + out_channels, + n_heads, + p_dropout=0.0, + window_size=None, + heads_share=True, + block_length=None, + proximal_bias=False, + proximal_init=False, + ): + super().__init__() + assert channels % n_heads == 0 + + self.channels = channels + self.out_channels = out_channels + self.n_heads = n_heads + self.p_dropout = p_dropout + self.window_size = window_size + self.heads_share = heads_share + self.block_length = block_length + self.proximal_bias = proximal_bias + self.proximal_init = proximal_init + self.attn = None + + self.k_channels = channels // n_heads + self.conv_q = nn.Conv1d(channels, channels, 1) + self.conv_k = nn.Conv1d(channels, channels, 1) + self.conv_v = nn.Conv1d(channels, channels, 1) + self.conv_o = nn.Conv1d(channels, out_channels, 1) + self.drop = nn.Dropout(p_dropout) + + if window_size is not None: + n_heads_rel = 1 if heads_share else n_heads + rel_stddev = self.k_channels**-0.5 + self.emb_rel_k = nn.Parameter( + torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) + * rel_stddev + ) + self.emb_rel_v = nn.Parameter( + torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) + * rel_stddev + ) + + nn.init.xavier_uniform_(self.conv_q.weight) + nn.init.xavier_uniform_(self.conv_k.weight) + nn.init.xavier_uniform_(self.conv_v.weight) + if proximal_init: + with torch.no_grad(): + self.conv_k.weight.copy_(self.conv_q.weight) + self.conv_k.bias.copy_(self.conv_q.bias) + + def forward(self, x, c, attn_mask=None): + q = self.conv_q(x) + k = self.conv_k(c) + v = self.conv_v(c) + + x, self.attn = self.attention(q, k, v, mask=attn_mask) + + x = self.conv_o(x) + return x + + def attention(self, query, key, value, mask=None): + # reshape [b, d, t] -> [b, n_h, t, d_k] + b, d, t_s, t_t = (*key.size(), query.size(2)) + query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) + key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + + scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) + if self.window_size is not None: + assert ( + t_s == t_t + ), "Relative attention is only available for self-attention." + key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) + rel_logits = self._matmul_with_relative_keys( + query / math.sqrt(self.k_channels), key_relative_embeddings + ) + scores_local = self._relative_position_to_absolute_position(rel_logits) + scores = scores + scores_local + if self.proximal_bias: + assert t_s == t_t, "Proximal bias is only available for self-attention." + scores = scores + self._attention_bias_proximal(t_s).to( + device=scores.device, dtype=scores.dtype + ) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e4) + if self.block_length is not None: + assert ( + t_s == t_t + ), "Local attention is only available for self-attention." + block_mask = ( + torch.ones_like(scores) + .triu(-self.block_length) + .tril(self.block_length) + ) + scores = scores.masked_fill(block_mask == 0, -1e4) + p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] + p_attn = self.drop(p_attn) + output = torch.matmul(p_attn, value) + if self.window_size is not None: + relative_weights = self._absolute_position_to_relative_position(p_attn) + value_relative_embeddings = self._get_relative_embeddings( + self.emb_rel_v, t_s + ) + output = output + self._matmul_with_relative_values( + relative_weights, value_relative_embeddings + ) + output = ( + output.transpose(2, 3).contiguous().view(b, d, t_t) + ) # [b, n_h, t_t, d_k] -> [b, d, t_t] + return output, p_attn + + def _matmul_with_relative_values(self, x, y): + """ + x: [b, h, l, m] + y: [h or 1, m, d] + ret: [b, h, l, d] + """ + ret = torch.matmul(x, y.unsqueeze(0)) + return ret + + def _matmul_with_relative_keys(self, x, y): + """ + x: [b, h, l, d] + y: [h or 1, m, d] + ret: [b, h, l, m] + """ + ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) + return ret + + def _get_relative_embeddings(self, relative_embeddings, length): + max_relative_position = 2 * self.window_size + 1 + # Pad first before slice to avoid using cond ops. + pad_length = max(length - (self.window_size + 1), 0) + slice_start_position = max((self.window_size + 1) - length, 0) + slice_end_position = slice_start_position + 2 * length - 1 + if pad_length > 0: + padded_relative_embeddings = F.pad( + relative_embeddings, + commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]), + ) + else: + padded_relative_embeddings = relative_embeddings + used_relative_embeddings = padded_relative_embeddings[ + :, slice_start_position:slice_end_position + ] + return used_relative_embeddings + + def _relative_position_to_absolute_position(self, x): + """ + x: [b, h, l, 2*l-1] + ret: [b, h, l, l] + """ + batch, heads, length, _ = x.size() + # Concat columns of pad to shift from relative to absolute indexing. + x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]])) + + # Concat extra elements so to add up to shape (len+1, 2*len-1). + x_flat = x.view([batch, heads, length * 2 * length]) + x_flat = F.pad( + x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]]) + ) + + # Reshape and slice out the padded elements. + x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[ + :, :, :length, length - 1 : + ] + return x_final + + def _absolute_position_to_relative_position(self, x): + """ + x: [b, h, l, l] + ret: [b, h, l, 2*l-1] + """ + batch, heads, length, _ = x.size() + # padd along column + x = F.pad( + x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]]) + ) + x_flat = x.view([batch, heads, length**2 + length * (length - 1)]) + # add 0's in the beginning that will skew the elements after reshape + x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]])) + x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:] + return x_final + + def _attention_bias_proximal(self, length): + """Bias for self-attention to encourage attention to close positions. + Args: + length: an integer scalar. + Returns: + a Tensor with shape [1, 1, length, length] + """ + r = torch.arange(length, dtype=torch.float32) + diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) + return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) + + +class FFN(nn.Module): + def __init__( + self, + in_channels, + out_channels, + filter_channels, + kernel_size, + p_dropout=0.0, + activation=None, + causal=False, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.activation = activation + self.causal = causal + + if causal: + self.padding = self._causal_padding + else: + self.padding = self._same_padding + + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) + self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) + self.drop = nn.Dropout(p_dropout) + + def forward(self, x, x_mask): + x = self.conv_1(self.padding(x * x_mask)) + if self.activation == "gelu": + x = x * torch.sigmoid(1.702 * x) + else: + x = torch.relu(x) + x = self.drop(x) + x = self.conv_2(self.padding(x * x_mask)) + return x * x_mask + + def _causal_padding(self, x): + if self.kernel_size == 1: + return x + pad_l = self.kernel_size - 1 + pad_r = 0 + padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad(x, commons.convert_pad_shape(padding)) + return x + + def _same_padding(self, x): + if self.kernel_size == 1: + return x + pad_l = (self.kernel_size - 1) // 2 + pad_r = self.kernel_size // 2 + padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad(x, commons.convert_pad_shape(padding)) + return x diff --git a/infer_pack/commons.py b/infer_pack/commons.py new file mode 100644 index 0000000000000000000000000000000000000000..54470986f37825b35d90d7efa7437d1c26b87215 --- /dev/null +++ b/infer_pack/commons.py @@ -0,0 +1,166 @@ +import math +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +def convert_pad_shape(pad_shape): + l = pad_shape[::-1] + pad_shape = [item for sublist in l for item in sublist] + return pad_shape + + +def kl_divergence(m_p, logs_p, m_q, logs_q): + """KL(P||Q)""" + kl = (logs_q - logs_p) - 0.5 + kl += ( + 0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q) + ) + return kl + + +def rand_gumbel(shape): + """Sample from the Gumbel distribution, protect from overflows.""" + uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 + return -torch.log(-torch.log(uniform_samples)) + + +def rand_gumbel_like(x): + g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) + return g + + +def slice_segments(x, ids_str, segment_size=4): + ret = torch.zeros_like(x[:, :, :segment_size]) + for i in range(x.size(0)): + idx_str = ids_str[i] + idx_end = idx_str + segment_size + ret[i] = x[i, :, idx_str:idx_end] + return ret + + +def slice_segments2(x, ids_str, segment_size=4): + ret = torch.zeros_like(x[:, :segment_size]) + for i in range(x.size(0)): + idx_str = ids_str[i] + idx_end = idx_str + segment_size + ret[i] = x[i, idx_str:idx_end] + return ret + + +def rand_slice_segments(x, x_lengths=None, segment_size=4): + b, d, t = x.size() + if x_lengths is None: + x_lengths = t + ids_str_max = x_lengths - segment_size + 1 + ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) + ret = slice_segments(x, ids_str, segment_size) + return ret, ids_str + + +def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4): + position = torch.arange(length, dtype=torch.float) + num_timescales = channels // 2 + log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / ( + num_timescales - 1 + ) + inv_timescales = min_timescale * torch.exp( + torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment + ) + scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) + signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) + signal = F.pad(signal, [0, 0, 0, channels % 2]) + signal = signal.view(1, channels, length) + return signal + + +def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return x + signal.to(dtype=x.dtype, device=x.device) + + +def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) + + +def subsequent_mask(length): + mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) + return mask + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a + input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +def convert_pad_shape(pad_shape): + l = pad_shape[::-1] + pad_shape = [item for sublist in l for item in sublist] + return pad_shape + + +def shift_1d(x): + x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] + return x + + +def sequence_mask(length, max_length=None): + if max_length is None: + max_length = length.max() + x = torch.arange(max_length, dtype=length.dtype, device=length.device) + return x.unsqueeze(0) < length.unsqueeze(1) + + +def generate_path(duration, mask): + """ + duration: [b, 1, t_x] + mask: [b, 1, t_y, t_x] + """ + device = duration.device + + b, _, t_y, t_x = mask.shape + cum_duration = torch.cumsum(duration, -1) + + cum_duration_flat = cum_duration.view(b * t_x) + path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) + path = path.view(b, t_x, t_y) + path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] + path = path.unsqueeze(1).transpose(2, 3) * mask + return path + + +def clip_grad_value_(parameters, clip_value, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + if clip_value is not None: + clip_value = float(clip_value) + + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item() ** norm_type + if clip_value is not None: + p.grad.data.clamp_(min=-clip_value, max=clip_value) + total_norm = total_norm ** (1.0 / norm_type) + return total_norm diff --git a/infer_pack/models.py b/infer_pack/models.py new file mode 100644 index 0000000000000000000000000000000000000000..5e4b2e72383efaee1fae4f5c42e3db2c627e4190 --- /dev/null +++ b/infer_pack/models.py @@ -0,0 +1,1124 @@ +import math, pdb, os +from time import time as ttime +import torch +from torch import nn +from torch.nn import functional as F +from infer_pack import modules +from infer_pack import attentions +from infer_pack import commons +from infer_pack.commons import init_weights, get_padding +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +from infer_pack.commons import init_weights +import numpy as np +from infer_pack import commons + + +class TextEncoder256(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(256, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class TextEncoder768(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(768, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class ResidualCouplingBlock(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0, + ): + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append( + modules.ResidualCouplingLayer( + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + mean_only=True, + ) + ) + self.flows.append(modules.Flip()) + + def forward(self, x, x_mask, g=None, reverse=False): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in reversed(self.flows): + x = flow(x, x_mask, g=g, reverse=reverse) + return x + + def remove_weight_norm(self): + for i in range(self.n_flows): + self.flows[i * 2].remove_weight_norm() + + +class PosteriorEncoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths, g=None): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + +class Generator(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=0, + ): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward(self, x, g=None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +class SineGen(torch.nn.Module): + """Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(np.pi) or cos(0) + """ + + def __init__( + self, + samp_rate, + harmonic_num=0, + sine_amp=0.1, + noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + ): + super(SineGen, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.dim = self.harmonic_num + 1 + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + + def _f02uv(self, f0): + # generate uv signal + uv = torch.ones_like(f0) + uv = uv * (f0 > self.voiced_threshold) + return uv + + def forward(self, f0, upp): + """sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, length, dim=1) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + with torch.no_grad(): + f0 = f0[:, None].transpose(1, 2) + f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device) + # fundamental component + f0_buf[:, :, 0] = f0[:, :, 0] + for idx in np.arange(self.harmonic_num): + f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * ( + idx + 2 + ) # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic + rad_values = (f0_buf / self.sampling_rate) % 1 ###%1意味着n_har的乘积无法后处理优化 + rand_ini = torch.rand( + f0_buf.shape[0], f0_buf.shape[2], device=f0_buf.device + ) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + tmp_over_one = torch.cumsum(rad_values, 1) # % 1 #####%1意味着后面的cumsum无法再优化 + tmp_over_one *= upp + tmp_over_one = F.interpolate( + tmp_over_one.transpose(2, 1), + scale_factor=upp, + mode="linear", + align_corners=True, + ).transpose(2, 1) + rad_values = F.interpolate( + rad_values.transpose(2, 1), scale_factor=upp, mode="nearest" + ).transpose( + 2, 1 + ) ####### + tmp_over_one %= 1 + tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 + cumsum_shift = torch.zeros_like(rad_values) + cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 + sine_waves = torch.sin( + torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi + ) + sine_waves = sine_waves * self.sine_amp + uv = self._f02uv(f0) + uv = F.interpolate( + uv.transpose(2, 1), scale_factor=upp, mode="nearest" + ).transpose(2, 1) + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + noise = noise_amp * torch.randn_like(sine_waves) + sine_waves = sine_waves * uv + noise + return sine_waves, uv, noise + + +class SourceModuleHnNSF(torch.nn.Module): + """SourceModule for hn-nsf + SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0) + sampling_rate: sampling_rate in Hz + harmonic_num: number of harmonic above F0 (default: 0) + sine_amp: amplitude of sine source signal (default: 0.1) + add_noise_std: std of additive Gaussian noise (default: 0.003) + note that amplitude of noise in unvoiced is decided + by sine_amp + voiced_threshold: threhold to set U/V given F0 (default: 0) + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + uv (batchsize, length, 1) + """ + + def __init__( + self, + sampling_rate, + harmonic_num=0, + sine_amp=0.1, + add_noise_std=0.003, + voiced_threshod=0, + is_half=True, + ): + super(SourceModuleHnNSF, self).__init__() + + self.sine_amp = sine_amp + self.noise_std = add_noise_std + self.is_half = is_half + # to produce sine waveforms + self.l_sin_gen = SineGen( + sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod + ) + + # to merge source harmonics into a single excitation + self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) + self.l_tanh = torch.nn.Tanh() + + def forward(self, x, upp=None): + sine_wavs, uv, _ = self.l_sin_gen(x, upp) + if self.is_half: + sine_wavs = sine_wavs.half() + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + return sine_merge, None, None # noise, uv + + +class GeneratorNSF(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels, + sr, + is_half=False, + ): + super(GeneratorNSF, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates)) + self.m_source = SourceModuleHnNSF( + sampling_rate=sr, harmonic_num=0, is_half=is_half + ) + self.noise_convs = nn.ModuleList() + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + c_cur = upsample_initial_channel // (2 ** (i + 1)) + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + if i + 1 < len(upsample_rates): + stride_f0 = np.prod(upsample_rates[i + 1 :]) + self.noise_convs.append( + Conv1d( + 1, + c_cur, + kernel_size=stride_f0 * 2, + stride=stride_f0, + padding=stride_f0 // 2, + ) + ) + else: + self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1)) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + self.upp = np.prod(upsample_rates) + + def forward(self, x, f0, g=None): + har_source, noi_source, uv = self.m_source(f0, self.upp) + har_source = har_source.transpose(1, 2) + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + x_source = self.noise_convs[i](har_source) + x = x + x_source + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +sr2sr = { + "32k": 32000, + "40k": 40000, + "48k": 48000, +} + + +class SynthesizerTrnMs256NSFsid(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ): + super().__init__() + if type(sr) == type("strr"): + sr = sr2sr[sr] + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder256( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + self.dec = GeneratorNSF( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + sr=sr, + is_half=kwargs["is_half"], + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def forward( + self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds + ): # 这里ds是id，[bs,1] + # print(1,pitch.shape)#[bs,t] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length) + pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size) + # print(-2,pitchf.shape,z_slice.shape) + o = self.dec(z_slice, pitchf, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None): + g = self.emb_g(sid).unsqueeze(-1) + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class SynthesizerTrnMs768NSFsid(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ): + super().__init__() + if type(sr) == type("strr"): + sr = sr2sr[sr] + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder768( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + self.dec = GeneratorNSF( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + sr=sr, + is_half=kwargs["is_half"], + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def forward( + self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds + ): # 这里ds是id，[bs,1] + # print(1,pitch.shape)#[bs,t] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length) + pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size) + # print(-2,pitchf.shape,z_slice.shape) + o = self.dec(z_slice, pitchf, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None): + g = self.emb_g(sid).unsqueeze(-1) + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class SynthesizerTrnMs256NSFsid_nono(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr=None, + **kwargs + ): + super().__init__() + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder256( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=False, + ) + self.dec = Generator( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def forward(self, phone, phone_lengths, y, y_lengths, ds): # 这里ds是id，[bs,1] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + o = self.dec(z_slice, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + def infer(self, phone, phone_lengths, sid, max_len=None): + g = self.emb_g(sid).unsqueeze(-1) + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], g=g) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class SynthesizerTrnMs768NSFsid_nono(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr=None, + **kwargs + ): + super().__init__() + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder768( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=False, + ) + self.dec = Generator( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def forward(self, phone, phone_lengths, y, y_lengths, ds): # 这里ds是id，[bs,1] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + o = self.dec(z_slice, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + def infer(self, phone, phone_lengths, sid, max_len=None): + g = self.emb_g(sid).unsqueeze(-1) + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], g=g) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminator, self).__init__() + periods = [2, 3, 5, 7, 11, 17] + # periods = [3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class MultiPeriodDiscriminatorV2(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminatorV2, self).__init__() + # periods = [2, 3, 5, 7, 11, 17] + periods = [2, 3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 16, 15, 1, padding=7)), + norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), + norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ] + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.use_spectral_norm = use_spectral_norm + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f( + Conv2d( + 1, + 32, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 32, + 128, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 128, + 512, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 512, + 1024, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 1024, + 1024, + (kernel_size, 1), + 1, + padding=(get_padding(kernel_size, 1), 0), + ) + ), + ] + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap diff --git a/infer_pack/models_onnx.py b/infer_pack/models_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..b945eac8e59aac38fbd166da49eda01e2b8f4bd4 --- /dev/null +++ b/infer_pack/models_onnx.py @@ -0,0 +1,818 @@ +import math, pdb, os +from time import time as ttime +import torch +from torch import nn +from torch.nn import functional as F +from infer_pack import modules +from infer_pack import attentions +from infer_pack import commons +from infer_pack.commons import init_weights, get_padding +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +from infer_pack.commons import init_weights +import numpy as np +from infer_pack import commons + + +class TextEncoder256(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(256, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class TextEncoder768(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(768, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class ResidualCouplingBlock(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0, + ): + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append( + modules.ResidualCouplingLayer( + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + mean_only=True, + ) + ) + self.flows.append(modules.Flip()) + + def forward(self, x, x_mask, g=None, reverse=False): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in reversed(self.flows): + x = flow(x, x_mask, g=g, reverse=reverse) + return x + + def remove_weight_norm(self): + for i in range(self.n_flows): + self.flows[i * 2].remove_weight_norm() + + +class PosteriorEncoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths, g=None): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + +class Generator(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=0, + ): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward(self, x, g=None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +class SineGen(torch.nn.Module): + """Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(np.pi) or cos(0) + """ + + def __init__( + self, + samp_rate, + harmonic_num=0, + sine_amp=0.1, + noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + ): + super(SineGen, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.dim = self.harmonic_num + 1 + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + + def _f02uv(self, f0): + # generate uv signal + uv = torch.ones_like(f0) + uv = uv * (f0 > self.voiced_threshold) + return uv + + def forward(self, f0, upp): + """sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, length, dim=1) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + with torch.no_grad(): + f0 = f0[:, None].transpose(1, 2) + f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device) + # fundamental component + f0_buf[:, :, 0] = f0[:, :, 0] + for idx in np.arange(self.harmonic_num): + f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * ( + idx + 2 + ) # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic + rad_values = (f0_buf / self.sampling_rate) % 1 ###%1意味着n_har的乘积无法后处理优化 + rand_ini = torch.rand( + f0_buf.shape[0], f0_buf.shape[2], device=f0_buf.device + ) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + tmp_over_one = torch.cumsum(rad_values, 1) # % 1 #####%1意味着后面的cumsum无法再优化 + tmp_over_one *= upp + tmp_over_one = F.interpolate( + tmp_over_one.transpose(2, 1), + scale_factor=upp, + mode="linear", + align_corners=True, + ).transpose(2, 1) + rad_values = F.interpolate( + rad_values.transpose(2, 1), scale_factor=upp, mode="nearest" + ).transpose( + 2, 1 + ) ####### + tmp_over_one %= 1 + tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 + cumsum_shift = torch.zeros_like(rad_values) + cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 + sine_waves = torch.sin( + torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi + ) + sine_waves = sine_waves * self.sine_amp + uv = self._f02uv(f0) + uv = F.interpolate( + uv.transpose(2, 1), scale_factor=upp, mode="nearest" + ).transpose(2, 1) + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + noise = noise_amp * torch.randn_like(sine_waves) + sine_waves = sine_waves * uv + noise + return sine_waves, uv, noise + + +class SourceModuleHnNSF(torch.nn.Module): + """SourceModule for hn-nsf + SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0) + sampling_rate: sampling_rate in Hz + harmonic_num: number of harmonic above F0 (default: 0) + sine_amp: amplitude of sine source signal (default: 0.1) + add_noise_std: std of additive Gaussian noise (default: 0.003) + note that amplitude of noise in unvoiced is decided + by sine_amp + voiced_threshold: threhold to set U/V given F0 (default: 0) + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + uv (batchsize, length, 1) + """ + + def __init__( + self, + sampling_rate, + harmonic_num=0, + sine_amp=0.1, + add_noise_std=0.003, + voiced_threshod=0, + is_half=True, + ): + super(SourceModuleHnNSF, self).__init__() + + self.sine_amp = sine_amp + self.noise_std = add_noise_std + self.is_half = is_half + # to produce sine waveforms + self.l_sin_gen = SineGen( + sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod + ) + + # to merge source harmonics into a single excitation + self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) + self.l_tanh = torch.nn.Tanh() + + def forward(self, x, upp=None): + sine_wavs, uv, _ = self.l_sin_gen(x, upp) + if self.is_half: + sine_wavs = sine_wavs.half() + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + return sine_merge, None, None # noise, uv + + +class GeneratorNSF(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels, + sr, + is_half=False, + ): + super(GeneratorNSF, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates)) + self.m_source = SourceModuleHnNSF( + sampling_rate=sr, harmonic_num=0, is_half=is_half + ) + self.noise_convs = nn.ModuleList() + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + c_cur = upsample_initial_channel // (2 ** (i + 1)) + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + if i + 1 < len(upsample_rates): + stride_f0 = np.prod(upsample_rates[i + 1 :]) + self.noise_convs.append( + Conv1d( + 1, + c_cur, + kernel_size=stride_f0 * 2, + stride=stride_f0, + padding=stride_f0 // 2, + ) + ) + else: + self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1)) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + self.upp = np.prod(upsample_rates) + + def forward(self, x, f0, g=None): + har_source, noi_source, uv = self.m_source(f0, self.upp) + har_source = har_source.transpose(1, 2) + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + x_source = self.noise_convs[i](har_source) + x = x + x_source + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +sr2sr = { + "32k": 32000, + "40k": 40000, + "48k": 48000, +} + + +class SynthesizerTrnMsNSFsidM(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ): + super().__init__() + if type(sr) == type("strr"): + sr = sr2sr[sr] + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + if self.gin_channels == 256: + self.enc_p = TextEncoder256( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + else: + self.enc_p = TextEncoder768( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + self.dec = GeneratorNSF( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + sr=sr, + is_half=kwargs["is_half"], + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + self.speaker_map = None + print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def construct_spkmixmap(self, n_speaker): + self.speaker_map = torch.zeros((n_speaker, 1, 1, self.gin_channels)) + for i in range(n_speaker): + self.speaker_map[i] = self.emb_g(torch.LongTensor([[i]])) + self.speaker_map = self.speaker_map.unsqueeze(0) + + def forward(self, phone, phone_lengths, pitch, nsff0, g, rnd, max_len=None): + if self.speaker_map is not None: # [N, S] * [S, B, 1, H] + g = g.reshape((g.shape[0], g.shape[1], 1, 1, 1)) # [N, S, B, 1, 1] + g = g * self.speaker_map # [N, S, B, 1, H] + g = torch.sum(g, dim=1) # [N, 1, B, 1, H] + g = g.transpose(0, -1).transpose(0, -2).squeeze(0) # [B, H, N] + else: + g = g.unsqueeze(0) + g = self.emb_g(g).transpose(1, 2) + + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * rnd) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g) + return o + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminator, self).__init__() + periods = [2, 3, 5, 7, 11, 17] + # periods = [3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class MultiPeriodDiscriminatorV2(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminatorV2, self).__init__() + # periods = [2, 3, 5, 7, 11, 17] + periods = [2, 3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 16, 15, 1, padding=7)), + norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), + norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ] + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.use_spectral_norm = use_spectral_norm + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f( + Conv2d( + 1, + 32, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 32, + 128, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 128, + 512, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 512, + 1024, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 1024, + 1024, + (kernel_size, 1), + 1, + padding=(get_padding(kernel_size, 1), 0), + ) + ), + ] + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap diff --git a/infer_pack/modules.py b/infer_pack/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..960481cedad9a6106f2bf0b9e86e82b120f7b33f --- /dev/null +++ b/infer_pack/modules.py @@ -0,0 +1,522 @@ +import copy +import math +import numpy as np +import scipy +import torch +from torch import nn +from torch.nn import functional as F + +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm + +from infer_pack import commons +from infer_pack.commons import init_weights, get_padding +from infer_pack.transforms import piecewise_rational_quadratic_transform + + +LRELU_SLOPE = 0.1 + + +class LayerNorm(nn.Module): + def __init__(self, channels, eps=1e-5): + super().__init__() + self.channels = channels + self.eps = eps + + self.gamma = nn.Parameter(torch.ones(channels)) + self.beta = nn.Parameter(torch.zeros(channels)) + + def forward(self, x): + x = x.transpose(1, -1) + x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) + return x.transpose(1, -1) + + +class ConvReluNorm(nn.Module): + def __init__( + self, + in_channels, + hidden_channels, + out_channels, + kernel_size, + n_layers, + p_dropout, + ): + super().__init__() + self.in_channels = in_channels + self.hidden_channels = hidden_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = p_dropout + assert n_layers > 1, "Number of layers should be larger than 0." + + self.conv_layers = nn.ModuleList() + self.norm_layers = nn.ModuleList() + self.conv_layers.append( + nn.Conv1d( + in_channels, hidden_channels, kernel_size, padding=kernel_size // 2 + ) + ) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout)) + for _ in range(n_layers - 1): + self.conv_layers.append( + nn.Conv1d( + hidden_channels, + hidden_channels, + kernel_size, + padding=kernel_size // 2, + ) + ) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.proj = nn.Conv1d(hidden_channels, out_channels, 1) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward(self, x, x_mask): + x_org = x + for i in range(self.n_layers): + x = self.conv_layers[i](x * x_mask) + x = self.norm_layers[i](x) + x = self.relu_drop(x) + x = x_org + self.proj(x) + return x * x_mask + + +class DDSConv(nn.Module): + """ + Dialted and Depth-Separable Convolution + """ + + def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0): + super().__init__() + self.channels = channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = p_dropout + + self.drop = nn.Dropout(p_dropout) + self.convs_sep = nn.ModuleList() + self.convs_1x1 = nn.ModuleList() + self.norms_1 = nn.ModuleList() + self.norms_2 = nn.ModuleList() + for i in range(n_layers): + dilation = kernel_size**i + padding = (kernel_size * dilation - dilation) // 2 + self.convs_sep.append( + nn.Conv1d( + channels, + channels, + kernel_size, + groups=channels, + dilation=dilation, + padding=padding, + ) + ) + self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) + self.norms_1.append(LayerNorm(channels)) + self.norms_2.append(LayerNorm(channels)) + + def forward(self, x, x_mask, g=None): + if g is not None: + x = x + g + for i in range(self.n_layers): + y = self.convs_sep[i](x * x_mask) + y = self.norms_1[i](y) + y = F.gelu(y) + y = self.convs_1x1[i](y) + y = self.norms_2[i](y) + y = F.gelu(y) + y = self.drop(y) + x = x + y + return x * x_mask + + +class WN(torch.nn.Module): + def __init__( + self, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + p_dropout=0, + ): + super(WN, self).__init__() + assert kernel_size % 2 == 1 + self.hidden_channels = hidden_channels + self.kernel_size = (kernel_size,) + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + self.p_dropout = p_dropout + + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.drop = nn.Dropout(p_dropout) + + if gin_channels != 0: + cond_layer = torch.nn.Conv1d( + gin_channels, 2 * hidden_channels * n_layers, 1 + ) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight") + + for i in range(n_layers): + dilation = dilation_rate**i + padding = int((kernel_size * dilation - dilation) / 2) + in_layer = torch.nn.Conv1d( + hidden_channels, + 2 * hidden_channels, + kernel_size, + dilation=dilation, + padding=padding, + ) + in_layer = torch.nn.utils.weight_norm(in_layer, name="weight") + self.in_layers.append(in_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2 * hidden_channels + else: + res_skip_channels = hidden_channels + + res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight") + self.res_skip_layers.append(res_skip_layer) + + def forward(self, x, x_mask, g=None, **kwargs): + output = torch.zeros_like(x) + n_channels_tensor = torch.IntTensor([self.hidden_channels]) + + if g is not None: + g = self.cond_layer(g) + + for i in range(self.n_layers): + x_in = self.in_layers[i](x) + if g is not None: + cond_offset = i * 2 * self.hidden_channels + g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :] + else: + g_l = torch.zeros_like(x_in) + + acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor) + acts = self.drop(acts) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + res_acts = res_skip_acts[:, : self.hidden_channels, :] + x = (x + res_acts) * x_mask + output = output + res_skip_acts[:, self.hidden_channels :, :] + else: + output = output + res_skip_acts + return output * x_mask + + def remove_weight_norm(self): + if self.gin_channels != 0: + torch.nn.utils.remove_weight_norm(self.cond_layer) + for l in self.in_layers: + torch.nn.utils.remove_weight_norm(l) + for l in self.res_skip_layers: + torch.nn.utils.remove_weight_norm(l) + + +class ResBlock1(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super(ResBlock1, self).__init__() + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]), + ) + ), + ] + ) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + ] + ) + self.convs2.apply(init_weights) + + def forward(self, x, x_mask=None): + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c1(xt) + xt = F.leaky_relu(xt, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c2(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + +class ResBlock2(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3)): + super(ResBlock2, self).__init__() + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + ] + ) + self.convs.apply(init_weights) + + def forward(self, x, x_mask=None): + for c in self.convs: + xt = F.leaky_relu(x, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + +class Log(nn.Module): + def forward(self, x, x_mask, reverse=False, **kwargs): + if not reverse: + y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask + logdet = torch.sum(-y, [1, 2]) + return y, logdet + else: + x = torch.exp(x) * x_mask + return x + + +class Flip(nn.Module): + def forward(self, x, *args, reverse=False, **kwargs): + x = torch.flip(x, [1]) + if not reverse: + logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) + return x, logdet + else: + return x + + +class ElementwiseAffine(nn.Module): + def __init__(self, channels): + super().__init__() + self.channels = channels + self.m = nn.Parameter(torch.zeros(channels, 1)) + self.logs = nn.Parameter(torch.zeros(channels, 1)) + + def forward(self, x, x_mask, reverse=False, **kwargs): + if not reverse: + y = self.m + torch.exp(self.logs) * x + y = y * x_mask + logdet = torch.sum(self.logs * x_mask, [1, 2]) + return y, logdet + else: + x = (x - self.m) * torch.exp(-self.logs) * x_mask + return x + + +class ResidualCouplingLayer(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + p_dropout=0, + gin_channels=0, + mean_only=False, + ): + assert channels % 2 == 0, "channels should be divisible by 2" + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.half_channels = channels // 2 + self.mean_only = mean_only + + self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) + self.enc = WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + p_dropout=p_dropout, + gin_channels=gin_channels, + ) + self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) + self.post.weight.data.zero_() + self.post.bias.data.zero_() + + def forward(self, x, x_mask, g=None, reverse=False): + x0, x1 = torch.split(x, [self.half_channels] * 2, 1) + h = self.pre(x0) * x_mask + h = self.enc(h, x_mask, g=g) + stats = self.post(h) * x_mask + if not self.mean_only: + m, logs = torch.split(stats, [self.half_channels] * 2, 1) + else: + m = stats + logs = torch.zeros_like(m) + + if not reverse: + x1 = m + x1 * torch.exp(logs) * x_mask + x = torch.cat([x0, x1], 1) + logdet = torch.sum(logs, [1, 2]) + return x, logdet + else: + x1 = (x1 - m) * torch.exp(-logs) * x_mask + x = torch.cat([x0, x1], 1) + return x + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + +class ConvFlow(nn.Module): + def __init__( + self, + in_channels, + filter_channels, + kernel_size, + n_layers, + num_bins=10, + tail_bound=5.0, + ): + super().__init__() + self.in_channels = in_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.num_bins = num_bins + self.tail_bound = tail_bound + self.half_channels = in_channels // 2 + + self.pre = nn.Conv1d(self.half_channels, filter_channels, 1) + self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0) + self.proj = nn.Conv1d( + filter_channels, self.half_channels * (num_bins * 3 - 1), 1 + ) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward(self, x, x_mask, g=None, reverse=False): + x0, x1 = torch.split(x, [self.half_channels] * 2, 1) + h = self.pre(x0) + h = self.convs(h, x_mask, g=g) + h = self.proj(h) * x_mask + + b, c, t = x0.shape + h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?] + + unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels) + unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt( + self.filter_channels + ) + unnormalized_derivatives = h[..., 2 * self.num_bins :] + + x1, logabsdet = piecewise_rational_quadratic_transform( + x1, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=reverse, + tails="linear", + tail_bound=self.tail_bound, + ) + + x = torch.cat([x0, x1], 1) * x_mask + logdet = torch.sum(logabsdet * x_mask, [1, 2]) + if not reverse: + return x, logdet + else: + return x diff --git a/infer_pack/modules/F0Predictor/DioF0Predictor.py b/infer_pack/modules/F0Predictor/DioF0Predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..eb60d8830714338448be009d1075e3594337db15 --- /dev/null +++ b/infer_pack/modules/F0Predictor/DioF0Predictor.py @@ -0,0 +1,90 @@ +from infer_pack.modules.F0Predictor.F0Predictor import F0Predictor +import pyworld +import numpy as np + + +class DioF0Predictor(F0Predictor): + def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): + self.hop_length = hop_length + self.f0_min = f0_min + self.f0_max = f0_max + self.sampling_rate = sampling_rate + + def interpolate_f0(self, f0): + """ + 对F0进行插值处理 + """ + + data = np.reshape(f0, (f0.size, 1)) + + vuv_vector = np.zeros((data.size, 1), dtype=np.float32) + vuv_vector[data > 0.0] = 1.0 + vuv_vector[data <= 0.0] = 0.0 + + ip_data = data + + frame_number = data.size + last_value = 0.0 + for i in range(frame_number): + if data[i] <= 0.0: + j = i + 1 + for j in range(i + 1, frame_number): + if data[j] > 0.0: + break + if j < frame_number - 1: + if last_value > 0.0: + step = (data[j] - data[i - 1]) / float(j - i) + for k in range(i, j): + ip_data[k] = data[i - 1] + step * (k - i + 1) + else: + for k in range(i, j): + ip_data[k] = data[j] + else: + for k in range(i, frame_number): + ip_data[k] = last_value + else: + ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 + last_value = data[i] + + return ip_data[:, 0], vuv_vector[:, 0] + + def resize_f0(self, x, target_len): + source = np.array(x) + source[source < 0.001] = np.nan + target = np.interp( + np.arange(0, len(source) * target_len, len(source)) / target_len, + np.arange(0, len(source)), + source, + ) + res = np.nan_to_num(target) + return res + + def compute_f0(self, wav, p_len=None): + if p_len is None: + p_len = wav.shape[0] // self.hop_length + f0, t = pyworld.dio( + wav.astype(np.double), + fs=self.sampling_rate, + f0_floor=self.f0_min, + f0_ceil=self.f0_max, + frame_period=1000 * self.hop_length / self.sampling_rate, + ) + f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) + for index, pitch in enumerate(f0): + f0[index] = round(pitch, 1) + return self.interpolate_f0(self.resize_f0(f0, p_len))[0] + + def compute_f0_uv(self, wav, p_len=None): + if p_len is None: + p_len = wav.shape[0] // self.hop_length + f0, t = pyworld.dio( + wav.astype(np.double), + fs=self.sampling_rate, + f0_floor=self.f0_min, + f0_ceil=self.f0_max, + frame_period=1000 * self.hop_length / self.sampling_rate, + ) + f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) + for index, pitch in enumerate(f0): + f0[index] = round(pitch, 1) + return self.interpolate_f0(self.resize_f0(f0, p_len)) diff --git a/infer_pack/modules/F0Predictor/F0Predictor.py b/infer_pack/modules/F0Predictor/F0Predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..f56e49e7f0e6eab3babf0711cae2933371b9f9cc --- /dev/null +++ b/infer_pack/modules/F0Predictor/F0Predictor.py @@ -0,0 +1,16 @@ +class F0Predictor(object): + def compute_f0(self, wav, p_len): + """ + input: wav:[signal_length] + p_len:int + output: f0:[signal_length//hop_length] + """ + pass + + def compute_f0_uv(self, wav, p_len): + """ + input: wav:[signal_length] + p_len:int + output: f0:[signal_length//hop_length],uv:[signal_length//hop_length] + """ + pass diff --git a/infer_pack/modules/F0Predictor/HarvestF0Predictor.py b/infer_pack/modules/F0Predictor/HarvestF0Predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..98d4e98b353008f81bde2c37e7da818763a992c9 --- /dev/null +++ b/infer_pack/modules/F0Predictor/HarvestF0Predictor.py @@ -0,0 +1,86 @@ +from infer_pack.modules.F0Predictor.F0Predictor import F0Predictor +import pyworld +import numpy as np + + +class HarvestF0Predictor(F0Predictor): + def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): + self.hop_length = hop_length + self.f0_min = f0_min + self.f0_max = f0_max + self.sampling_rate = sampling_rate + + def interpolate_f0(self, f0): + """ + 对F0进行插值处理 + """ + + data = np.reshape(f0, (f0.size, 1)) + + vuv_vector = np.zeros((data.size, 1), dtype=np.float32) + vuv_vector[data > 0.0] = 1.0 + vuv_vector[data <= 0.0] = 0.0 + + ip_data = data + + frame_number = data.size + last_value = 0.0 + for i in range(frame_number): + if data[i] <= 0.0: + j = i + 1 + for j in range(i + 1, frame_number): + if data[j] > 0.0: + break + if j < frame_number - 1: + if last_value > 0.0: + step = (data[j] - data[i - 1]) / float(j - i) + for k in range(i, j): + ip_data[k] = data[i - 1] + step * (k - i + 1) + else: + for k in range(i, j): + ip_data[k] = data[j] + else: + for k in range(i, frame_number): + ip_data[k] = last_value + else: + ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 + last_value = data[i] + + return ip_data[:, 0], vuv_vector[:, 0] + + def resize_f0(self, x, target_len): + source = np.array(x) + source[source < 0.001] = np.nan + target = np.interp( + np.arange(0, len(source) * target_len, len(source)) / target_len, + np.arange(0, len(source)), + source, + ) + res = np.nan_to_num(target) + return res + + def compute_f0(self, wav, p_len=None): + if p_len is None: + p_len = wav.shape[0] // self.hop_length + f0, t = pyworld.harvest( + wav.astype(np.double), + fs=self.hop_length, + f0_ceil=self.f0_max, + f0_floor=self.f0_min, + frame_period=1000 * self.hop_length / self.sampling_rate, + ) + f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.fs) + return self.interpolate_f0(self.resize_f0(f0, p_len))[0] + + def compute_f0_uv(self, wav, p_len=None): + if p_len is None: + p_len = wav.shape[0] // self.hop_length + f0, t = pyworld.harvest( + wav.astype(np.double), + fs=self.sampling_rate, + f0_floor=self.f0_min, + f0_ceil=self.f0_max, + frame_period=1000 * self.hop_length / self.sampling_rate, + ) + f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) + return self.interpolate_f0(self.resize_f0(f0, p_len)) diff --git a/infer_pack/modules/F0Predictor/PMF0Predictor.py b/infer_pack/modules/F0Predictor/PMF0Predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..ab523020325fa3f30676ad20125c6a9f059a9d84 --- /dev/null +++ b/infer_pack/modules/F0Predictor/PMF0Predictor.py @@ -0,0 +1,97 @@ +from infer_pack.modules.F0Predictor.F0Predictor import F0Predictor +import parselmouth +import numpy as np + + +class PMF0Predictor(F0Predictor): + def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): + self.hop_length = hop_length + self.f0_min = f0_min + self.f0_max = f0_max + self.sampling_rate = sampling_rate + + def interpolate_f0(self, f0): + """ + 对F0进行插值处理 + """ + + data = np.reshape(f0, (f0.size, 1)) + + vuv_vector = np.zeros((data.size, 1), dtype=np.float32) + vuv_vector[data > 0.0] = 1.0 + vuv_vector[data <= 0.0] = 0.0 + + ip_data = data + + frame_number = data.size + last_value = 0.0 + for i in range(frame_number): + if data[i] <= 0.0: + j = i + 1 + for j in range(i + 1, frame_number): + if data[j] > 0.0: + break + if j < frame_number - 1: + if last_value > 0.0: + step = (data[j] - data[i - 1]) / float(j - i) + for k in range(i, j): + ip_data[k] = data[i - 1] + step * (k - i + 1) + else: + for k in range(i, j): + ip_data[k] = data[j] + else: + for k in range(i, frame_number): + ip_data[k] = last_value + else: + ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 + last_value = data[i] + + return ip_data[:, 0], vuv_vector[:, 0] + + def compute_f0(self, wav, p_len=None): + x = wav + if p_len is None: + p_len = x.shape[0] // self.hop_length + else: + assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" + time_step = self.hop_length / self.sampling_rate * 1000 + f0 = ( + parselmouth.Sound(x, self.sampling_rate) + .to_pitch_ac( + time_step=time_step / 1000, + voicing_threshold=0.6, + pitch_floor=self.f0_min, + pitch_ceiling=self.f0_max, + ) + .selected_array["frequency"] + ) + + pad_size = (p_len - len(f0) + 1) // 2 + if pad_size > 0 or p_len - len(f0) - pad_size > 0: + f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") + f0, uv = self.interpolate_f0(f0) + return f0 + + def compute_f0_uv(self, wav, p_len=None): + x = wav + if p_len is None: + p_len = x.shape[0] // self.hop_length + else: + assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" + time_step = self.hop_length / self.sampling_rate * 1000 + f0 = ( + parselmouth.Sound(x, self.sampling_rate) + .to_pitch_ac( + time_step=time_step / 1000, + voicing_threshold=0.6, + pitch_floor=self.f0_min, + pitch_ceiling=self.f0_max, + ) + .selected_array["frequency"] + ) + + pad_size = (p_len - len(f0) + 1) // 2 + if pad_size > 0 or p_len - len(f0) - pad_size > 0: + f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") + f0, uv = self.interpolate_f0(f0) + return f0, uv diff --git a/infer_pack/modules/F0Predictor/__init__.py b/infer_pack/modules/F0Predictor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infer_pack/onnx_inference.py b/infer_pack/onnx_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..322572820dfc75d789e40ce5bbd9415066a03979 --- /dev/null +++ b/infer_pack/onnx_inference.py @@ -0,0 +1,139 @@ +import onnxruntime +import librosa +import numpy as np +import soundfile + + +class ContentVec: + def __init__(self, vec_path="pretrained/vec-768-layer-12.onnx", device=None): + print("load model(s) from {}".format(vec_path)) + if device == "cpu" or device is None: + providers = ["CPUExecutionProvider"] + elif device == "cuda": + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + else: + raise RuntimeError("Unsportted Device") + self.model = onnxruntime.InferenceSession(vec_path, providers=providers) + + def __call__(self, wav): + return self.forward(wav) + + def forward(self, wav): + feats = wav + if feats.ndim == 2: # double channels + feats = feats.mean(-1) + assert feats.ndim == 1, feats.ndim + feats = np.expand_dims(np.expand_dims(feats, 0), 0) + onnx_input = {self.model.get_inputs()[0].name: feats} + logits = self.model.run(None, onnx_input)[0] + return logits.transpose(0, 2, 1) + + +def get_f0_predictor(f0_predictor, hop_length, sampling_rate, **kargs): + if f0_predictor == "pm": + from infer_pack.modules.F0Predictor.PMF0Predictor import PMF0Predictor + + f0_predictor_object = PMF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + elif f0_predictor == "harvest": + from infer_pack.modules.F0Predictor.HarvestF0Predictor import HarvestF0Predictor + + f0_predictor_object = HarvestF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + elif f0_predictor == "dio": + from infer_pack.modules.F0Predictor.DioF0Predictor import DioF0Predictor + + f0_predictor_object = DioF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + else: + raise Exception("Unknown f0 predictor") + return f0_predictor_object + + +class OnnxRVC: + def __init__( + self, + model_path, + sr=40000, + hop_size=512, + vec_path="vec-768-layer-12", + device="cpu", + ): + vec_path = f"pretrained/{vec_path}.onnx" + self.vec_model = ContentVec(vec_path, device) + if device == "cpu" or device is None: + providers = ["CPUExecutionProvider"] + elif device == "cuda": + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + else: + raise RuntimeError("Unsportted Device") + self.model = onnxruntime.InferenceSession(model_path, providers=providers) + self.sampling_rate = sr + self.hop_size = hop_size + + def forward(self, hubert, hubert_length, pitch, pitchf, ds, rnd): + onnx_input = { + self.model.get_inputs()[0].name: hubert, + self.model.get_inputs()[1].name: hubert_length, + self.model.get_inputs()[2].name: pitch, + self.model.get_inputs()[3].name: pitchf, + self.model.get_inputs()[4].name: ds, + self.model.get_inputs()[5].name: rnd, + } + return (self.model.run(None, onnx_input)[0] * 32767).astype(np.int16) + + def inference( + self, + raw_path, + sid, + f0_method="dio", + f0_up_key=0, + pad_time=0.5, + cr_threshold=0.02, + ): + f0_min = 50 + f0_max = 1100 + f0_mel_min = 1127 * np.log(1 + f0_min / 700) + f0_mel_max = 1127 * np.log(1 + f0_max / 700) + f0_predictor = get_f0_predictor( + f0_method, + hop_length=self.hop_size, + sampling_rate=self.sampling_rate, + threshold=cr_threshold, + ) + wav, sr = librosa.load(raw_path, sr=self.sampling_rate) + org_length = len(wav) + if org_length / sr > 50.0: + raise RuntimeError("Reached Max Length") + + wav16k = librosa.resample(wav, orig_sr=self.sampling_rate, target_sr=16000) + wav16k = wav16k + + hubert = self.vec_model(wav16k) + hubert = np.repeat(hubert, 2, axis=2).transpose(0, 2, 1).astype(np.float32) + hubert_length = hubert.shape[1] + + pitchf = f0_predictor.compute_f0(wav, hubert_length) + pitchf = pitchf * 2 ** (f0_up_key / 12) + pitch = pitchf.copy() + f0_mel = 1127 * np.log(1 + pitch / 700) + f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / ( + f0_mel_max - f0_mel_min + ) + 1 + f0_mel[f0_mel <= 1] = 1 + f0_mel[f0_mel > 255] = 255 + pitch = np.rint(f0_mel).astype(np.int64) + + pitchf = pitchf.reshape(1, len(pitchf)).astype(np.float32) + pitch = pitch.reshape(1, len(pitch)) + ds = np.array([sid]).astype(np.int64) + + rnd = np.random.randn(1, 192, hubert_length).astype(np.float32) + hubert_length = np.array([hubert_length]).astype(np.int64) + + out_wav = self.forward(hubert, hubert_length, pitch, pitchf, ds, rnd).squeeze() + out_wav = np.pad(out_wav, (0, 2 * self.hop_size), "constant") + return out_wav[0:org_length] diff --git a/infer_pack/transforms.py b/infer_pack/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..a11f799e023864ff7082c1f49c0cc18351a13b47 --- /dev/null +++ b/infer_pack/transforms.py @@ -0,0 +1,209 @@ +import torch +from torch.nn import functional as F + +import numpy as np + + +DEFAULT_MIN_BIN_WIDTH = 1e-3 +DEFAULT_MIN_BIN_HEIGHT = 1e-3 +DEFAULT_MIN_DERIVATIVE = 1e-3 + + +def piecewise_rational_quadratic_transform( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + tails=None, + tail_bound=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + if tails is None: + spline_fn = rational_quadratic_spline + spline_kwargs = {} + else: + spline_fn = unconstrained_rational_quadratic_spline + spline_kwargs = {"tails": tails, "tail_bound": tail_bound} + + outputs, logabsdet = spline_fn( + inputs=inputs, + unnormalized_widths=unnormalized_widths, + unnormalized_heights=unnormalized_heights, + unnormalized_derivatives=unnormalized_derivatives, + inverse=inverse, + min_bin_width=min_bin_width, + min_bin_height=min_bin_height, + min_derivative=min_derivative, + **spline_kwargs + ) + return outputs, logabsdet + + +def searchsorted(bin_locations, inputs, eps=1e-6): + bin_locations[..., -1] += eps + return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1 + + +def unconstrained_rational_quadratic_spline( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + tails="linear", + tail_bound=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound) + outside_interval_mask = ~inside_interval_mask + + outputs = torch.zeros_like(inputs) + logabsdet = torch.zeros_like(inputs) + + if tails == "linear": + unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1)) + constant = np.log(np.exp(1 - min_derivative) - 1) + unnormalized_derivatives[..., 0] = constant + unnormalized_derivatives[..., -1] = constant + + outputs[outside_interval_mask] = inputs[outside_interval_mask] + logabsdet[outside_interval_mask] = 0 + else: + raise RuntimeError("{} tails are not implemented.".format(tails)) + + ( + outputs[inside_interval_mask], + logabsdet[inside_interval_mask], + ) = rational_quadratic_spline( + inputs=inputs[inside_interval_mask], + unnormalized_widths=unnormalized_widths[inside_interval_mask, :], + unnormalized_heights=unnormalized_heights[inside_interval_mask, :], + unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :], + inverse=inverse, + left=-tail_bound, + right=tail_bound, + bottom=-tail_bound, + top=tail_bound, + min_bin_width=min_bin_width, + min_bin_height=min_bin_height, + min_derivative=min_derivative, + ) + + return outputs, logabsdet + + +def rational_quadratic_spline( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + left=0.0, + right=1.0, + bottom=0.0, + top=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + if torch.min(inputs) < left or torch.max(inputs) > right: + raise ValueError("Input to a transform is not within its domain") + + num_bins = unnormalized_widths.shape[-1] + + if min_bin_width * num_bins > 1.0: + raise ValueError("Minimal bin width too large for the number of bins") + if min_bin_height * num_bins > 1.0: + raise ValueError("Minimal bin height too large for the number of bins") + + widths = F.softmax(unnormalized_widths, dim=-1) + widths = min_bin_width + (1 - min_bin_width * num_bins) * widths + cumwidths = torch.cumsum(widths, dim=-1) + cumwidths = F.pad(cumwidths, pad=(1, 0), mode="constant", value=0.0) + cumwidths = (right - left) * cumwidths + left + cumwidths[..., 0] = left + cumwidths[..., -1] = right + widths = cumwidths[..., 1:] - cumwidths[..., :-1] + + derivatives = min_derivative + F.softplus(unnormalized_derivatives) + + heights = F.softmax(unnormalized_heights, dim=-1) + heights = min_bin_height + (1 - min_bin_height * num_bins) * heights + cumheights = torch.cumsum(heights, dim=-1) + cumheights = F.pad(cumheights, pad=(1, 0), mode="constant", value=0.0) + cumheights = (top - bottom) * cumheights + bottom + cumheights[..., 0] = bottom + cumheights[..., -1] = top + heights = cumheights[..., 1:] - cumheights[..., :-1] + + if inverse: + bin_idx = searchsorted(cumheights, inputs)[..., None] + else: + bin_idx = searchsorted(cumwidths, inputs)[..., None] + + input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0] + input_bin_widths = widths.gather(-1, bin_idx)[..., 0] + + input_cumheights = cumheights.gather(-1, bin_idx)[..., 0] + delta = heights / widths + input_delta = delta.gather(-1, bin_idx)[..., 0] + + input_derivatives = derivatives.gather(-1, bin_idx)[..., 0] + input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0] + + input_heights = heights.gather(-1, bin_idx)[..., 0] + + if inverse: + a = (inputs - input_cumheights) * ( + input_derivatives + input_derivatives_plus_one - 2 * input_delta + ) + input_heights * (input_delta - input_derivatives) + b = input_heights * input_derivatives - (inputs - input_cumheights) * ( + input_derivatives + input_derivatives_plus_one - 2 * input_delta + ) + c = -input_delta * (inputs - input_cumheights) + + discriminant = b.pow(2) - 4 * a * c + assert (discriminant >= 0).all() + + root = (2 * c) / (-b - torch.sqrt(discriminant)) + outputs = root * input_bin_widths + input_cumwidths + + theta_one_minus_theta = root * (1 - root) + denominator = input_delta + ( + (input_derivatives + input_derivatives_plus_one - 2 * input_delta) + * theta_one_minus_theta + ) + derivative_numerator = input_delta.pow(2) * ( + input_derivatives_plus_one * root.pow(2) + + 2 * input_delta * theta_one_minus_theta + + input_derivatives * (1 - root).pow(2) + ) + logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) + + return outputs, -logabsdet + else: + theta = (inputs - input_cumwidths) / input_bin_widths + theta_one_minus_theta = theta * (1 - theta) + + numerator = input_heights * ( + input_delta * theta.pow(2) + input_derivatives * theta_one_minus_theta + ) + denominator = input_delta + ( + (input_derivatives + input_derivatives_plus_one - 2 * input_delta) + * theta_one_minus_theta + ) + outputs = input_cumheights + numerator / denominator + + derivative_numerator = input_delta.pow(2) * ( + input_derivatives_plus_one * theta.pow(2) + + 2 * input_delta * theta_one_minus_theta + + input_derivatives * (1 - theta).pow(2) + ) + logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) + + return outputs, logabsdet diff --git a/model/GROWLtest/GROWLtest.pth b/model/GROWLtest/GROWLtest.pth new file mode 100644 index 0000000000000000000000000000000000000000..80d60c9461a37d83f727b9616bacb14d0bbdcae5 --- /dev/null +++ b/model/GROWLtest/GROWLtest.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561021c2f18158e5d6dc07766f079ef47b0d87e88938633ec3373f48cf592fc5 +size 55224656 diff --git a/model/GROWLtest/config.json b/model/GROWLtest/config.json new file mode 100644 index 0000000000000000000000000000000000000000..af8f6604998ce5f5c8ea35703eaf8b39b8db092d --- /dev/null +++ b/model/GROWLtest/config.json @@ -0,0 +1,12 @@ +{ + "model": "GROWLtest.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + + "name": "DUBSTEP GROWL", + "author": "Rubin", + "source": "Rubin", + "note": "250 EPOCH", + "icon": "" +} diff --git a/model/Ice Spice 11k/IceSpice.pth b/model/Ice Spice 11k/IceSpice.pth new file mode 100644 index 0000000000000000000000000000000000000000..85c0c91b1079b4647c0423d8028e5228334dc2cb --- /dev/null +++ b/model/Ice Spice 11k/IceSpice.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10dceb4a1444639dac718f131c5bca8ddf080e101b281dc8e82bc0e2f3f4b80 +size 55027130 diff --git a/model/Ice Spice 11k/added_IVF189_Flat_nprobe_4.index b/model/Ice Spice 11k/added_IVF189_Flat_nprobe_4.index new file mode 100644 index 0000000000000000000000000000000000000000..f13811eb3857d0039bba4e4f054218a6de4f209a --- /dev/null +++ b/model/Ice Spice 11k/added_IVF189_Flat_nprobe_4.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89054f829574cbe3612f8023e64d56d7ec01fa81e9cc99904dcbac595d69693 +size 7821667 diff --git a/model/Ice Spice 11k/config.json b/model/Ice Spice 11k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0ece4a6f7accac8f2c6c7dd710cdf7d19f402254 --- /dev/null +++ b/model/Ice Spice 11k/config.json @@ -0,0 +1,11 @@ +{ + "model": "IceSpice.pth", + "feat_index": "added_IVF189_Flat_nprobe_4.index", + "feat_npy": "total_fea.npy", + "speaker_id": 0, + "name": "IceSpice", + "author": "Rubin", + "source": "Rubin", + "note": "Ice Spice 11k", + "icon": "" +} diff --git a/model/Ice Spice 11k/total_fea.npy b/model/Ice Spice 11k/total_fea.npy new file mode 100644 index 0000000000000000000000000000000000000000..664b13971e2d415fc99cf0479957d91d533118bf --- /dev/null +++ b/model/Ice Spice 11k/total_fea.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f483a3f81a6b296f3fec277b5c114c5e8dcd6960d4a1050a7d69ff47eac2b5d5 +size 7567488 diff --git a/model/Ice Spice Unknown Steps/IceSpice.pth b/model/Ice Spice Unknown Steps/IceSpice.pth new file mode 100644 index 0000000000000000000000000000000000000000..02043e618d3214e1693d203954b6bb6923009a34 --- /dev/null +++ b/model/Ice Spice Unknown Steps/IceSpice.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf10980385699dc4080064752ce9a3a9ca1592dc51e2e5ac17977cb66708d0d +size 548679711 diff --git a/model/Ice Spice Unknown Steps/config.json b/model/Ice Spice Unknown Steps/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3d869ea9d05aa4d9f52ed8fe51595b25d15095d3 --- /dev/null +++ b/model/Ice Spice Unknown Steps/config.json @@ -0,0 +1,11 @@ +{ + "model": "IceSpice.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + "name": "IceSpice", + "author": "Rubin", + "source": "Rubin", + "note": "Ice Spice Unknown Steps", + "icon": "" +} diff --git a/model/Ice Spice Unknown Steps/config2.json b/model/Ice Spice Unknown Steps/config2.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd2d4e55e476829f5f4229c32e37310acbb5122 --- /dev/null +++ b/model/Ice Spice Unknown Steps/config2.json @@ -0,0 +1,98 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 1000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 0.0001, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, + "batch_size": 6, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 10240, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 512, + "port": "8001", + "keep_ckpts": 3 + }, + "data": { + "training_files": "filelists/train.txt", + "validation_files": "filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 44100, + "filter_length": 2048, + "hop_length": 512, + "win_length": 2048, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 22050, + "contentvec_final_proj": false + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [ + 3, + 7, + 11 + ], + "resblock_dilation_sizes": [ + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ] + ], + "upsample_rates": [ + 8, + 8, + 2, + 2, + 2 + ], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [ + 16, + 16, + 4, + 4, + 4 + ], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 768, + "n_speakers": 200 + }, + "spk": { + "IceSpice": 0 + } +, + "feat_index": "" +, + "feat_npy": "" +} diff --git a/model/IceSpice Test/IceSpice.pth b/model/IceSpice Test/IceSpice.pth new file mode 100644 index 0000000000000000000000000000000000000000..85c0c91b1079b4647c0423d8028e5228334dc2cb --- /dev/null +++ b/model/IceSpice Test/IceSpice.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10dceb4a1444639dac718f131c5bca8ddf080e101b281dc8e82bc0e2f3f4b80 +size 55027130 diff --git a/model/IceSpice Test/added_IVF189_Flat_nprobe_4.index b/model/IceSpice Test/added_IVF189_Flat_nprobe_4.index new file mode 100644 index 0000000000000000000000000000000000000000..f13811eb3857d0039bba4e4f054218a6de4f209a --- /dev/null +++ b/model/IceSpice Test/added_IVF189_Flat_nprobe_4.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89054f829574cbe3612f8023e64d56d7ec01fa81e9cc99904dcbac595d69693 +size 7821667 diff --git a/model/IceSpice Test/config.json b/model/IceSpice Test/config.json new file mode 100644 index 0000000000000000000000000000000000000000..45fac42a317121a0c5ba856ecb2697d9c329a15e --- /dev/null +++ b/model/IceSpice Test/config.json @@ -0,0 +1,11 @@ +{ + "model": "IceSpice.pth", + "feat_index": "added_IVF189_Flat_nprobe_4.index", + "feat_npy": "total_fea.npy", + "speaker_id": 0, + "name": "IceSpice", + "author": "Rubin", + "source": "Rubin", + "note": "icespice", + "icon": "" +} diff --git a/model/IceSpice Test/total_fea.npy b/model/IceSpice Test/total_fea.npy new file mode 100644 index 0000000000000000000000000000000000000000..664b13971e2d415fc99cf0479957d91d533118bf --- /dev/null +++ b/model/IceSpice Test/total_fea.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f483a3f81a6b296f3fec277b5c114c5e8dcd6960d4a1050a7d69ff47eac2b5d5 +size 7567488 diff --git a/model/Justin Bieber 500/added_IVF954_Flat_nprobe_1_v2.index b/model/Justin Bieber 500/added_IVF954_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..4dfc9bf23ae66b49903bd0f9f83f93dcc3408cb9 --- /dev/null +++ b/model/Justin Bieber 500/added_IVF954_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2516622153c2db7f4d83c6e259c91f62cf570d1d8ddbe7edeefa1ff386bed5 +size 117548339 diff --git a/model/Justin Bieber 500/config.json b/model/Justin Bieber 500/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fdac79c03b07246e6e107f45b6de8d12933aae01 --- /dev/null +++ b/model/Justin Bieber 500/config.json @@ -0,0 +1,11 @@ +{ + "model": "justinbieber.pth", + "feat_index": "added_IVF954_Flat_nprobe_1_v2.index", + "feat_npy": "", + "speaker_id": 0, + "name": "justinbieber", + "author": "Rubin", + "source": "Rubin", + "note": "2JustinBieber500", + "icon": "" +} diff --git a/model/Justin Bieber 500/justinbieber.pth b/model/Justin Bieber 500/justinbieber.pth new file mode 100644 index 0000000000000000000000000000000000000000..d4647c1e59c4db69fb3d78376162d8cbc334bfc2 --- /dev/null +++ b/model/Justin Bieber 500/justinbieber.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8015293c3e710def931e6c91e437166fb0b04b6a975a5ed0886a27f779e270 +size 55225574 diff --git a/model/Justin Bieber 67k/Justin Bieber.pth b/model/Justin Bieber 67k/Justin Bieber.pth new file mode 100644 index 0000000000000000000000000000000000000000..a0c32dcd76ec24aae938c844ecce97d54fbc8a02 --- /dev/null +++ b/model/Justin Bieber 67k/Justin Bieber.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6e6936ef757e5fbef526b25c0a6004c3c02fb2a87cd1ab82f13142f314f498a +size 542789469 diff --git a/model/Justin Bieber 67k/config.json b/model/Justin Bieber 67k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..de0e90dc72e94bdb419e1b7bbda28f5bf06a7e03 --- /dev/null +++ b/model/Justin Bieber 67k/config.json @@ -0,0 +1,11 @@ +{ + "model": "Justin Bieber.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + "name": "Justin Bieber", + "author": "Rubin", + "source": "Rubin", + "note": "Justin Bieber 67k", + "icon": "" +} diff --git a/model/Justin Bieber 67k/config2.json b/model/Justin Bieber 67k/config2.json new file mode 100644 index 0000000000000000000000000000000000000000..261e9132c641cbc241d6dcdd88e856df085c8ed4 --- /dev/null +++ b/model/Justin Bieber 67k/config2.json @@ -0,0 +1,96 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 800, + "seed": 1234, + "epochs": 10000, + "learning_rate": 0.0001, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, + "batch_size": 16, + "fp16_run": false, + "bf16_run": false, + "lr_decay": 0.999875, + "segment_size": 10240, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 512, + "port": "8001", + "keep_ckpts": 3, + "num_workers": 4, + "log_version": 0 + }, + "data": { + "training_files": "filelists/44k/train.txt", + "validation_files": "filelists/44k/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 44100, + "filter_length": 2048, + "hop_length": 512, + "win_length": 2048, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 22050 + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [ + 3, + 7, + 11 + ], + "resblock_dilation_sizes": [ + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ] + ], + "upsample_rates": [ + 8, + 8, + 2, + 2, + 2 + ], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [ + 16, + 16, + 4, + 4, + 4 + ], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 256, + "n_speakers": 200 + }, + "spk": { + "justinbieber": 0 + } +} \ No newline at end of file diff --git a/model/Post Malone 9.6k/Post Malone.pth b/model/Post Malone 9.6k/Post Malone.pth new file mode 100644 index 0000000000000000000000000000000000000000..9ae282b3960848c80f7dd4993f4e978749c38476 --- /dev/null +++ b/model/Post Malone 9.6k/Post Malone.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8086aa565cac905b1671de39fd38c35b4d1fd5f45674c0eedcb0fbae6009893 +size 542170143 diff --git a/model/Post Malone 9.6k/config.json b/model/Post Malone 9.6k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6913701ee3611b9815ff4f8aa7739b63e297ec2d --- /dev/null +++ b/model/Post Malone 9.6k/config.json @@ -0,0 +1,11 @@ +{ + "model": "Post Malone.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + "name": "Post Malone", + "author": "Rubin", + "source": "Rubin", + "note": "Post Malone 9.6k", + "icon": "" +} diff --git a/model/Post Malone 9.6k/config2.json b/model/Post Malone 9.6k/config2.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba8c53d43c4aedc9b030068627748e9914f5e59 --- /dev/null +++ b/model/Post Malone 9.6k/config2.json @@ -0,0 +1,94 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 800, + "seed": 1234, + "epochs": 10000, + "learning_rate": 0.0001, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, + "batch_size": 6, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 10240, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 512, + "port": "8001", + "keep_ckpts": 3, + "all_in_mem": false + }, + "data": { + "training_files": "filelists/train.txt", + "validation_files": "filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 44100, + "filter_length": 2048, + "hop_length": 512, + "win_length": 2048, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 22050 + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [ + 3, + 7, + 11 + ], + "resblock_dilation_sizes": [ + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ] + ], + "upsample_rates": [ + 8, + 8, + 2, + 2, + 2 + ], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [ + 16, + 16, + 4, + 4, + 4 + ], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 256, + "n_speakers": 1 + }, + "spk": { + "PostMalone": 0 + } +} \ No newline at end of file diff --git a/model/Post Malone Test/PostMalone.pth b/model/Post Malone Test/PostMalone.pth new file mode 100644 index 0000000000000000000000000000000000000000..b21065561c356394366c3bbdbde1c220dd61fabb --- /dev/null +++ b/model/Post Malone Test/PostMalone.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b7472dfc3e7c789f8667147529f4900f792760d869069d39915a8128c9dab1f +size 55028048 diff --git a/model/Post Malone Test/added_IVF382_Flat_nprobe_5.index b/model/Post Malone Test/added_IVF382_Flat_nprobe_5.index new file mode 100644 index 0000000000000000000000000000000000000000..bd9b43bc315146ff68bf5b005c14172059ad5e01 --- /dev/null +++ b/model/Post Malone Test/added_IVF382_Flat_nprobe_5.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69791a0f49eb27f86128db0721848b0aeec83db0fe51cf8d9b72526a68343d29 +size 15769099 diff --git a/model/Post Malone Test/config.json b/model/Post Malone Test/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3e7ea9468493ac0b1d56f6f29e792936fcd6a569 --- /dev/null +++ b/model/Post Malone Test/config.json @@ -0,0 +1,11 @@ +{ + "model": "PostMalone.pth", + "feat_index": "added_IVF382_Flat_nprobe_5.index", + "feat_npy": "", + "speaker_id": 0, + "name": "PostMalone", + "author": "Rubin", + "source": "Rubin", + "note": "postmalone", + "icon": "" +} diff --git a/model/The Weekend/TheWeekend.pth b/model/The Weekend/TheWeekend.pth new file mode 100644 index 0000000000000000000000000000000000000000..f53674695a02c26416739b9aabd2fdba286cea29 --- /dev/null +++ b/model/The Weekend/TheWeekend.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0732e4c018f2295bf68f23beb21645191f0e6cec07aafd29da6fdf1d3092b072 +size 55118451 diff --git a/model/The Weekend/added_IVF797_Flat_nprobe_1.index b/model/The Weekend/added_IVF797_Flat_nprobe_1.index new file mode 100644 index 0000000000000000000000000000000000000000..0177056e2b88bbef2138f44e22a6bbeca755db62 --- /dev/null +++ b/model/The Weekend/added_IVF797_Flat_nprobe_1.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbdf9263febc5bd87b66b51e490fd05efc9d13336a28761a7b9911ad4c2f4e55 +size 32900299 diff --git a/model/The Weekend/config.json b/model/The Weekend/config.json new file mode 100644 index 0000000000000000000000000000000000000000..436fa881daac97489e9d0faf00b898c5a13de801 --- /dev/null +++ b/model/The Weekend/config.json @@ -0,0 +1,11 @@ +{ + "model": "TheWeekend.pth", + "feat_index": "added_IVF797_Flat_nprobe_1.index", + "feat_npy": "", + "speaker_id": 0, + "name": "TheWeekend", + "author": "Rubin", + "source": "Rubin", + "note": "The Weekend", + "icon": "" +} diff --git a/model/Travis Scott 100k/Travis Scott.pth b/model/Travis Scott 100k/Travis Scott.pth new file mode 100644 index 0000000000000000000000000000000000000000..1a479ed849b5b54876879a79a78153872dc93a6c --- /dev/null +++ b/model/Travis Scott 100k/Travis Scott.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924cc2d2d3c902054b37acbd9e28952f8550bde3e2c08708273f88b71d69c655 +size 542792859 diff --git a/model/Travis Scott 100k/config.json b/model/Travis Scott 100k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c650225fe6a642973aacf327855b78a3ca7cfc75 --- /dev/null +++ b/model/Travis Scott 100k/config.json @@ -0,0 +1,11 @@ +{ + "model": "Travis Scott.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + "name": "Travis Scott", + "author": "Rubin", + "source": "Rubin", + "note": "Travis Scott 100k", + "icon": "" +} diff --git a/model/Travis Scott 100k/config2.json b/model/Travis Scott 100k/config2.json new file mode 100644 index 0000000000000000000000000000000000000000..160f5e2c2ea31c9c1ada7199a9bd465ae0b7be46 --- /dev/null +++ b/model/Travis Scott 100k/config2.json @@ -0,0 +1,39 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 800, + "seed": 1234, + "epochs": 10000, + "learning_rate": 0.0001, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, + "batch_size": 6, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 10240, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 512, + "port": "8001", + "keep_ckpts": 3 + }, + "data": { + "training_files": "filelists/train.txt", + "validation_files": "filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 44100, + "filter_length": 2048, + "hop_length": 512, + "win_length": 2048, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 22050 + }, + "model": { + } \ No newline at end of file diff --git a/model/Travis Scott 6720/Travis Scott.pth b/model/Travis Scott 6720/Travis Scott.pth new file mode 100644 index 0000000000000000000000000000000000000000..54d0cd0e1c663029b5c22167d3052b7653adadff --- /dev/null +++ b/model/Travis Scott 6720/Travis Scott.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df059a2d63e6036ea3beb3cea2f9662de868257ef75748db5c9f13ba9419ee59 +size 55027589 diff --git a/model/Travis Scott 6720/added_IVF677_Flat_nprobe_7.index b/model/Travis Scott 6720/added_IVF677_Flat_nprobe_7.index new file mode 100644 index 0000000000000000000000000000000000000000..dc193b2f60d082898f8e5626a36a6b87d7164721 --- /dev/null +++ b/model/Travis Scott 6720/added_IVF677_Flat_nprobe_7.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f520d1aaaa2105330f400530dc54066eb94c5080faa5a635126bc0c3624fda +size 27953923 diff --git a/model/Travis Scott 6720/config.json b/model/Travis Scott 6720/config.json new file mode 100644 index 0000000000000000000000000000000000000000..afc8861e7d4a35dcc605085f1d1439220170758b --- /dev/null +++ b/model/Travis Scott 6720/config.json @@ -0,0 +1,11 @@ +{ + "model": "Travis Scott.pth", + "feat_index": "added_IVF677_Flat_nprobe_7.index", + "feat_npy": "total_fea.npy", + "speaker_id": 0, + "name": "Travis Scott", + "author": "Rubin", + "source": "Rubin", + "note": "Travis Scott 6720", + "icon": "" +} diff --git a/model/Travis Scott 6720/total_fea.npy b/model/Travis Scott 6720/total_fea.npy new file mode 100644 index 0000000000000000000000000000000000000000..f21aea91f872a43a52661607e74048765859a741 --- /dev/null +++ b/model/Travis Scott 6720/total_fea.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec1da1cfd5387b6e28498dc1114152f57ae50dc412b452c98c88acf8ad533c53 +size 27043968 diff --git a/model/Travis Scott 77k/Travis Scott.pth b/model/Travis Scott 77k/Travis Scott.pth new file mode 100644 index 0000000000000000000000000000000000000000..6834cf09de635d8d70a56f005b0583429c7124cc --- /dev/null +++ b/model/Travis Scott 77k/Travis Scott.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5920bb955140b165d70bf8bb02efd8593f3a1ae7a7cc2492a12cec757f5fa7e8 +size 542789405 diff --git a/model/Travis Scott 77k/config.json b/model/Travis Scott 77k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2502e4980f1956c33e8e8603a4c1db165b3c7b06 --- /dev/null +++ b/model/Travis Scott 77k/config.json @@ -0,0 +1,11 @@ +{ + "model": "Travis Scott.pth", + "feat_index": "", + "feat_npy": "", + "speaker_id": 0, + "name": "Travis Scott", + "author": "Rubin", + "source": "Rubin", + "note": "Travis Scott 77k", + "icon": "" +} diff --git a/model/Travis Scott 77k/config2.json b/model/Travis Scott 77k/config2.json new file mode 100644 index 0000000000000000000000000000000000000000..74aed0c59aaa0e8c03e35c85b01c943fa000f87c --- /dev/null +++ b/model/Travis Scott 77k/config2.json @@ -0,0 +1,93 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 1000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 0.0001, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, + "batch_size": 6, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 10240, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 512, + "port": "8001", + "keep_ckpts": 3 + }, + "data": { + "training_files": "filelists/train.txt", + "validation_files": "filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 44100, + "filter_length": 2048, + "hop_length": 512, + "win_length": 2048, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 22050 + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [ + 3, + 7, + 11 + ], + "resblock_dilation_sizes": [ + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ] + ], + "upsample_rates": [ + 8, + 8, + 2, + 2, + 2 + ], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [ + 16, + 16, + 4, + 4, + 4 + ], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 256, + "n_speakers": 200 + }, + "spk": { + "travis01": 0 + } +} \ No newline at end of file diff --git a/model/Travis Scott 77k/kmeans_4000.pt b/model/Travis Scott 77k/kmeans_4000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f1571196dfd40e5cfe518930da83951a75b917b --- /dev/null +++ b/model/Travis Scott 77k/kmeans_4000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e758bf3a9d2352e099d0872e2cc5115c5b338ffd0af56d8c99fb84b2a84c3c5c +size 6167543 diff --git a/model/Travis Scott/config.json b/model/Travis Scott/config.json new file mode 100644 index 0000000000000000000000000000000000000000..40b909579b0212375f504a5edba15d3a4ad816c8 --- /dev/null +++ b/model/Travis Scott/config.json @@ -0,0 +1,11 @@ +{ + "model": "travisscott.pth", + "feat_index": "trained_IVF1542_Flat_nprobe_9.index", + "feat_npy": "total_fea.npy", + "speaker_id": 0, + "name": "travisscott", + "author": "Rubin", + "source": "Rubin", + "note": "Travis Scott", + "icon": "" +} diff --git a/model/Travis Scott/total_fea.npy b/model/Travis Scott/total_fea.npy new file mode 100644 index 0000000000000000000000000000000000000000..5a12d8c58a2503cdc374741d9415f855d0bb8451 --- /dev/null +++ b/model/Travis Scott/total_fea.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2180243b1acb4001576bcb5e789f5f9ac3ee42498afe03cd7ab66e4d56806cf2 +size 61610112 diff --git a/model/Travis Scott/trained_IVF1542_Flat_nprobe_9.index b/model/Travis Scott/trained_IVF1542_Flat_nprobe_9.index new file mode 100644 index 0000000000000000000000000000000000000000..82aaaa6251b72cda99aba3b0f5e1aa9bf4da9239 --- /dev/null +++ b/model/Travis Scott/trained_IVF1542_Flat_nprobe_9.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2143e13f17fdb727f7567f68ba3364a6011fae4c21cac3f21c4383af504303 +size 1579147 diff --git a/model/Travis Scott/travisscott.pth b/model/Travis Scott/travisscott.pth new file mode 100644 index 0000000000000000000000000000000000000000..e477963d2226072429089676a5627435bb2509fa --- /dev/null +++ b/model/Travis Scott/travisscott.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3636c951d92b4923cf0f26305e2af92d1e961e4330009bbd39bad669db647f7d +size 55127121 diff --git a/model/angele/Angele.png b/model/angele/Angele.png new file mode 100644 index 0000000000000000000000000000000000000000..9a97378d2f42d981456585cb9ef060137d0ef1dd --- /dev/null +++ b/model/angele/Angele.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc8b91d593b67833db8b2a0203183b846c1e7789d0a23f662dddb697491eb62 +size 1277423 diff --git a/model/angele/added_IVF1260_Flat_nprobe_1_v2.index b/model/angele/added_IVF1260_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..b115a84c2720cef799011ca1ec793fe9e7056433 --- /dev/null +++ b/model/angele/added_IVF1260_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4d58e31eefe7837897dd1a1091e61deda2124a1b85594b7d76d7bf97a5f3aa +size 155269099 diff --git a/model/angele/angele.pth b/model/angele/angele.pth new file mode 100644 index 0000000000000000000000000000000000000000..514fc9f7e660e51f582335256ca5ded40404f042 --- /dev/null +++ b/model/angele/angele.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43d2990f49e108eb512ce00ba76866850d6c7270cb8ef39a42cfc9995c9a85d9 +size 55216420 diff --git a/model/angele/config.json b/model/angele/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0daed2328c8a4279fbfb0259ecefecbe43aa1d14 --- /dev/null +++ b/model/angele/config.json @@ -0,0 +1,11 @@ +{ + "model": "angele.pth", + "feat_index": "added_IVF1260_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Angele", + "author": "Aalex", + "source": "All", + "note": "Trained in RVC v2, All the album", + "icon": "Angele.png" +} \ No newline at end of file diff --git a/model/arianagrande/Ariana.png b/model/arianagrande/Ariana.png new file mode 100644 index 0000000000000000000000000000000000000000..3cce2f6789ba6be94a4460aab571fcb9a09cb7d0 Binary files /dev/null and b/model/arianagrande/Ariana.png differ diff --git a/model/arianagrande/added_IVF1033_Flat_nprobe_1_v2.index b/model/arianagrande/added_IVF1033_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..cdb3d2d4ca3bcd9a30c20cc67cd55eac957f18a9 --- /dev/null +++ b/model/arianagrande/added_IVF1033_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a47bef476fc29dde668b2727ad4ba3dbcf526a62bcea85204595d28b0b854bbb +size 127336579 diff --git a/model/arianagrande/arianagrande.pth b/model/arianagrande/arianagrande.pth new file mode 100644 index 0000000000000000000000000000000000000000..fd6b46d9559d663f6e06ca7eb379014b85f73ff6 --- /dev/null +++ b/model/arianagrande/arianagrande.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f8199a3f13fed7d6f71d98e89c8bf49cbc00701e0d6581383e320997fd8ed20 +size 55226492 diff --git a/model/arianagrande/config.json b/model/arianagrande/config.json new file mode 100644 index 0000000000000000000000000000000000000000..137af6c1ffce0b768d99189da491550f99fa01ab --- /dev/null +++ b/model/arianagrande/config.json @@ -0,0 +1,11 @@ +{ + "model": "arianagrande.pth", + "feat_index": "added_IVF1033_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Ariana Grande", + "author": "Arithyst", + "source": "ALL", + "note": "7 minute dataset (All of the dataset are from her Pro-Tools Dataset), Trained in RVC v2, Crepe Hop Length - 30", + "icon": "Ariana.png" +} \ No newline at end of file diff --git a/model/ayaka-jp/added_IVF1830_Flat_nprobe_9.index b/model/ayaka-jp/added_IVF1830_Flat_nprobe_9.index new file mode 100644 index 0000000000000000000000000000000000000000..c9b4a6d83689065f3c59c00a3a8e5099d67bec04 --- /dev/null +++ b/model/ayaka-jp/added_IVF1830_Flat_nprobe_9.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8029f85a9f8e39c0dde55d37bdedaef919b0fb6f1ae7bbeb2e6807395e06673c +size 75570403 diff --git a/model/ayaka-jp/ayaka-jp.pth b/model/ayaka-jp/ayaka-jp.pth new file mode 100644 index 0000000000000000000000000000000000000000..6016802be9f0bf05d0040006967714ca4baa50fc --- /dev/null +++ b/model/ayaka-jp/ayaka-jp.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a53fc9d3534f87803f435909b98f9b0f6ce762cdaae6935b4c8323b017bd795e +size 55029425 diff --git a/model/ayaka-jp/config.json b/model/ayaka-jp/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cce8bfb7274dc5f1f148f152744acf0bf721bec8 --- /dev/null +++ b/model/ayaka-jp/config.json @@ -0,0 +1,12 @@ +{ + "model": "ayaka-jp.pth", + "feat_index": "added_IVF1830_Flat_nprobe_9.index", + "feat_npy": "total_fea.npy", + "speaker_id": 0, + + "name": "ayaka-jp", + "author": "Rubin", + "source": "Rubin", + "note": "", + "icon": "" +} diff --git a/model/ayaka-jp/cover.png b/model/ayaka-jp/cover.png new file mode 100644 index 0000000000000000000000000000000000000000..eea64959df3fd0988b023f9efb526124c174f93e Binary files /dev/null and b/model/ayaka-jp/cover.png differ diff --git a/model/ayaka-jp/total_fea.npy b/model/ayaka-jp/total_fea.npy new file mode 100644 index 0000000000000000000000000000000000000000..573feba17f175e4e577b3d119931cd45b84c59fd --- /dev/null +++ b/model/ayaka-jp/total_fea.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8802ce1a106272f6cb35195f7890935bd375e0571a7b07b8a354e543eeb77032 +size 73110656 diff --git a/model/captainprice/Captain.png b/model/captainprice/Captain.png new file mode 100644 index 0000000000000000000000000000000000000000..6ab0290878a3c1d112db54aa4b893bca2c087d4d Binary files /dev/null and b/model/captainprice/Captain.png differ diff --git a/model/captainprice/CaptainPrice-BarrySloane.pth b/model/captainprice/CaptainPrice-BarrySloane.pth new file mode 100644 index 0000000000000000000000000000000000000000..af2a383bca72f6772207cdc73a8d2bf4bfb301a6 --- /dev/null +++ b/model/captainprice/CaptainPrice-BarrySloane.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2021f6670376ad323ea85ba70b40edca1bee6535d4065f1d65691eb6d84aabbb +size 55233441 diff --git a/model/captainprice/added_IVF3998_Flat_nprobe_1_v2.index b/model/captainprice/added_IVF3998_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..8ffe9a670c16d9e78f72232a7374576e436e0edc --- /dev/null +++ b/model/captainprice/added_IVF3998_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6ac7b8877ec4388b3f64e4aa7f8e8f2c4df0642905f9daf2e25bd2a3069100 +size 492661539 diff --git a/model/captainprice/config.json b/model/captainprice/config.json new file mode 100644 index 0000000000000000000000000000000000000000..992f0c8a3ccb44283eb6c7632bd921037aac6341 --- /dev/null +++ b/model/captainprice/config.json @@ -0,0 +1,11 @@ +{ + "model": "CaptainPrice-BarrySloane.pth", + "feat_index": "added_IVF3998_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Captain Price", + "author": "McMessenger", + "source": "MW", + "note": "Trained in RVC v2 (150 epochs), 52 minutes of in-game vocal lines (and some cutscenes) pulled directly from MW2019, BOCW, and MWII.", + "icon": "Captain.png" +} \ No newline at end of file diff --git a/model/damso/Damso.png b/model/damso/Damso.png new file mode 100644 index 0000000000000000000000000000000000000000..eb87859501c9adbfdd191945e4dd888bbf00dc92 Binary files /dev/null and b/model/damso/Damso.png differ diff --git a/model/damso/added_IVF1084_Flat_nprobe_1_v2.index b/model/damso/added_IVF1084_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..b1c61cf2af995e8a3adccf54a2a8526e2eca43b4 --- /dev/null +++ b/model/damso/added_IVF1084_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94685c7906bbcd462f846e9a6fcc4c72c382e1feb5db39c46dc6acf673f7f03f +size 133641339 diff --git a/model/damso/config.json b/model/damso/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e35a62bbf332f8c967db0f083cebee0a7944cf9 --- /dev/null +++ b/model/damso/config.json @@ -0,0 +1,11 @@ +{ + "model": "damso.pth", + "feat_index": "added_IVF1084_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Damso", + "author": "BartPoint", + "source": "ALL", + "note": "Trained in RVC v2, Crepes 128, 600 epochs", + "icon": "Damso.png" +} \ No newline at end of file diff --git a/model/damso/damso.pth b/model/damso/damso.pth new file mode 100644 index 0000000000000000000000000000000000000000..663f84e406b2770c2c2843b806e7db447977b450 --- /dev/null +++ b/model/damso/damso.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:083a9d7bc3e09c3b40de5f26197fe8b77794fe55e2eec3e2d4d434f990207966 +size 55193241 diff --git a/model/leto/Leto.png b/model/leto/Leto.png new file mode 100644 index 0000000000000000000000000000000000000000..331c77850dac3456365a9b883b76597bd1dd4195 --- /dev/null +++ b/model/leto/Leto.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a4e56fb45c003bb3f62cb9513aca3395528f7296bad6df3f3878c77ba37f577 +size 2272754 diff --git a/model/leto/added_IVF408_Flat_nprobe_1_v2.index b/model/leto/added_IVF408_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..9d00fe3277f0c51b2e2e069aca5bee6d2f24aae4 --- /dev/null +++ b/model/leto/added_IVF408_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb3ba7fad42b6c25a3d0019d84fd017851724aaf7bfcf80861cb9abceaf232e2 +size 50296539 diff --git a/model/leto/config.json b/model/leto/config.json new file mode 100644 index 0000000000000000000000000000000000000000..693c8fd60d1eb43279d7cfa7220224c32fd25e3a --- /dev/null +++ b/model/leto/config.json @@ -0,0 +1,11 @@ +{ + "model": "leto.pth", + "feat_index": "added_IVF408_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Leto", + "author": "BartPoint", + "source": "Album", + "note": "Trained in RVC v2, Crepes 128, 600 epochs", + "icon": "Leto.png" +} \ No newline at end of file diff --git a/model/leto/leto.pth b/model/leto/leto.pth new file mode 100644 index 0000000000000000000000000000000000000000..6b6948dd9e80012b642a798aa069cf335ec219ac --- /dev/null +++ b/model/leto/leto.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:357488113dc890fdb15d68672d34e847a3355b58548e7bb3071aa1dec268b201 +size 55216420 diff --git a/model/macmiller/added_IVF2124_Flat_nprobe_1_v2.index b/model/macmiller/added_IVF2124_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..da45d7561b7ae532b1ec7fa74b6aab3c0c06e14b --- /dev/null +++ b/model/macmiller/added_IVF2124_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a83907d49984646512ee119fb5ee8e2402d45795c8d7244981820ce6d14dc11 +size 261741619 diff --git a/model/macmiller/config.json b/model/macmiller/config.json new file mode 100644 index 0000000000000000000000000000000000000000..33daa829451df1e205fc1c7fd4cbf6eb49221e34 --- /dev/null +++ b/model/macmiller/config.json @@ -0,0 +1,11 @@ +{ + "model": "macmillerv3.pth", + "feat_index": "added_IVF2124_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "MacMiller", + "source": "ALL", + "author": "HZY", + "note": "Trained on Crepe, 28 minute data-set, 300 epochs", + "icon": "macmiller.png" +} diff --git a/model/macmiller/macmiller.png b/model/macmiller/macmiller.png new file mode 100644 index 0000000000000000000000000000000000000000..9e956c70e09465f4d29d73663d4b089b00d3b23c --- /dev/null +++ b/model/macmiller/macmiller.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1b4b3d0e625e4c0045f678b312463ec185b9ea36a241931049c5d363a7fe00f +size 2138638 diff --git a/model/macmiller/macmillerv3.pth b/model/macmiller/macmillerv3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c5a7dcc34219d3b54de370eaf57a1938573697a2 --- /dev/null +++ b/model/macmiller/macmillerv3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4b2e20687c0288c79b1ddbb07107c9c00e458af46f2e62f6c713328a459ae3e +size 55225115 diff --git a/model/mickaeljackson/Mickael.png b/model/mickaeljackson/Mickael.png new file mode 100644 index 0000000000000000000000000000000000000000..f5af872afa5606b8e905251c08ac268449812737 Binary files /dev/null and b/model/mickaeljackson/Mickael.png differ diff --git a/model/mickaeljackson/added_IVF1448_Flat_nprobe_1_v2.index b/model/mickaeljackson/added_IVF1448_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..b09cb79e4bb05f119ae8890096241d2372b78718 --- /dev/null +++ b/model/mickaeljackson/added_IVF1448_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f98424e9a36aa1fc3f753ba1fcded6f6992c8094cd383576c5af17aa8c9d301 +size 178421459 diff --git a/model/mickaeljackson/config.json b/model/mickaeljackson/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ed561b714f425ad4ad7320acb5cce8a64ea995a2 --- /dev/null +++ b/model/mickaeljackson/config.json @@ -0,0 +1,11 @@ +{ + "model": "michael-jackson.pth", + "feat_index": "added_IVF1448_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Mickael Jackson", + "author": "REU Music", + "source": "ALL", + "note": "Trained by me with a 20 minute dataset, based on ERA Off the Wall + Thriller", + "icon": "Mickael.png" +} \ No newline at end of file diff --git a/model/mickaeljackson/michael-jackson.pth b/model/mickaeljackson/michael-jackson.pth new file mode 100644 index 0000000000000000000000000000000000000000..ef4bf22eaf035d9a1dbeee1bd76996e0ea99d9e7 --- /dev/null +++ b/model/mickaeljackson/michael-jackson.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae7aecdd7fb262d2f38c24a24ad167cf7ac45e05437b51e1df0fe37d22e75f2f +size 55229246 diff --git a/model/oboy/Oboy.png b/model/oboy/Oboy.png new file mode 100644 index 0000000000000000000000000000000000000000..a0d7d1d3be6273de9496591c69dc1fcb3efdb01e Binary files /dev/null and b/model/oboy/Oboy.png differ diff --git a/model/oboy/added_IVF419_Flat_nprobe_1_v2.index b/model/oboy/added_IVF419_Flat_nprobe_1_v2.index new file mode 100644 index 0000000000000000000000000000000000000000..5f32762b656026ee87ca6187c6fa82d7376e5b1b --- /dev/null +++ b/model/oboy/added_IVF419_Flat_nprobe_1_v2.index @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb6b348e7ccac14ba8fe1ff264d152393a6c07e44abae257744053a3c17fa457 +size 51642499 diff --git a/model/oboy/config.json b/model/oboy/config.json new file mode 100644 index 0000000000000000000000000000000000000000..309fe7b798d3f40e0024c95115a44c5f7555951b --- /dev/null +++ b/model/oboy/config.json @@ -0,0 +1,11 @@ +{ + "model": "oboy.pth", + "feat_index": "added_IVF419_Flat_nprobe_1_v2.index", + "speaker_id": 0, + + "name": "Oboy", + "author": "BartPoint", + "source": "ALL", + "note": "(Dataset not by BartPoint), Trained in RVC v2, Crepe 78", + "icon": "Oboy.png" +} \ No newline at end of file diff --git a/model/oboy/oboy.pth b/model/oboy/oboy.pth new file mode 100644 index 0000000000000000000000000000000000000000..e8c7f3de8056b7d1dba4a0233050421bfa13fc8c --- /dev/null +++ b/model/oboy/oboy.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f1308db21eae7336098479e562d3d6d27eff2329b21c772a44f74d5d9c9f43 +size 55192782 diff --git a/model/update_config.sh b/model/update_config.sh new file mode 100644 index 0000000000000000000000000000000000000000..fb3b1112f93b6e8531f593094d44808289af40ee --- /dev/null +++ b/model/update_config.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Function to check if config.json already exists in a subfolder and if an image file exists +check_exists() { + local folder="$1" + local config_file="$folder/config.json" + local icon_file=$(find "$folder" -maxdepth 1 -type f \( -iname "*.jpg" -o -iname "*.gif" -o -iname "*.png" \) -print -quit) + + if [ -f "$config_file" ]; then + if [ -n "$icon_file" ]; then + icon=$(basename "$icon_file") + fi + return 0 + else + return 1 + fi +} + +# Function to update config.json if there are missing fields +update_config() { + local folder="$1" + local config_file="$folder/config.json" + + local model_file=$(find "$folder" -maxdepth 1 -type f -name "*.pth" -exec basename {} \; -quit) + local feat_index_file=$(find "$folder" -maxdepth 1 -type f -name "*.index" -exec basename {} \; -quit) + local feat_npy_file=$(find "$folder" -maxdepth 1 -type f -name "*.npy" -exec basename {} \; -quit) + + # Remove the .pth extension from the model file name + local name=$(basename "$model_file" .pth) + + # Replace underscores with spaces, remove "RVC" or "(RVC)", "Epoch", square brackets, parentheses, and curly brackets from the folder name, + # and replace any occurrences of consecutive spaces with a single space + local note=$(basename "$folder" | sed 's/_/ /g; s/\(RVC\)//g; s/RVC//g; s/[(){}]//g; s/\[\]//g; s/Epoch//g; s/ */ /g; s/^ //; s/ $//') + + # Set other configuration values + local speaker_id=0 + local author="Rubin" + local source="Rubin" + local icon="" + + if check_exists "$folder"; then + if [ -n "$icon" ]; then + icon=$(basename "$icon_file") + fi + + # Update the fields for the existing config.json + # Creating a new json string with the updated fields + local new_config=$(cat < "$config_file" + echo "Updated config.json file in $folder" + else + # Create the config.json file + cat > "$folder/config.json" <=0.3.2 +faiss-cpu==1.7.3 +praat-parselmouth>=0.4.2 +librosa==0.9.1 +edge-tts +torchcrepe==0.0.18 diff --git a/util.py b/util.py new file mode 100644 index 0000000000000000000000000000000000000000..8d6bcff1135c2d97e4caad7922f03f05c98484da --- /dev/null +++ b/util.py @@ -0,0 +1,81 @@ +import sys +import asyncio +from io import BytesIO + +from fairseq import checkpoint_utils + +import torch + +import edge_tts +import librosa + + +# https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI/blob/main/config.py#L43-L55 # noqa +def has_mps() -> bool: + if sys.platform != "darwin": + return False + else: + if not getattr(torch, 'has_mps', False): + return False + + try: + torch.zeros(1).to(torch.device("mps")) + return True + except Exception: + return False + + +def is_half(device: str) -> bool: + if not device.startswith('cuda'): + return False + else: + gpu_name = torch.cuda.get_device_name( + int(device.split(':')[-1]) + ).upper() + + # ...regex? + if ( + ('16' in gpu_name and 'V100' not in gpu_name) + or 'P40' in gpu_name + or '1060' in gpu_name + or '1070' in gpu_name + or '1080' in gpu_name + ): + return False + + return True + + +def load_hubert_model(device: str, model_path: str = 'hubert_base.pt'): + model = checkpoint_utils.load_model_ensemble_and_task( + [model_path] + )[0][0].to(device) + + if is_half(device): + return model.half() + else: + return model.float() + + +async def call_edge_tts(speaker_name: str, text: str): + tts_com = edge_tts.Communicate(text, speaker_name) + tts_raw = b'' + + # Stream TTS audio to bytes + async for chunk in tts_com.stream(): + if chunk['type'] == 'audio': + tts_raw += chunk['data'] + + # Convert mp3 stream to wav + ffmpeg_proc = await asyncio.create_subprocess_exec( + 'ffmpeg', + '-f', 'mp3', + '-i', '-', + '-f', 'wav', + '-', + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE + ) + (tts_wav, _) = await ffmpeg_proc.communicate(tts_raw) + + return librosa.load(BytesIO(tts_wav)) diff --git a/vc_infer_pipeline.py b/vc_infer_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..8272b4642ac16d063caf75867cb01d4a6a9bd6a9 --- /dev/null +++ b/vc_infer_pipeline.py @@ -0,0 +1,385 @@ +import numpy as np, parselmouth, torch, pdb +from time import time as ttime +import torch.nn.functional as F +import scipy.signal as signal +import pyworld, os, traceback, faiss, librosa, torchcrepe +from scipy import signal +from functools import lru_cache + +bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=16000) + +input_audio_path2wav={} + +@lru_cache +def cache_harvest_f0(input_audio_path,fs,f0max,f0min,frame_period): + audio=input_audio_path2wav[input_audio_path] + f0, t = pyworld.harvest( + audio, + fs=fs, + f0_ceil=f0max, + f0_floor=f0min, + frame_period=frame_period, + ) + f0 = pyworld.stonemask(audio, f0, t, fs) + return f0 + +def change_rms(data1,sr1,data2,sr2,rate):#1是输入音频，2是输出音频,rate是2的占比 + # print(data1.max(),data2.max()) + rms1 = librosa.feature.rms(y=data1, frame_length=sr1//2*2, hop_length=sr1//2)#每半秒一个点 + rms2 = librosa.feature.rms(y=data2, frame_length=sr2//2*2, hop_length=sr2//2) + rms1=torch.from_numpy(rms1) + rms1=F.interpolate(rms1.unsqueeze(0), size=data2.shape[0],mode='linear').squeeze() + rms2=torch.from_numpy(rms2) + rms2=F.interpolate(rms2.unsqueeze(0), size=data2.shape[0],mode='linear').squeeze() + rms2=torch.max(rms2,torch.zeros_like(rms2)+1e-6) + data2*=(torch.pow(rms1,torch.tensor(1-rate))*torch.pow(rms2,torch.tensor(rate-1))).numpy() + return data2 + +class VC(object): + def __init__(self, tgt_sr, config): + self.x_pad, self.x_query, self.x_center, self.x_max, self.is_half = ( + config.x_pad, + config.x_query, + config.x_center, + config.x_max, + config.is_half, + ) + self.sr = 16000 # hubert输入采样率 + self.window = 160 # 每帧点数 + self.t_pad = self.sr * self.x_pad # 每条前后pad时间 + self.t_pad_tgt = tgt_sr * self.x_pad + self.t_pad2 = self.t_pad * 2 + self.t_query = self.sr * self.x_query # 查询切点前后查询时间 + self.t_center = self.sr * self.x_center # 查询切点位置 + self.t_max = self.sr * self.x_max # 免查询时长阈值 + self.device = config.device + + def get_f0(self, input_audio_path,x, p_len, f0_up_key, f0_method,filter_radius, inp_f0=None): + global input_audio_path2wav + time_step = self.window / self.sr * 1000 + f0_min = 50 + f0_max = 1100 + f0_mel_min = 1127 * np.log(1 + f0_min / 700) + f0_mel_max = 1127 * np.log(1 + f0_max / 700) + if f0_method == "pm": + f0 = ( + parselmouth.Sound(x, self.sr) + .to_pitch_ac( + time_step=time_step / 1000, + voicing_threshold=0.6, + pitch_floor=f0_min, + pitch_ceiling=f0_max, + ) + .selected_array["frequency"] + ) + pad_size = (p_len - len(f0) + 1) // 2 + if pad_size > 0 or p_len - len(f0) - pad_size > 0: + f0 = np.pad( + f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant" + ) + elif f0_method == "harvest": + input_audio_path2wav[input_audio_path]=x.astype(np.double) + f0=cache_harvest_f0(input_audio_path,self.sr,f0_max,f0_min,10) + if(filter_radius>2): + f0 = signal.medfilt(f0, 3) + elif f0_method == "crepe": + model = "full" + # Pick a batch size that doesn't cause memory errors on your gpu + batch_size = 512 + # Compute pitch using first gpu + audio = torch.tensor(np.copy(x))[None].float() + f0, pd = torchcrepe.predict( + audio, + self.sr, + self.window, + f0_min, + f0_max, + model, + batch_size=batch_size, + device=self.device, + return_periodicity=True, + ) + pd = torchcrepe.filter.median(pd, 3) + f0 = torchcrepe.filter.mean(f0, 3) + f0[pd < 0.1] = 0 + f0 = f0[0].cpu().numpy() + f0 *= pow(2, f0_up_key / 12) + # with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()])) + tf0 = self.sr // self.window # 每秒f0点数 + if inp_f0 is not None: + delta_t = np.round( + (inp_f0[:, 0].max() - inp_f0[:, 0].min()) * tf0 + 1 + ).astype("int16") + replace_f0 = np.interp( + list(range(delta_t)), inp_f0[:, 0] * 100, inp_f0[:, 1] + ) + shape = f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)].shape[0] + f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)] = replace_f0[ + :shape + ] + # with open("test_opt.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()])) + f0bak = f0.copy() + f0_mel = 1127 * np.log(1 + f0 / 700) + f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / ( + f0_mel_max - f0_mel_min + ) + 1 + f0_mel[f0_mel <= 1] = 1 + f0_mel[f0_mel > 255] = 255 + f0_coarse = np.rint(f0_mel).astype(int) + return f0_coarse, f0bak # 1-0 + + def vc( + self, + model, + net_g, + sid, + audio0, + pitch, + pitchf, + times, + index, + big_npy, + index_rate, + version, + ): # ,file_index,file_big_npy + feats = torch.from_numpy(audio0) + if self.is_half: + feats = feats.half() + else: + feats = feats.float() + if feats.dim() == 2: # double channels + feats = feats.mean(-1) + assert feats.dim() == 1, feats.dim() + feats = feats.view(1, -1) + padding_mask = torch.BoolTensor(feats.shape).to(self.device).fill_(False) + + inputs = { + "source": feats.to(self.device), + "padding_mask": padding_mask, + "output_layer": 9 if version == "v1" else 12, + } + t0 = ttime() + with torch.no_grad(): + logits = model.extract_features(**inputs) + feats = model.final_proj(logits[0])if version=="v1"else logits[0] + + if ( + isinstance(index, type(None)) == False + and isinstance(big_npy, type(None)) == False + and index_rate != 0 + ): + npy = feats[0].cpu().numpy() + if self.is_half: + npy = npy.astype("float32") + + # _, I = index.search(npy, 1) + # npy = big_npy[I.squeeze()] + + score, ix = index.search(npy, k=8) + weight = np.square(1 / score) + weight /= weight.sum(axis=1, keepdims=True) + npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1) + + if self.is_half: + npy = npy.astype("float16") + feats = ( + torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate + + (1 - index_rate) * feats + ) + + feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) + t1 = ttime() + p_len = audio0.shape[0] // self.window + if feats.shape[1] < p_len: + p_len = feats.shape[1] + if pitch != None and pitchf != None: + pitch = pitch[:, :p_len] + pitchf = pitchf[:, :p_len] + p_len = torch.tensor([p_len], device=self.device).long() + with torch.no_grad(): + if pitch != None and pitchf != None: + audio1 = ( + (net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]) + .data.cpu() + .float() + .numpy() + ) + else: + audio1 = ( + (net_g.infer(feats, p_len, sid)[0][0, 0]) + .data.cpu() + .float() + .numpy() + ) + del feats, p_len, padding_mask + if torch.cuda.is_available(): + torch.cuda.empty_cache() + t2 = ttime() + times[0] += t1 - t0 + times[2] += t2 - t1 + return audio1 + + def pipeline( + self, + model, + net_g, + sid, + audio, + input_audio_path, + times, + f0_up_key, + f0_method, + file_index, + # file_big_npy, + index_rate, + if_f0, + filter_radius, + tgt_sr, + resample_sr, + rms_mix_rate, + version, + f0_file=None, + ): + if ( + file_index != "" + # and file_big_npy != "" + # and os.path.exists(file_big_npy) == True + and os.path.exists(file_index) == True + and index_rate != 0 + ): + try: + index = faiss.read_index(file_index) + # big_npy = np.load(file_big_npy) + big_npy = index.reconstruct_n(0, index.ntotal) + except: + traceback.print_exc() + index = big_npy = None + else: + index = big_npy = None + audio = signal.filtfilt(bh, ah, audio) + audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect") + opt_ts = [] + if audio_pad.shape[0] > self.t_max: + audio_sum = np.zeros_like(audio) + for i in range(self.window): + audio_sum += audio_pad[i : i - self.window] + for t in range(self.t_center, audio.shape[0], self.t_center): + opt_ts.append( + t + - self.t_query + + np.where( + np.abs(audio_sum[t - self.t_query : t + self.t_query]) + == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min() + )[0][0] + ) + s = 0 + audio_opt = [] + t = None + t1 = ttime() + audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect") + p_len = audio_pad.shape[0] // self.window + inp_f0 = None + if hasattr(f0_file, "name") == True: + try: + with open(f0_file.name, "r") as f: + lines = f.read().strip("\n").split("\n") + inp_f0 = [] + for line in lines: + inp_f0.append([float(i) for i in line.split(",")]) + inp_f0 = np.array(inp_f0, dtype="float32") + except: + traceback.print_exc() + sid = torch.tensor(sid, device=self.device).unsqueeze(0).long() + pitch, pitchf = None, None + if if_f0 == 1: + pitch, pitchf = self.get_f0(input_audio_path,audio_pad, p_len, f0_up_key, f0_method,filter_radius, inp_f0) + pitch = pitch[:p_len] + pitchf = pitchf[:p_len] + if self.device == "mps": + pitchf = pitchf.astype(np.float32) + pitch = torch.tensor(pitch, device=self.device).unsqueeze(0).long() + pitchf = torch.tensor(pitchf, device=self.device).unsqueeze(0).float() + t2 = ttime() + times[1] += t2 - t1 + for t in opt_ts: + t = t // self.window * self.window + if if_f0 == 1: + audio_opt.append( + self.vc( + model, + net_g, + sid, + audio_pad[s : t + self.t_pad2 + self.window], + pitch[:, s // self.window : (t + self.t_pad2) // self.window], + pitchf[:, s // self.window : (t + self.t_pad2) // self.window], + times, + index, + big_npy, + index_rate, + version, + )[self.t_pad_tgt : -self.t_pad_tgt] + ) + else: + audio_opt.append( + self.vc( + model, + net_g, + sid, + audio_pad[s : t + self.t_pad2 + self.window], + None, + None, + times, + index, + big_npy, + index_rate, + version, + )[self.t_pad_tgt : -self.t_pad_tgt] + ) + s = t + if if_f0 == 1: + audio_opt.append( + self.vc( + model, + net_g, + sid, + audio_pad[t:], + pitch[:, t // self.window :] if t is not None else pitch, + pitchf[:, t // self.window :] if t is not None else pitchf, + times, + index, + big_npy, + index_rate, + version, + )[self.t_pad_tgt : -self.t_pad_tgt] + ) + else: + audio_opt.append( + self.vc( + model, + net_g, + sid, + audio_pad[t:], + None, + None, + times, + index, + big_npy, + index_rate, + version, + )[self.t_pad_tgt : -self.t_pad_tgt] + ) + audio_opt = np.concatenate(audio_opt) + if(rms_mix_rate!=1): + audio_opt=change_rms(audio,16000,audio_opt,tgt_sr,rms_mix_rate) + if(resample_sr>=16000 and tgt_sr!=resample_sr): + audio_opt = librosa.resample( + audio_opt, orig_sr=tgt_sr, target_sr=resample_sr + ) + audio_max=np.abs(audio_opt).max()/0.99 + max_int16=32768 + if(audio_max>1):max_int16/=audio_max + audio_opt=(audio_opt * max_int16).astype(np.int16) + del pitch, pitchf, sid + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return audio_opt \ No newline at end of file