Spaces:
Sleeping
Sleeping
Staticaliza
commited on
Update app.py
Browse files
app.py
CHANGED
@@ -1,75 +1,481 @@
|
|
1 |
import gradio as gr
|
2 |
-
import
|
3 |
-
import
|
4 |
import spaces
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
|
6 |
-
from
|
7 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
-
#
|
10 |
-
|
|
|
11 |
|
12 |
-
#
|
13 |
-
|
14 |
|
15 |
-
|
|
|
|
|
|
|
|
|
16 |
|
17 |
-
|
18 |
-
def __init__(self):
|
19 |
-
self.model = None # Model will be loaded when GPU is allocated
|
20 |
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
31 |
|
32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
|
34 |
-
|
35 |
-
|
36 |
-
print("Inputs are on device:", inputs['input_ids'].device)
|
37 |
|
38 |
-
|
39 |
-
|
|
|
|
|
|
|
|
|
40 |
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
47 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
48 |
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
# Create the Gradio interface
|
63 |
-
interface = gr.Interface(
|
64 |
-
fn=model_wrapper.generate,
|
65 |
-
inputs=gr.Textbox(lines=5, label="Input Prompt"),
|
66 |
-
outputs=gr.Textbox(label="Generated Text", lines=10),
|
67 |
-
title="Mistral-Large-Instruct-2407 Text Completion",
|
68 |
-
description="Enter a prompt and receive a text completion using the Mistral-Large-Instruct-2407 INT4 model.",
|
69 |
-
allow_flagging='never',
|
70 |
-
live=False,
|
71 |
-
cache_examples=False
|
72 |
)
|
73 |
|
74 |
-
|
75 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import gradio as gr
|
2 |
+
import numpy as np
|
3 |
+
import soundfile as sf
|
4 |
import spaces
|
5 |
+
import torch
|
6 |
+
import torchaudio
|
7 |
+
import librosa
|
8 |
+
import yaml
|
9 |
+
import tempfile
|
10 |
+
import os
|
11 |
|
12 |
+
from huggingface_hub import hf_hub_download
|
13 |
+
from transformers import AutoFeatureExtractor, WhisperModel
|
14 |
+
from torch.nn.utils import parametrizations
|
15 |
+
from modules.commons import build_model, load_checkpoint, recursive_munch
|
16 |
+
from modules.campplus.DTDNN import CAMPPlus
|
17 |
+
from modules.bigvgan import bigvgan
|
18 |
+
from modules.rmvpe import RMVPE
|
19 |
+
from modules.audio import mel_spectrogram
|
20 |
|
21 |
+
# ----------------------------
|
22 |
+
# Optimization Settings
|
23 |
+
# ----------------------------
|
24 |
|
25 |
+
# Set the number of threads to the number of CPU cores
|
26 |
+
torch.set_num_threads(os.cpu_count())
|
27 |
|
28 |
+
# Enable optimized backends
|
29 |
+
torch.backends.openmp.enabled = True
|
30 |
+
torch.backends.mkldnn.enabled = True
|
31 |
+
torch.backends.cudnn.enabled = False
|
32 |
+
torch.backends.cuda.enabled = False
|
33 |
|
34 |
+
torch.set_grad_enabled(False)
|
|
|
|
|
35 |
|
36 |
+
# Force CPU usage
|
37 |
+
device = torch.device("cpu")
|
38 |
+
print(f"[DEVICE] | Using device: {device}")
|
39 |
+
|
40 |
+
# ----------------------------
|
41 |
+
# Load Models and Configuration
|
42 |
+
# ----------------------------
|
43 |
+
|
44 |
+
def load_custom_model_from_hf(repo_id, model_filename="pytorch_model.bin", config_filename="config.yml"):
|
45 |
+
os.makedirs("./checkpoints", exist_ok=True)
|
46 |
+
model_path = hf_hub_download(repo_id=repo_id, filename=model_filename, cache_dir="./checkpoints")
|
47 |
+
if config_filename is None:
|
48 |
+
return model_path
|
49 |
+
config_path = hf_hub_download(repo_id=repo_id, filename=config_filename, cache_dir="./checkpoints")
|
50 |
+
|
51 |
+
return model_path, config_path
|
52 |
+
|
53 |
+
# Load DiT model
|
54 |
+
dit_checkpoint_path, dit_config_path = load_custom_model_from_hf(
|
55 |
+
"Plachta/Seed-VC",
|
56 |
+
"DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
|
57 |
+
"config_dit_mel_seed_uvit_whisper_small_wavenet.yml"
|
58 |
+
)
|
59 |
+
config = yaml.safe_load(open(dit_config_path, 'r'))
|
60 |
+
model_params = recursive_munch(config['model_params'])
|
61 |
+
model = build_model(model_params, stage='DiT')
|
62 |
+
|
63 |
+
# Debug: Print model keys to identify correct key
|
64 |
+
print(f"[INFO] | Model keys: {model.keys()}")
|
65 |
+
|
66 |
+
hop_length = config['preprocess_params']['spect_params']['hop_length']
|
67 |
+
sr = config['preprocess_params']['sr']
|
68 |
|
69 |
+
# Load DiT checkpoints
|
70 |
+
model, _, _, _ = load_checkpoint(model, None, dit_checkpoint_path, load_only_params=True, ignore_modules=[], is_distributed=False)
|
71 |
+
for key in model:
|
72 |
+
model[key].eval()
|
73 |
+
model[key].to(device)
|
74 |
+
print("[INFO] | DiT model loaded and set to eval mode.")
|
75 |
+
|
76 |
+
model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
|
77 |
+
|
78 |
+
# Ensure 'CAMPPlus' is correctly imported and defined
|
79 |
+
try:
|
80 |
+
campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
|
81 |
+
print("[INFO] | CAMPPlus model instantiated.")
|
82 |
+
except NameError:
|
83 |
+
print("[ERROR] | CAMPPlus is not defined. Please check the import path and ensure CAMPPlus is correctly defined.")
|
84 |
+
raise
|
85 |
+
|
86 |
+
# Set weights_only=True for security
|
87 |
+
campplus_ckpt_path = load_custom_model_from_hf("funasr/campplus", "campplus_cn_common.bin", config_filename=None)
|
88 |
+
campplus_state = torch.load(campplus_ckpt_path, map_location="cpu", weights_only=True)
|
89 |
+
campplus_model.load_state_dict(campplus_state)
|
90 |
+
campplus_model.eval()
|
91 |
+
campplus_model.to(device)
|
92 |
+
print("[INFO] | CAMPPlus model loaded, set to eval mode, and moved to CPU.")
|
93 |
+
|
94 |
+
# Load BigVGAN model
|
95 |
+
bigvgan_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_22khz_80band_256x', use_cuda_kernel=False)
|
96 |
+
bigvgan_model.remove_weight_norm()
|
97 |
+
bigvgan_model = bigvgan_model.eval().to(device)
|
98 |
+
print("[INFO] | BigVGAN model loaded, weight norm removed, set to eval mode, and moved to CPU.")
|
99 |
+
|
100 |
+
# Load FAcodec model
|
101 |
+
ckpt_path, config_path = load_custom_model_from_hf("Plachta/FAcodec", 'pytorch_model.bin', 'config.yml')
|
102 |
+
codec_config = yaml.safe_load(open(config_path))
|
103 |
+
codec_model_params = recursive_munch(codec_config['model_params'])
|
104 |
+
codec_encoder = build_model(codec_model_params, stage="codec")
|
105 |
+
ckpt_params = torch.load(ckpt_path, map_location="cpu", weights_only=True)
|
106 |
+
for key in codec_encoder:
|
107 |
+
codec_encoder[key].load_state_dict(ckpt_params[key], strict=False)
|
108 |
+
codec_encoder = {k: v.eval().to(device) for k, v in codec_encoder.items()}
|
109 |
+
print("[INFO] | FAcodec model loaded, set to eval mode, and moved to CPU.")
|
110 |
+
|
111 |
+
# Load Whisper model with float32 and compatible size
|
112 |
+
whisper_name = model_params.speech_tokenizer.whisper_name if hasattr(model_params.speech_tokenizer, 'whisper_name') else "openai/whisper-small"
|
113 |
+
whisper_model = WhisperModel.from_pretrained(whisper_name, torch_dtype=torch.float32).to(device)
|
114 |
+
del whisper_model.decoder # Remove decoder as it's not used
|
115 |
+
whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
|
116 |
+
print(f"[INFO] | Whisper model '{whisper_name}' loaded with dtype {whisper_model.dtype} and moved to CPU.")
|
117 |
+
|
118 |
+
# Generate mel spectrograms with optimized parameters
|
119 |
+
mel_fn_args = {
|
120 |
+
"n_fft": 1024,
|
121 |
+
"win_size": 1024,
|
122 |
+
"hop_size": 256,
|
123 |
+
"num_mels": 80,
|
124 |
+
"sampling_rate": sr,
|
125 |
+
"fmin": 0,
|
126 |
+
"fmax": None,
|
127 |
+
"center": False
|
128 |
+
}
|
129 |
+
to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
|
130 |
+
|
131 |
+
# Load F0 conditioned model
|
132 |
+
dit_checkpoint_path_f0, dit_config_path_f0 = load_custom_model_from_hf(
|
133 |
+
"Plachta/Seed-VC",
|
134 |
+
"DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth",
|
135 |
+
"config_dit_mel_seed_uvit_whisper_base_f0_44k.yml"
|
136 |
+
)
|
137 |
+
config_f0 = yaml.safe_load(open(dit_config_path_f0, 'r'))
|
138 |
+
model_params_f0 = recursive_munch(config_f0['model_params'])
|
139 |
+
model_f0 = build_model(model_params_f0, stage='DiT')
|
140 |
|
141 |
+
hop_length_f0 = config_f0['preprocess_params']['spect_params']['hop_length']
|
142 |
+
sr_f0 = config_f0['preprocess_params']['sr']
|
|
|
143 |
|
144 |
+
# Load F0 model checkpoints
|
145 |
+
model_f0, _, _, _ = load_checkpoint(model_f0, None, dit_checkpoint_path_f0, load_only_params=True, ignore_modules=[], is_distributed=False)
|
146 |
+
for key in model_f0:
|
147 |
+
model_f0[key].eval()
|
148 |
+
model_f0[key].to(device)
|
149 |
+
print("[INFO] | F0 conditioned DiT model loaded and set to eval mode.")
|
150 |
|
151 |
+
model_f0.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
|
152 |
+
|
153 |
+
# Load F0 extractor
|
154 |
+
model_path = load_custom_model_from_hf("lj1995/VoiceConversionWebUI", "rmvpe.pt", None)
|
155 |
+
rmvpe = RMVPE(model_path, is_half=False, device=device)
|
156 |
+
print("[INFO] | RMVPE model loaded and moved to CPU.")
|
157 |
+
|
158 |
+
mel_fn_args_f0 = {
|
159 |
+
"n_fft": config_f0['preprocess_params']['spect_params']['n_fft'],
|
160 |
+
"win_size": config_f0['preprocess_params']['spect_params']['win_length'],
|
161 |
+
"hop_size": config_f0['preprocess_params']['spect_params']['hop_length'],
|
162 |
+
"num_mels": 80, # Ensure this matches the primary model
|
163 |
+
"sampling_rate": sr_f0,
|
164 |
+
"fmin": 0,
|
165 |
+
"fmax": None,
|
166 |
+
"center": False
|
167 |
+
}
|
168 |
+
to_mel_f0 = lambda x: mel_spectrogram(x, **mel_fn_args_f0)
|
169 |
+
|
170 |
+
# Load BigVGAN 44kHz model
|
171 |
+
bigvgan_44k_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_44khz_128band_512x', use_cuda_kernel=False)
|
172 |
+
bigvgan_44k_model.remove_weight_norm()
|
173 |
+
bigvgan_44k_model = bigvgan_44k_model.eval().to(device)
|
174 |
+
print("[INFO] | BigVGAN 44kHz model loaded, weight norm removed, set to eval mode, and moved to CPU.")
|
175 |
+
|
176 |
+
# ----------------------------
|
177 |
+
# Helper Functions
|
178 |
+
# ----------------------------
|
179 |
+
|
180 |
+
def adjust_f0_semitones(f0_sequence, n_semitones):
|
181 |
+
factor = 2 ** (n_semitones / 12)
|
182 |
+
return f0_sequence * factor
|
183 |
+
|
184 |
+
def crossfade(chunk1, chunk2, overlap):
|
185 |
+
fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
|
186 |
+
fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
|
187 |
+
chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
|
188 |
+
return chunk2
|
189 |
+
|
190 |
+
# ----------------------------
|
191 |
+
# Voice Conversion Function
|
192 |
+
# ----------------------------
|
193 |
+
|
194 |
+
@torch.no_grad()
|
195 |
+
@torch.inference_mode()
|
196 |
+
def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate, f0_condition, auto_f0_adjust, pitch_shift):
|
197 |
+
print("[INFO] | Voice conversion started.")
|
198 |
+
|
199 |
+
inference_module = model if not f0_condition else model_f0
|
200 |
+
mel_fn = to_mel if not f0_condition else to_mel_f0
|
201 |
+
bigvgan_fn = bigvgan_model if not f0_condition else bigvgan_44k_model
|
202 |
+
sr_current = 22050 if not f0_condition else 44100
|
203 |
+
hop_length_current = 256 if not f0_condition else 512
|
204 |
+
max_context_window = sr_current // hop_length_current * 30
|
205 |
+
overlap_wave_len = 16 * hop_length_current
|
206 |
+
bitrate = "320k"
|
207 |
+
|
208 |
+
# Load audio using librosa
|
209 |
+
print("[INFO] | Loading source and reference audio.")
|
210 |
+
source_audio, _ = librosa.load(source, sr=sr_current)
|
211 |
+
ref_audio, _ = librosa.load(target, sr=sr_current)
|
212 |
+
|
213 |
+
# Clip reference audio to 25 seconds
|
214 |
+
ref_audio = ref_audio[:sr_current * 25]
|
215 |
+
print(f"[INFO] | Source audio length: {len(source_audio)/sr_current:.2f}s, Reference audio length: {len(ref_audio)/sr_current:.2f}s")
|
216 |
+
|
217 |
+
# Convert audio to tensors
|
218 |
+
source_audio_tensor = torch.tensor(source_audio).unsqueeze(0).float().to(device)
|
219 |
+
ref_audio_tensor = torch.tensor(ref_audio).unsqueeze(0).float().to(device)
|
220 |
+
|
221 |
+
# Resample to 16kHz
|
222 |
+
ref_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, 16000)
|
223 |
+
converted_waves_16k = torchaudio.functional.resample(source_audio_tensor, sr_current, 16000)
|
224 |
+
|
225 |
+
# Generate Whisper features
|
226 |
+
print("[INFO] | Generating Whisper features for source audio.")
|
227 |
+
if converted_waves_16k.size(-1) <= 16000 * 30:
|
228 |
+
alt_inputs = whisper_feature_extractor(
|
229 |
+
[converted_waves_16k.squeeze(0).cpu().numpy()],
|
230 |
+
return_tensors="pt",
|
231 |
+
return_attention_mask=True,
|
232 |
+
sampling_rate=16000
|
233 |
+
)
|
234 |
+
alt_input_features = whisper_model._mask_input_features(
|
235 |
+
alt_inputs.input_features, attention_mask=alt_inputs.attention_mask
|
236 |
+
).to(device)
|
237 |
+
alt_outputs = whisper_model.encoder(
|
238 |
+
alt_input_features.to(torch.float32),
|
239 |
+
head_mask=None,
|
240 |
+
output_attentions=False,
|
241 |
+
output_hidden_states=False,
|
242 |
+
return_dict=True
|
243 |
)
|
244 |
+
S_alt = alt_outputs.last_hidden_state.to(torch.float32)
|
245 |
+
S_alt = S_alt[:, :converted_waves_16k.size(-1) // 320 + 1]
|
246 |
+
print(f"[INFO] | S_alt shape: {S_alt.shape}")
|
247 |
+
else:
|
248 |
+
# Process in chunks
|
249 |
+
print("[INFO] | Processing source audio in chunks.")
|
250 |
+
overlapping_time = 5 # seconds
|
251 |
+
chunk_size = 16000 * 30 # 30 seconds
|
252 |
+
overlap_size = 16000 * overlapping_time
|
253 |
+
S_alt_list = []
|
254 |
+
buffer = None
|
255 |
+
traversed_time = 0
|
256 |
+
total_length = converted_waves_16k.size(-1)
|
257 |
+
|
258 |
+
while traversed_time < total_length:
|
259 |
+
if buffer is None:
|
260 |
+
chunk = converted_waves_16k[:, traversed_time:traversed_time + chunk_size]
|
261 |
+
else:
|
262 |
+
chunk = torch.cat([
|
263 |
+
buffer,
|
264 |
+
converted_waves_16k[:, traversed_time:traversed_time + chunk_size - overlap_size]
|
265 |
+
], dim=-1)
|
266 |
+
alt_inputs = whisper_feature_extractor(
|
267 |
+
[chunk.squeeze(0).cpu().numpy()],
|
268 |
+
return_tensors="pt",
|
269 |
+
return_attention_mask=True,
|
270 |
+
sampling_rate=16000
|
271 |
+
)
|
272 |
+
alt_input_features = whisper_model._mask_input_features(
|
273 |
+
alt_inputs.input_features, attention_mask=alt_inputs.attention_mask
|
274 |
+
).to(device)
|
275 |
+
alt_outputs = whisper_model.encoder(
|
276 |
+
alt_input_features.to(torch.float32),
|
277 |
+
head_mask=None,
|
278 |
+
output_attentions=False,
|
279 |
+
output_hidden_states=False,
|
280 |
+
return_dict=True
|
281 |
+
)
|
282 |
+
S_chunk = alt_outputs.last_hidden_state.to(torch.float32)
|
283 |
+
S_chunk = S_chunk[:, :chunk.size(-1) // 320 + 1]
|
284 |
+
print(f"[INFO] | Processed chunk with S_chunk shape: {S_chunk.shape}")
|
285 |
+
|
286 |
+
if traversed_time == 0:
|
287 |
+
S_alt_list.append(S_chunk)
|
288 |
+
else:
|
289 |
+
skip_frames = 50 * overlapping_time
|
290 |
+
S_alt_list.append(S_chunk[:, skip_frames:])
|
291 |
+
|
292 |
+
buffer = chunk[:, -overlap_size:]
|
293 |
+
traversed_time += chunk_size - overlap_size
|
294 |
+
|
295 |
+
S_alt = torch.cat(S_alt_list, dim=1)
|
296 |
+
print(f"[INFO] | Final S_alt shape after chunk processing: {S_alt.shape}")
|
297 |
+
|
298 |
+
# Original Whisper features
|
299 |
+
print("[INFO] | Generating Whisper features for reference audio.")
|
300 |
+
ori_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, 16000)
|
301 |
+
ori_inputs = whisper_feature_extractor(
|
302 |
+
[ori_waves_16k.squeeze(0).cpu().numpy()],
|
303 |
+
return_tensors="pt",
|
304 |
+
return_attention_mask=True,
|
305 |
+
sampling_rate=16000
|
306 |
+
)
|
307 |
+
ori_input_features = whisper_model._mask_input_features(
|
308 |
+
ori_inputs.input_features, attention_mask=ori_inputs.attention_mask
|
309 |
+
).to(device)
|
310 |
+
ori_outputs = whisper_model.encoder(
|
311 |
+
ori_input_features.to(torch.float32),
|
312 |
+
head_mask=None,
|
313 |
+
output_attentions=False,
|
314 |
+
output_hidden_states=False,
|
315 |
+
return_dict=True
|
316 |
+
)
|
317 |
+
S_ori = ori_outputs.last_hidden_state.to(torch.float32)
|
318 |
+
S_ori = S_ori[:, :ori_waves_16k.size(-1) // 320 + 1]
|
319 |
+
print(f"[INFO] | S_ori shape: {S_ori.shape}")
|
320 |
+
|
321 |
+
# Generate mel spectrograms
|
322 |
+
print("[INFO] | Generating mel spectrograms.")
|
323 |
+
mel = mel_fn(source_audio_tensor.float())
|
324 |
+
mel2 = mel_fn(ref_audio_tensor.float())
|
325 |
+
print(f"[INFO] | Mel spectrogram shapes: mel={mel.shape}, mel2={mel2.shape}")
|
326 |
+
|
327 |
+
# Length adjustment
|
328 |
+
target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
|
329 |
+
target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
|
330 |
+
print(f"[INFO] | Target lengths: {target_lengths.item()}, {target2_lengths.item()}")
|
331 |
+
|
332 |
+
# Extract style features
|
333 |
+
print("[INFO] | Extracting style features from reference audio.")
|
334 |
+
feat2 = torchaudio.compliance.kaldi.fbank(
|
335 |
+
ref_waves_16k,
|
336 |
+
num_mel_bins=80,
|
337 |
+
dither=0,
|
338 |
+
sample_frequency=16000
|
339 |
+
)
|
340 |
+
feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
|
341 |
+
style2 = campplus_model(feat2.unsqueeze(0))
|
342 |
+
print(f"[INFO] | Style2 shape: {style2.shape}")
|
343 |
+
|
344 |
+
# F0 Conditioning
|
345 |
+
if f0_condition:
|
346 |
+
print("[INFO] | Performing F0 conditioning.")
|
347 |
+
F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.5)
|
348 |
+
F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.5)
|
349 |
+
|
350 |
+
F0_ori = torch.from_numpy(F0_ori).to(device)[None].float()
|
351 |
+
F0_alt = torch.from_numpy(F0_alt).to(device)[None].float()
|
352 |
+
|
353 |
+
voiced_F0_ori = F0_ori[F0_ori > 1]
|
354 |
+
voiced_F0_alt = F0_alt[F0_alt > 1]
|
355 |
+
|
356 |
+
log_f0_alt = torch.log(F0_alt + 1e-5)
|
357 |
+
voiced_log_f0_ori = torch.log(voiced_F0_ori + 1e-5)
|
358 |
+
voiced_log_f0_alt = torch.log(voiced_F0_alt + 1e-5)
|
359 |
+
|
360 |
+
median_log_f0_ori = torch.median(voiced_log_f0_ori)
|
361 |
+
median_log_f0_alt = torch.median(voiced_log_f0_alt)
|
362 |
+
|
363 |
+
# Shift F0 levels
|
364 |
+
shifted_log_f0_alt = log_f0_alt.clone()
|
365 |
+
if auto_f0_adjust:
|
366 |
+
shifted_log_f0_alt[F0_alt > 1] = (
|
367 |
+
log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori
|
368 |
+
)
|
369 |
+
shifted_f0_alt = torch.exp(shifted_log_f0_alt)
|
370 |
+
if pitch_shift != 0:
|
371 |
+
shifted_f0_alt[F0_alt > 1] = adjust_f0_semitones(shifted_f0_alt[F0_alt > 1], pitch_shift)
|
372 |
+
print("[INFO] | F0 conditioning completed.")
|
373 |
+
else:
|
374 |
+
F0_ori = None
|
375 |
+
F0_alt = None
|
376 |
+
shifted_f0_alt = None
|
377 |
+
print("[INFO] | F0 conditioning not applied.")
|
378 |
+
|
379 |
+
# Length Regulation
|
380 |
+
print("[INFO] | Applying length regulation.")
|
381 |
+
cond, _, _, _, _ = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt)
|
382 |
+
prompt_condition, _, _, _, _ = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori)
|
383 |
+
print(f"[INFO] | Cond shape: {cond.shape}, Prompt condition shape: {prompt_condition.shape}")
|
384 |
+
|
385 |
+
# Initialize variables for audio generation
|
386 |
+
max_source_window = max_context_window - mel2.size(2)
|
387 |
+
processed_frames = 0
|
388 |
+
generated_wave_chunks = []
|
389 |
+
|
390 |
+
print("[INFO] | Starting inference and audio generation.")
|
391 |
+
|
392 |
+
while processed_frames < cond.size(1):
|
393 |
+
chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
|
394 |
+
is_last_chunk = processed_frames + max_source_window >= cond.size(1)
|
395 |
+
cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
|
396 |
+
|
397 |
+
# Perform inference
|
398 |
+
vc_target = inference_module.cfm.inference(
|
399 |
+
cat_condition,
|
400 |
+
torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
|
401 |
+
mel2,
|
402 |
+
style2,
|
403 |
+
None,
|
404 |
+
diffusion_steps,
|
405 |
+
inference_cfg_rate=inference_cfg_rate
|
406 |
+
)
|
407 |
+
vc_target = vc_target[:, :, mel2.size(2):]
|
408 |
+
print(f"[INFO] | vc_target shape: {vc_target.shape}")
|
409 |
+
|
410 |
+
# Generate waveform using BigVGAN
|
411 |
+
vc_wave = bigvgan_fn(vc_target.float())[0]
|
412 |
+
print(f"[INFO] | vc_wave shape: {vc_wave.shape}")
|
413 |
+
|
414 |
+
# Handle the generated waveform
|
415 |
+
output_wave = vc_wave[0].cpu().numpy()
|
416 |
+
generated_wave_chunks.append(output_wave)
|
417 |
+
|
418 |
+
# Fix: Ensure processed_frames increments correctly to avoid infinite loop
|
419 |
+
processed_frames += vc_target.size(2) # Changed from 'vc_target.size(2) - 16' to 'vc_target.size(2)'
|
420 |
+
print(f"[INFO] | Processed frames updated to: {processed_frames}")
|
421 |
+
|
422 |
+
# Concatenate all generated wave chunks
|
423 |
+
final_audio = np.concatenate(generated_wave_chunks).astype(np.float32)
|
424 |
+
|
425 |
+
# Normalize the audio to ensure it's within [-1.0, 1.0]
|
426 |
+
max_val = np.max(np.abs(final_audio))
|
427 |
+
if max_val > 1.0:
|
428 |
+
final_audio = final_audio / max_val
|
429 |
+
print("[INFO] | Final audio normalized.")
|
430 |
+
|
431 |
+
# Save the audio to a temporary WAV file
|
432 |
+
print("[INFO] | Saving final audio to a temporary WAV file.")
|
433 |
+
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_file:
|
434 |
+
sf.write(tmp_file.name, final_audio, sr_current, format='WAV')
|
435 |
+
temp_file_path = tmp_file.name
|
436 |
+
|
437 |
+
print(f"[INFO] | Final audio saved to {temp_file_path}")
|
438 |
+
|
439 |
+
return temp_file_path
|
440 |
+
|
441 |
+
# ----------------------------
|
442 |
+
# Bypass GPU Initialization Error (KEEP THIS FUNCTION AND NEVER DELETE, OTHERWISE IT WILL ERROR DUE TO NOT AT LEAST ONE FUNCTION HAVING USE OF GPU)
|
443 |
+
# ----------------------------
|
444 |
|
445 |
+
@spaces.GPU(duration=15)
|
446 |
+
def gpu():
|
447 |
+
return
|
448 |
+
|
449 |
+
# ----------------------------
|
450 |
+
# Gradio Interface
|
451 |
+
# ----------------------------
|
452 |
+
|
453 |
+
description = (
|
454 |
+
"🪄 **Voice Conversion Tool**\n\n"
|
455 |
+
"Upload your **Source Audio** and **Reference Audio** files to perform voice conversion. "
|
456 |
+
"Adjust the sliders and checkboxes to customize the conversion process."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
457 |
)
|
458 |
|
459 |
+
inputs = [
|
460 |
+
gr.Audio(type="filepath", label="Source Audio"),
|
461 |
+
gr.Audio(type="filepath", label="Reference Audio"),
|
462 |
+
gr.Slider(minimum=1, maximum=100, value=25, step=1, label="Diffusion Steps", info="Default is 25. Use 50-100 for best quality."),
|
463 |
+
gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjustment", info="<1.0 to speed up speech, >1.0 to slow down speech."),
|
464 |
+
gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="Has a subtle influence."),
|
465 |
+
gr.Checkbox(label="Use F0 Conditioned Model", value=False, info="Must be enabled for singing voice conversion."),
|
466 |
+
gr.Checkbox(label="Auto F0 Adjustment", value=True, info="Roughly adjusts F0 to match target voice. Only works when 'Use F0 Conditioned Model' is enabled."),
|
467 |
+
gr.Slider(label='Pitch Shift (semitones)', minimum=-12, maximum=12, step=1, value=0, info="Pitch shift in semitones. Only works when 'Use F0 Conditioned Model' is enabled."),
|
468 |
+
]
|
469 |
+
|
470 |
+
# Set outputs to a single gr.Audio component with type="filepath"
|
471 |
+
outputs = gr.Audio(label="Full Output Audio", type="filepath")
|
472 |
+
|
473 |
+
gr.Interface(
|
474 |
+
fn=voice_conversion,
|
475 |
+
description=description,
|
476 |
+
inputs=inputs,
|
477 |
+
outputs=outputs,
|
478 |
+
title="Seed Voice Conversion",
|
479 |
+
cache_examples=False,
|
480 |
+
allow_flagging="never"
|
481 |
+
).launch(share=True)
|