Update app.py

app.py

@@ -51,11 +51,7 @@ def load_custom_model_from_hf(repo_id, model_filename="pytorch_model.bin", confi
     return model_path, config_path
     
 # Load DiT model
-dit_checkpoint_path, dit_config_path = load_custom_model_from_hf(
-    "Plachta/Seed-VC",
-    "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
-    "config_dit_mel_seed_uvit_whisper_small_wavenet.yml"
-)
+dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC", "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth", "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
 config = yaml.safe_load(open(dit_config_path, 'r'))
 model_params = recursive_munch(config['model_params'])
 model = build_model(model_params, stage='DiT')
@@ -129,11 +125,7 @@ mel_fn_args = {
 to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
 
 # Load F0 conditioned model
-dit_checkpoint_path_f0, dit_config_path_f0 = load_custom_model_from_hf(
-    "Plachta/Seed-VC", 
-    "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth", 
-    "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml"
-)
+dit_checkpoint_path_f0, dit_config_path_f0 = load_custom_model_from_hf("Plachta/Seed-VC", "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth", "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml")
 config_f0 = yaml.safe_load(open(dit_config_path_f0, 'r'))
 model_params_f0 = recursive_munch(config_f0['model_params'])
 model_f0 = build_model(model_params_f0, stage='DiT')
@@ -220,22 +212,9 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     # Generate Whisper features
     print("[INFO] | Generating Whisper features for source audio.")
     if converted_waves_16k.size(-1) <= 16000 * 30:
-        alt_inputs = whisper_feature_extractor(
-            [converted_waves_16k.squeeze(0).cpu().numpy()],
-            return_tensors="pt",
-            return_attention_mask=True,
-            sampling_rate=16000
-        )
-        alt_input_features = whisper_model._mask_input_features(
-            alt_inputs.input_features, attention_mask=alt_inputs.attention_mask
-        ).to(device)
-        alt_outputs = whisper_model.encoder(
-            alt_input_features.to(torch.float32),
-            head_mask=None,
-            output_attentions=False,
-            output_hidden_states=False,
-            return_dict=True
-        )
+        alt_inputs = whisper_feature_extractor([converted_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+        alt_input_features = whisper_model._mask_input_features(alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
+        alt_outputs = whisper_model.encoder(alt_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
         S_alt = alt_outputs.last_hidden_state.to(torch.float32)
         S_alt = S_alt[:, :converted_waves_16k.size(-1) // 320 + 1]
         print(f"[INFO] | S_alt shape: {S_alt.shape}")
@@ -254,26 +233,10 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
             if buffer is None:
                 chunk = converted_waves_16k[:, traversed_time:traversed_time + chunk_size]
             else:
-                chunk = torch.cat([
-                    buffer,
-                    converted_waves_16k[:, traversed_time:traversed_time + chunk_size - overlap_size]
-                ], dim=-1)
-            alt_inputs = whisper_feature_extractor(
-                [chunk.squeeze(0).cpu().numpy()],
-                return_tensors="pt",
-                return_attention_mask=True,
-                sampling_rate=16000
-            )
-            alt_input_features = whisper_model._mask_input_features(
-                alt_inputs.input_features, attention_mask=alt_inputs.attention_mask
-            ).to(device)
-            alt_outputs = whisper_model.encoder(
-                alt_input_features.to(torch.float32),
-                head_mask=None,
-                output_attentions=False,
-                output_hidden_states=False,
-                return_dict=True
-            )
+                chunk = torch.cat([buffer, converted_waves_16k[:, traversed_time:traversed_time + chunk_size - overlap_size]], dim=-1)
+            alt_inputs = whisper_feature_extractor([chunk.squeeze(0).cpu().numpy()],return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+            alt_input_features = whisper_model._mask_input_features(alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
+            alt_outputs = whisper_model.encoder(alt_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
             S_chunk = alt_outputs.last_hidden_state.to(torch.float32)
             S_chunk = S_chunk[:, :chunk.size(-1) // 320 + 1]
             print(f"[INFO] | Processed chunk with S_chunk shape: {S_chunk.shape}")
@@ -293,22 +256,9 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     # Original Whisper features
     print("[INFO] | Generating Whisper features for reference audio.")
     ori_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, 16000)
-    ori_inputs = whisper_feature_extractor(
-        [ori_waves_16k.squeeze(0).cpu().numpy()],
-        return_tensors="pt",
-        return_attention_mask=True,
-        sampling_rate=16000
-    )
-    ori_input_features = whisper_model._mask_input_features(
-        ori_inputs.input_features, attention_mask=ori_inputs.attention_mask
-    ).to(device)
-    ori_outputs = whisper_model.encoder(
-        ori_input_features.to(torch.float32),
-        head_mask=None,
-        output_attentions=False,
-        output_hidden_states=False,
-        return_dict=True
-    )
+    ori_inputs = whisper_feature_extractor([ori_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+    ori_input_features = whisper_model._mask_input_features(ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
+    ori_outputs = whisper_model.encoder(ori_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
     S_ori = ori_outputs.last_hidden_state.to(torch.float32)
     S_ori = S_ori[:, :ori_waves_16k.size(-1) // 320 + 1]
     print(f"[INFO] | S_ori shape: {S_ori.shape}")
@@ -326,12 +276,7 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     
     # Extract style features
     print("[INFO] | Extracting style features from reference audio.")
-    feat2 = torchaudio.compliance.kaldi.fbank(
-        ref_waves_16k,
-        num_mel_bins=80,
-        dither=0,
-        sample_frequency=16000
-    )
+    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000)
     feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
     style2 = campplus_model(feat2.unsqueeze(0))
     print(f"[INFO] | Style2 shape: {style2.shape}")
@@ -358,9 +303,7 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
         # Shift F0 levels
         shifted_log_f0_alt = log_f0_alt.clone()
         if auto_f0_adjust:
-            shifted_log_f0_alt[F0_alt > 1] = (
-                log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori
-            )
+            shifted_log_f0_alt[F0_alt > 1] = (log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori)
         shifted_f0_alt = torch.exp(shifted_log_f0_alt)
         if pitch != 0:
             shifted_f0_alt[F0_alt > 1] = adjust_f0_semitones(shifted_f0_alt[F0_alt > 1], pitch)
@@ -390,15 +333,7 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
         cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
         
         # Perform inference
-        vc_target = inference_module.cfm.inference(
-            cat_condition, 
-            torch.LongTensor([cat_condition.size(1)]).to(mel2.device), 
-            mel2, 
-            style2, 
-            None, 
-            steps, 
-            inference_cfg_rate=guidance
-        )
+        vc_target = inference_module.cfm.inference(cat_condition, torch.LongTensor([cat_condition.size(1)]).to(mel2.device), mel2, style2, None, steps, inference_cfg_rate=guidance)
         vc_target = vc_target[:, :, mel2.size(2):]
         print(f"[INFO] | vc_target shape: {vc_target.shape}")
         
@@ -458,7 +393,7 @@ with gr.Blocks(css=css) as main:
         speed = gr.Slider(label="Speed", value=1.0, minimum=0.5, maximum=2.0, step=0.1)
 
     with gr.Column():
-        use_conditioned = gr.Checkbox(label="Use 'F0 Conditioned Model'", value=False),
+        use_conditioned = gr.Checkbox(label="Use 'F0 Conditioned Model'", value=False)
         use_auto_adjustment = gr.Checkbox(label="Use 'Auto F0 Adjustment' with 'F0 Conditioned Model'", value=True)
         pitch = gr.Slider(label="Pitch with 'F0 Conditioned Model'", value=0, minimum=-12, maximum=12, step=1)