Merge branch 'main' into ps

checkpoints/ps_normal_exp/config.yaml

@@ -82,7 +82,6 @@ fvae_kernel_size: 5
 fvae_noise_scale: 1.0
 fvae_strides: 4
 gen_dir_name: ''
-glow_kernel_size: 3
 griffin_lim_iters: 30
 hidden_size: 192
 hop_size: 256
@@ -127,8 +126,6 @@ out_wav_norm: false
 pitch_extractor: parselmouth
 pitch_key: pitch
 pitch_type: frame
-post_decoder: false
-post_decoder_detach_ling: false
 post_flow_lr: 0.001
 post_glow_hidden: 192
 post_glow_kernel_size: 3
@@ -157,8 +154,9 @@ preprocess_args:
   with_phsep: true
 preprocess_cls: egs.datasets.audio.lj.preprocess.LJPreprocess
 print_nan_grads: false
-prior_glow_hidden: 64
-prior_glow_n_blocks: 4
+prior_flow_hidden: 64
+prior_flow_kernel_size: 3
+prior_flow_n_blocks: 4
 processed_data_dir: data/processed/ljspeech
 profile_infer: false
 raw_data_dir: data/raw/LJSpeech-1.1

checkpoints/ps_small_exp/config.yaml

@@ -82,7 +82,6 @@ fvae_kernel_size: 3
 fvae_noise_scale: 1.0
 fvae_strides: 4
 gen_dir_name: ''
-glow_kernel_size: 3
 griffin_lim_iters: 30
 hidden_size: 128
 hop_size: 256
@@ -127,8 +126,6 @@ out_wav_norm: false
 pitch_extractor: parselmouth
 pitch_key: pitch
 pitch_type: frame
-post_decoder: false
-post_decoder_detach_ling: false
 post_flow_lr: 0.001
 post_glow_hidden: 128
 post_glow_kernel_size: 3
@@ -157,8 +154,9 @@ preprocess_args:
   with_phsep: true
 preprocess_cls: egs.datasets.audio.lj.preprocess.LJPreprocess
 print_nan_grads: false
-prior_glow_hidden: 32
-prior_glow_n_blocks: 3
+prior_flow_hidden: 32
+prior_flow_kernel_size: 3
+prior_flow_n_blocks: 3
 processed_data_dir: data/processed/ljspeech
 profile_infer: false
 raw_data_dir: data/raw/LJSpeech-1.1

egs/egs_bases/tts/ps.yaml

@@ -38,14 +38,12 @@ fvae_enc_n_layers: 8
 fvae_dec_n_layers: 4
 fvae_strides: 4
 fvae_noise_scale: 1.0
-post_decoder: false
-post_decoder_detach_ling: false
 
 # prior flow
 use_prior_flow: true
-prior_glow_hidden: 64
-glow_kernel_size: 3
-prior_glow_n_blocks: 4
+prior_flow_hidden: 64
+prior_flow_kernel_size: 3
+prior_flow_n_blocks: 4
 
 ###########################
 # training and inference

egs/egs_bases/tts/ps_flow_small.yaml

@@ -30,9 +30,9 @@ fvae_noise_scale: 1.0
 
 # prior flow
 use_prior_flow: true
-prior_glow_hidden: 32
-glow_kernel_size: 3
-prior_glow_n_blocks: 3
+prior_flow_hidden: 32
+prior_flow_kernel_size: 3
+prior_flow_n_blocks: 3
 # post flow
 post_glow_hidden: 128
 post_glow_kernel_size: 3

modules/tts/portaspeech/portaspeech.py

@@ -74,9 +74,9 @@ class PortaSpeech(FastSpeech):
                 dec_n_layers=hparams['fvae_dec_n_layers'],
                 c_cond=self.hidden_size,
                 use_prior_flow=hparams['use_prior_flow'],
-                flow_hidden=hparams['prior_glow_hidden'],
-                flow_kernel_size=hparams['glow_kernel_size'],
-                flow_n_steps=hparams['prior_glow_n_blocks'],
+                flow_hidden=hparams['prior_flow_hidden'],
+                flow_kernel_size=hparams['prior_flow_kernel_size'],
+                flow_n_steps=hparams['prior_flow_n_blocks'],
                 strides=[hparams['fvae_strides']],
                 encoder_type=hparams['fvae_encoder_type'],
                 decoder_type=hparams['fvae_decoder_type'],
@@ -88,11 +88,6 @@ class PortaSpeech(FastSpeech):
             self.pitch_embed = Embedding(300, self.hidden_size, 0)
         if self.hparams['add_word_pos']:
             self.word_pos_proj = Linear(self.hidden_size, self.hidden_size)
-        if self.hparams['post_decoder']:
-            self.post_decoder_proj_in = Linear(self.out_dims, self.hidden_size)
-            self.post_decoder = ConditionalConvBlocks(
-                self.hidden_size, self.hidden_size, self.out_dims, None,
-                hparams['dec_kernel_size'], num_layers=4)
 
     def build_embedding(self, dictionary, embed_dim):
         num_embeddings = len(dictionary)
@@ -188,11 +183,6 @@ class PortaSpeech(FastSpeech):
                     z = torch.randn_like(z)
             x_recon = self.fvae.decoder(z, nonpadding=tgt_nonpadding_BHT, cond=x).transpose(1, 2)
             ret['pre_mel_out'] = x_recon
-            if self.hparams['post_decoder']:
-                x_recon = self.post_decoder_proj_in(x_recon.detach())
-                if self.hparams['post_decoder_detach_ling']:
-                    decoder_inp = decoder_inp.detach()
-                x_recon = self.post_decoder(x_recon, decoder_inp) * tgt_nonpadding
             return x_recon
 
     def forward_dur(self, dur_input, mel2word, ret, **kwargs):

tasks/tts/ps.py

@@ -58,8 +58,6 @@ class PortaSpeechTask(FastSpeechTask):
             losses_kl = min(self.global_step / hparams['kl_start_steps'], 1) * losses_kl
             losses_kl = losses_kl * hparams['lambda_kl']
             losses['kl'] = losses_kl
-            if hparams['post_decoder']:
-                self.add_mel_loss(output['pre_mel_out'], sample['mels'], losses, '_post')
             self.add_mel_loss(output['mel_out'], sample['mels'], losses)
             if hparams['dur_level'] == 'word':
                 self.add_dur_loss(