Upload folder using huggingface_hub

best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaf0629ee6895e6f9ddc834bb6d9d3e768bfa29fbffc8abf7747ba8414cf366c
+size 1039966218

best_model_85756.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaf0629ee6895e6f9ddc834bb6d9d3e768bfa29fbffc8abf7747ba8414cf366c
+size 1039966218

checkpoint_110000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55492c876d90b6498e9d18f2076c61d49f0394398a58cb8a01d22ace55d9bf5d
+size 1039966218

checkpoint_115000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76bac8f2bfca048e86ea9d779374222d2d7e7e672e36994437f589b69fea9e26
+size 1039966218

config.json

@@ -0,0 +1,289 @@
+{
+    "output_path": "/home/kk/storage",
+    "logger_uri": null,
+    "run_name": "YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only",
+    "project_name": "YourTTS",
+    "run_description": "\n            - Original YourTTS trained using VCTK dataset\n        ",
+    "print_step": 50,
+    "plot_step": 100,
+    "model_param_stats": false,
+    "wandb_entity": "kleber",
+    "dashboard_logger": "wandb",
+    "save_on_interrupt": true,
+    "log_model_step": 1000,
+    "save_step": 5000,
+    "save_n_checkpoints": 2,
+    "save_checkpoints": true,
+    "save_all_best": false,
+    "save_best_after": 0,
+    "target_loss": "loss_1",
+    "print_eval": false,
+    "test_delay_epochs": 0,
+    "run_eval": true,
+    "run_eval_steps": null,
+    "distributed_backend": "nccl",
+    "distributed_url": "tcp://localhost:54321",
+    "mixed_precision": false,
+    "precision": "fp16",
+    "epochs": 1000,
+    "batch_size": 12,
+    "eval_batch_size": 12,
+    "grad_clip": [
+        1000,
+        1000
+    ],
+    "scheduler_after_epoch": true,
+    "lr": 0.001,
+    "optimizer": "AdamW",
+    "optimizer_params": {
+        "betas": [
+            0.8,
+            0.99
+        ],
+        "eps": 1e-09,
+        "weight_decay": 0.01
+    },
+    "lr_scheduler": null,
+    "lr_scheduler_params": {},
+    "use_grad_scaler": false,
+    "allow_tf32": false,
+    "cudnn_enable": true,
+    "cudnn_deterministic": false,
+    "cudnn_benchmark": false,
+    "training_seed": 54321,
+    "model": "vits",
+    "num_loader_workers": 8,
+    "num_eval_loader_workers": 0,
+    "use_noise_augment": false,
+    "audio": {
+        "fft_size": 1024,
+        "sample_rate": 16000,
+        "win_length": 1024,
+        "hop_length": 256,
+        "num_mels": 80,
+        "mel_fmin": 0.0,
+        "mel_fmax": null
+    },
+    "use_phonemes": false,
+    "phonemizer": "espeak",
+    "phoneme_language": "en",
+    "compute_input_seq_cache": true,
+    "text_cleaner": "multilingual_cleaners",
+    "enable_eos_bos_chars": false,
+    "test_sentences_file": "",
+    "phoneme_cache_path": null,
+    "characters": {
+        "characters_class": "TTS.tts.models.vits.VitsCharacters",
+        "vocab_dict": null,
+        "pad": "_",
+        "eos": "&",
+        "bos": "*",
+        "blank": null,
+        "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\u00af\u00b7\u00df\u00e0\u00e1\u00e2\u00e3\u00e4\u00e6\u00e7\u00e8\u00e9\u00ea\u00eb\u00ec\u00ed\u00ee\u00ef\u00f1\u00f2\u00f3\u00f4\u00f5\u00f6\u00f9\u00fa\u00fb\u00fc\u00ff\u0101\u0105\u0107\u0113\u0119\u011b\u012b\u0131\u0142\u0144\u014d\u0151\u0153\u015b\u016b\u0171\u017a\u017c\u01ce\u01d0\u01d2\u01d4\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043a\u043b\u043c\u043d\u043e\u043f\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044a\u044b\u044c\u044d\u044e\u044f\u0451\u0454\u0456\u0457\u0491\u2013!'(),-.:;? ",
+        "punctuations": "!'(),-.:;? ",
+        "phonemes": "",
+        "is_unique": true,
+        "is_sorted": true
+    },
+    "add_blank": true,
+    "batch_group_size": 48,
+    "loss_masking": null,
+    "min_audio_len": 1,
+    "max_audio_len": 640000,
+    "min_text_len": 1,
+    "max_text_len": Infinity,
+    "compute_f0": false,
+    "compute_energy": false,
+    "compute_linear_spec": true,
+    "precompute_num_workers": 12,
+    "start_by_longest": true,
+    "shuffle": false,
+    "drop_last": false,
+    "datasets": [
+        {
+            "formatter": "du",
+            "dataset_name": "du",
+            "path": "/home/kk/kinya_dataset",
+            "meta_file_train": "actress_train_manifest.tsv",
+            "ignored_speakers": null,
+            "language": "rw",
+            "phonemizer": "",
+            "meta_file_val": "actress_val_manifest.tsv",
+            "meta_file_attn_mask": ""
+        }
+    ],
+    "test_sentences": [
+        [
+            "Umunyamwuga w'ubuzima ashobora gufasha muribi:",
+            "Actress",
+            null,
+            "rw"
+        ],
+        [
+            "Ambasaderi yavuze ko AGUKA izagirwamo uruhare n'ibigo bikora imirimo itandukanye.",
+            "Actress",
+            null,
+            "rw"
+        ],
+        [
+            "Kuri iyi nshuro biratandukanye uraza ukayisaba ugahita uyitahana.",
+            "Actress",
+            null,
+            "rw"
+        ],
+        [
+            "Avuga ko muri ubu bukwe nta nzoga zigeze zihabwa abari babufitemo inshingano cyane cyane abambariye umugeni.",
+            "Actress",
+            null,
+            "rw"
+        ],
+        [
+            "Twe rero ikintu turi gukora cyane ni ukubyirinda.",
+            "Actress",
+            null,
+            "rw"
+        ]
+    ],
+    "eval_split_max_size": 256,
+    "eval_split_size": 0.01,
+    "use_speaker_weighted_sampler": false,
+    "speaker_weighted_sampler_alpha": 1.0,
+    "use_language_weighted_sampler": false,
+    "language_weighted_sampler_alpha": 1.0,
+    "use_length_weighted_sampler": false,
+    "length_weighted_sampler_alpha": 1.0,
+    "model_args": {
+        "num_chars": 165,
+        "out_channels": 513,
+        "spec_segment_size": 32,
+        "hidden_channels": 192,
+        "hidden_channels_ffn_text_encoder": 768,
+        "num_heads_text_encoder": 2,
+        "num_layers_text_encoder": 10,
+        "kernel_size_text_encoder": 3,
+        "dropout_p_text_encoder": 0.1,
+        "dropout_p_duration_predictor": 0.5,
+        "kernel_size_posterior_encoder": 5,
+        "dilation_rate_posterior_encoder": 1,
+        "num_layers_posterior_encoder": 16,
+        "kernel_size_flow": 5,
+        "dilation_rate_flow": 1,
+        "num_layers_flow": 4,
+        "resblock_type_decoder": "2",
+        "resblock_kernel_sizes_decoder": [
+            3,
+            7,
+            11
+        ],
+        "resblock_dilation_sizes_decoder": [
+            [
+                1,
+                3,
+                5
+            ],
+            [
+                1,
+                3,
+                5
+            ],
+            [
+                1,
+                3,
+                5
+            ]
+        ],
+        "upsample_rates_decoder": [
+            8,
+            8,
+            2,
+            2
+        ],
+        "upsample_initial_channel_decoder": 512,
+        "upsample_kernel_sizes_decoder": [
+            16,
+            16,
+            4,
+            4
+        ],
+        "periods_multi_period_discriminator": [
+            2,
+            3,
+            5,
+            7,
+            11
+        ],
+        "use_sdp": true,
+        "noise_scale": 1.0,
+        "inference_noise_scale": 0.667,
+        "length_scale": 1,
+        "noise_scale_dp": 1.0,
+        "inference_noise_scale_dp": 1.0,
+        "max_inference_len": null,
+        "init_discriminator": true,
+        "use_spectral_norm_disriminator": false,
+        "use_speaker_embedding": false,
+        "num_speakers": 0,
+        "speakers_file": "/home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/speakers.pth",
+        "d_vector_file": [
+            "/home/kk/kinya_dataset/speakers.pth"
+        ],
+        "speaker_embedding_channels": 256,
+        "use_d_vector_file": true,
+        "d_vector_dim": 512,
+        "detach_dp_input": true,
+        "use_language_embedding": false,
+        "embedded_language_dim": 4,
+        "num_languages": 0,
+        "language_ids_file": null,
+        "use_speaker_encoder_as_loss": false,
+        "speaker_encoder_config_path": "https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/config_se.json",
+        "speaker_encoder_model_path": "https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/model_se.pth.tar",
+        "condition_dp_on_speaker": true,
+        "freeze_encoder": false,
+        "freeze_DP": false,
+        "freeze_PE": false,
+        "freeze_flow_decoder": false,
+        "freeze_waveform_decoder": false,
+        "encoder_sample_rate": null,
+        "interpolate_z": true,
+        "reinit_DP": false,
+        "reinit_text_encoder": false
+    },
+    "lr_gen": 0.0002,
+    "lr_disc": 0.0002,
+    "lr_scheduler_gen": "ExponentialLR",
+    "lr_scheduler_gen_params": {
+        "gamma": 0.999875,
+        "last_epoch": -1
+    },
+    "lr_scheduler_disc": "ExponentialLR",
+    "lr_scheduler_disc_params": {
+        "gamma": 0.999875,
+        "last_epoch": -1
+    },
+    "kl_loss_alpha": 1.0,
+    "disc_loss_alpha": 1.0,
+    "gen_loss_alpha": 1.0,
+    "feat_loss_alpha": 1.0,
+    "mel_loss_alpha": 45.0,
+    "dur_loss_alpha": 1.0,
+    "speaker_encoder_loss_alpha": 9.0,
+    "return_wav": true,
+    "use_weighted_sampler": true,
+    "weighted_sampler_attrs": {
+        "speaker_name": 1.0
+    },
+    "weighted_sampler_multipliers": {},
+    "r": 1,
+    "num_speakers": 0,
+    "use_speaker_embedding": false,
+    "speakers_file": "/home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/speakers.pth",
+    "speaker_embedding_channels": 256,
+    "language_ids_file": null,
+    "use_language_embedding": false,
+    "use_d_vector_file": true,
+    "d_vector_file": [
+        "/home/kk/kinya_dataset/speakers.pth"
+    ],
+    "d_vector_dim": 512
+}

speakers.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6e56cd70f88a7b49de9c123fd938c95f6c4e9fea28d8c1d10a348528740143d
+size 928

train.py

@@ -0,0 +1,254 @@
+import os
+
+import torch
+from trainer import Trainer, TrainerArgs
+
+from TTS.bin.compute_embeddings import compute_embeddings
+from TTS.bin.resample import resample_files
+from TTS.config.shared_configs import BaseDatasetConfig
+from TTS.tts.configs.vits_config import VitsConfig
+from TTS.tts.datasets import load_tts_samples
+from TTS.tts.models.vits import CharactersConfig, Vits, VitsArgs, VitsAudioConfig
+from TTS.utils.downloaders import download_vctk
+
+torch.set_num_threads(24)
+
+# pylint: disable=W0105
+"""
+    This recipe replicates the first experiment proposed in the YourTTS paper (https://arxiv.org/abs/2112.02418).
+    YourTTS model is based on the VITS model however it uses external speaker embeddings extracted from a pre-trained speaker encoder and has small architecture changes.
+    In addition, YourTTS can be trained in multilingual data, however, this recipe replicates the single language training using the VCTK dataset.
+    If you are interested in multilingual training, we have commented on parameters on the VitsArgs class instance that should be enabled for multilingual training.
+    In addition, you will need to add the extra datasets following the VCTK as an example.
+"""
+CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
+
+# Name of the run for the Trainer
+RUN_NAME = "YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only"
+
+# Path where you want to save the models outputs (configs, checkpoints and tensorboard logs)
+OUT_PATH = "/home/kk/storage" #os.path.dirname(os.path.abspath(__file__))  # "/raid/coqui/Checkpoints/original-YourTTS/"
+
+# If you want to do transfer learning and speedup your training you can set here the path to the original YourTTS model
+RESTORE_PATH = "/home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_02+47AM-dbf1a08a/checkpoint_85000.pth" # None  # "/root/.local/share/tts/tts_models--multilingual--multi-dataset--your_tts/model_file.pth"
+
+# This paramter is useful to debug, it skips the training epochs and just do the evaluation  and produce the test sentences
+SKIP_TRAIN_EPOCH = False
+
+# Set here the batch size to be used in training and evaluation
+BATCH_SIZE = 12
+
+# Training Sampling rate and the target sampling rate for resampling the downloaded dataset (Note: If you change this you might need to redownload the dataset !!)
+# Note: If you add new datasets, please make sure that the dataset sampling rate and this parameter are matching, otherwise resample your audios
+SAMPLE_RATE = 16000
+
+# Max audio length in seconds to be used in training (every audio bigger than it will be ignored)
+MAX_AUDIO_LEN_IN_SECONDS = 40
+
+### Download VCTK dataset
+#VCTK_DOWNLOAD_PATH = os.path.join(CURRENT_PATH, "VCTK")
+# Define the number of threads used during the audio resampling
+NUM_RESAMPLE_THREADS = 10
+# Check if VCTK dataset is not already downloaded, if not download it
+#if not os.path.exists(VCTK_DOWNLOAD_PATH):
+#    print(">>> Downloading VCTK dataset:")
+#    download_vctk(VCTK_DOWNLOAD_PATH)
+#    resample_files(VCTK_DOWNLOAD_PATH, SAMPLE_RATE, file_ext="flac", n_jobs=NUM_RESAMPLE_THREADS)
+
+# init configs
+vctk_config = BaseDatasetConfig(
+    formatter="du",
+    dataset_name="du",
+    meta_file_train="actress_train_manifest.tsv",
+    meta_file_val="actress_val_manifest.tsv",
+    path="/home/kk/kinya_dataset",
+    language="rw",
+   # ignored_speakers=[
+   #     "p261",
+   #     "p225",
+   #     "p294",
+   #     "p347",
+   #     "p238",
+   #     "p234",
+   #     "p248",
+   #     "p335",
+   #     "p245",
+   #     "p326",
+   #     "p302",
+   # ],  # Ignore the test speakers to full replicate the paper experiment
+)
+
+# Add here all datasets configs, in our case we just want to train with the VCTK dataset then we need to add just VCTK. Note: If you want to add new datasets, just add them here and it will automatically compute the speaker embeddings (d-vectors) for this new dataset :)
+DATASETS_CONFIG_LIST = [vctk_config]
+
+### Extract speaker embeddings
+SPEAKER_ENCODER_CHECKPOINT_PATH = (
+    "https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/model_se.pth.tar"
+)
+SPEAKER_ENCODER_CONFIG_PATH = "https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/config_se.json"
+
+D_VECTOR_FILES = []  # List of speaker embeddings/d-vectors to be used during the training
+
+# Iterates all the dataset configs checking if the speakers embeddings are already computated, if not compute it
+for dataset_conf in DATASETS_CONFIG_LIST:
+    # Check if the embeddings weren't already computed, if not compute it
+    embeddings_file = os.path.join(dataset_conf.path, "speakers.pth")
+    if not os.path.isfile(embeddings_file):
+        print(f">>> Computing the speaker embeddings for the {dataset_conf.dataset_name} dataset")
+        compute_embeddings(
+            SPEAKER_ENCODER_CHECKPOINT_PATH,
+            SPEAKER_ENCODER_CONFIG_PATH,
+            embeddings_file,
+            old_speakers_file=None,
+            config_dataset_path=None,
+            formatter_name=dataset_conf.formatter,
+            dataset_name=dataset_conf.dataset_name,
+            dataset_path=dataset_conf.path,
+            meta_file_train=dataset_conf.meta_file_train,
+            meta_file_val=dataset_conf.meta_file_val,
+            disable_cuda=False,
+            no_eval=False,
+        )
+    D_VECTOR_FILES.append(embeddings_file)
+
+
+# Audio config used in training.
+audio_config = VitsAudioConfig(
+    sample_rate=SAMPLE_RATE,
+    hop_length=256,
+    win_length=1024,
+    fft_size=1024,
+    mel_fmin=0.0,
+    mel_fmax=None,
+    num_mels=80,
+)
+
+# Init VITSArgs setting the arguments that are needed for the YourTTS model
+model_args = VitsArgs(
+    d_vector_file=D_VECTOR_FILES,
+    use_d_vector_file=True,
+    d_vector_dim=512,
+    num_layers_text_encoder=10,
+    speaker_encoder_model_path=SPEAKER_ENCODER_CHECKPOINT_PATH,
+    speaker_encoder_config_path=SPEAKER_ENCODER_CONFIG_PATH,
+    resblock_type_decoder="2",  # In the paper, we accidentally trained the YourTTS using ResNet blocks type 2, if you like you can use the ResNet blocks type 1 like the VITS model
+    # Useful parameters to enable the Speaker Consistency Loss (SCL) described in the paper
+    # use_speaker_encoder_as_loss=True,
+    # Useful parameters to enable multilingual training
+    # use_language_embedding=True,
+    # embedded_language_dim=4,
+)
+
+# General training config, here you can change the batch size and others useful parameters
+config = VitsConfig(
+    output_path=OUT_PATH,
+    model_args=model_args,
+    run_name=RUN_NAME,
+    project_name="YourTTS",
+    run_description="""
+            - Original YourTTS trained using VCTK dataset
+        """,
+    dashboard_logger="wandb",
+    wandb_entity="kleber",
+    logger_uri=None,
+    audio=audio_config,
+    batch_size=BATCH_SIZE,
+    batch_group_size=48,
+    eval_batch_size=BATCH_SIZE,
+    num_loader_workers=8,
+    eval_split_max_size=256,
+    print_step=50,
+    plot_step=100,
+    log_model_step=1000,
+    save_step=5000,
+    save_n_checkpoints=2,
+    save_checkpoints=True,
+    target_loss="loss_1",
+    print_eval=False,
+    use_phonemes=False,
+    phonemizer="espeak",
+    phoneme_language="en",
+    compute_input_seq_cache=True,
+    add_blank=True,
+    text_cleaner="multilingual_cleaners",
+    characters=CharactersConfig(
+        characters_class="TTS.tts.models.vits.VitsCharacters",
+        pad="_",
+        eos="&",
+        bos="*",
+        blank=None,
+        characters="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\u00af\u00b7\u00df\u00e0\u00e1\u00e2\u00e3\u00e4\u00e6\u00e7\u00e8\u00e9\u00ea\u00eb\u00ec\u00ed\u00ee\u00ef\u00f1\u00f2\u00f3\u00f4\u00f5\u00f6\u00f9\u00fa\u00fb\u00fc\u00ff\u0101\u0105\u0107\u0113\u0119\u011b\u012b\u0131\u0142\u0144\u014d\u0151\u0153\u015b\u016b\u0171\u017a\u017c\u01ce\u01d0\u01d2\u01d4\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043a\u043b\u043c\u043d\u043e\u043f\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044a\u044b\u044c\u044d\u044e\u044f\u0451\u0454\u0456\u0457\u0491\u2013!'(),-.:;? ",
+        punctuations="!'(),-.:;? ",
+        phonemes="",
+        is_unique=True,
+        is_sorted=True,
+    ),
+    phoneme_cache_path=None,
+    precompute_num_workers=12,
+    start_by_longest=True,
+    datasets=DATASETS_CONFIG_LIST,
+    cudnn_benchmark=False,
+    max_audio_len=SAMPLE_RATE * MAX_AUDIO_LEN_IN_SECONDS,
+    mixed_precision=False,
+    test_sentences=[
+        [
+            "Umunyamwuga w'ubuzima ashobora gufasha muribi:",
+            "Actress",
+            None,
+            "rw",
+        ],
+        [
+            "Ambasaderi yavuze ko AGUKA izagirwamo uruhare n'ibigo bikora imirimo itandukanye.",
+            "Actress",
+            None,
+            "rw",
+        ],
+        [
+            "Kuri iyi nshuro biratandukanye uraza ukayisaba ugahita uyitahana.",
+            "Actress",
+            None,
+            "rw",
+        ],
+        [
+            "Avuga ko muri ubu bukwe nta nzoga zigeze zihabwa abari babufitemo inshingano cyane cyane abambariye umugeni.",
+            "Actress",
+            None,
+            "rw",
+        ],
+        [
+            "Twe rero ikintu turi gukora cyane ni ukubyirinda.",
+            "Actress",
+            None,
+            "rw",
+        ],
+    ],
+    # Enable the weighted sampler
+    use_weighted_sampler=True,
+    # Ensures that all speakers are seen in the training batch equally no matter how many samples each speaker has
+    weighted_sampler_attrs={"speaker_name": 1.0},
+    weighted_sampler_multipliers={},
+    # It defines the Speaker Consistency Loss (SCL) α to 9 like the paper
+    speaker_encoder_loss_alpha=9.0,
+)
+
+# Load all the datasets samples and split traning and evaluation sets
+train_samples, eval_samples = load_tts_samples(
+    config.datasets,
+    eval_split=True,
+    eval_split_max_size=config.eval_split_max_size,
+    eval_split_size=config.eval_split_size,
+)
+
+# Init the model
+model = Vits.init_from_config(config)
+
+# Init the trainer and 🚀
+trainer = Trainer(
+    TrainerArgs(restore_path=RESTORE_PATH, skip_train_epoch=SKIP_TRAIN_EPOCH),
+    config,
+    output_path=OUT_PATH,
+    model=model,
+    train_samples=train_samples,
+    eval_samples=eval_samples,
+)
+trainer.fit()