Upload 6 files

Dockerfile

@@ -0,0 +1,56 @@
+FROM ubuntu:22.04
+
+# Install Python and necessary packages
+RUN apt-get update && apt-get install -y python3.10 python3-pip
+
+# Create a user and necessary directories
+RUN useradd -m -u 1001 user && mkdir -p /home/user/app/cache
+
+RUN apt-get install -y git git-lfs ffmpeg libsm6 libxext6 cmake rsync libgl1-mesa-glx
+RUN git lfs install
+
+# Ensure the cache directory is writable by the user
+RUN chown -R user:user /home/user/app/cache && chmod -R 777 /home/user/app/cache
+
+# Install additional packages from packages.txt
+RUN --mount=target=/tmp/packages.txt,source=packages.txt apt-get update && \
+    xargs -r -a /tmp/packages.txt apt-get install -y && rm -rf /var/lib/apt/lists/*
+
+# Install Python requirements
+RUN --mount=target=/tmp/requirements.txt,source=requirements.txt pip install --no-cache-dir -r /tmp/requirements.txt
+
+WORKDIR /home/user/app
+
+# Install specific versions of pip and other packages
+RUN pip install --no-cache-dir pip==22.3.1 && \
+    pip install --no-cache-dir datasets "huggingface-hub>=0.19" "hf-transfer>=0.1.4" "protobuf<4" "click<8.1" "pydantic~=1.0"
+RUN pip freeze > /tmp/freeze.txt
+
+# Install Gradio and other dependencies
+RUN pip install --no-cache-dir gradio[oauth]==4.38.1 "uvicorn>=0.14.0" spaces
+
+# Copy application files and set correct ownership
+COPY --link --chown=1001:1001 ./ /home/user/app
+
+#EXPOSE 7860
+
+# Moved up to where the user is added
+#RUN mkdir cache
+
+# Set environment variables
+#ENV TRANSFORMERS_CACHE=/home/user/app/cache (deprecated)
+ENV HF_HOME=/home/user/app/cache
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+ENV GRADIO_SERVER_PORT=7860
+
+# Switch to the user
+USER user
+
+# Copy the app file
+COPY app.py .
+
+# Debug: Check permissions of cache directory
+RUN ls -l /home/user/app && ls -l /home/user/app/cache
+
+# Run the app
+CMD ["python3", "app.py"]

README.md

@@ -1,13 +1,11 @@
 ---
-title: Zonia Chatbot Tts
-emoji: 🚀
-colorFrom: red
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.13.1
+title: Explore MMS Finetuning 
+emoji: 🌍
+colorFrom: blue
+colorTo: red
+sdk: docker
 app_file: app.py
 pinned: false
-license: cc-by-nc-4.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,219 @@
+import torch
+
+from transformers import pipeline
+
+import numpy as np
+import gradio as gr
+
+def _grab_best_device(use_gpu=True):
+    if torch.cuda.device_count() > 0 and use_gpu:
+        device = 0 #"cuda"
+    else:
+        device = -1 #"cpu"
+    #device = 0 if torch.cuda.is_available() else -1
+    
+    return device
+
+device = _grab_best_device()
+
+default_model_per_language = {
+    "spanish": "facebook/mms-tts-spa",
+    "tamil": "facebook/mms-tts-tam",
+    "gujarati": "facebook/mms-tts-guj",
+    "marathi": "facebook/mms-tts-mar",
+    #"english": "kakao-enterprise/vits-ljs",
+    "english": "facebook/mms-tts-eng",
+}
+
+models_per_language = {
+    "english": [
+        "ylacombe/vits_ljs_midlands_male_monospeaker",
+    ],
+    "spanish": [
+        "ylacombe/mms-spa-finetuned-chilean-monospeaker",       
+    ],
+    "tamil": [
+        "ylacombe/mms-tam-finetuned-monospeaker",
+    ],
+    "gujarati" : ["ylacombe/mms-guj-finetuned-monospeaker"],
+    "marathi": ["ylacombe/mms-mar-finetuned-monospeaker"]
+}
+
+HUB_PATH = "ylacombe/vits_ljs_midlands_male_monospeaker"
+
+
+pipe_dict = {
+    "current_model": "ylacombe/vits_ljs_midlands_male_monospeaker",
+    "pipe":  pipeline("text-to-speech", model=HUB_PATH, device=device),
+    "original_pipe": pipeline("text-to-speech", model=default_model_per_language["english"], device=device),
+    "language": "english",
+}
+
+title =      """
+# Explore MMS finetuning
+## Or how to access truely multilingual TTS
+
+Massively Multilingual Speech (MMS) models are light-weight, low-latency TTS models based on the [VITS architecture](https://huggingface.co/docs/transformers/model_doc/vits).
+
+Meta's [MMS](https://arxiv.org/abs/2305.13516) project, aiming to provide speech technology across a diverse range of languages. You can find more details about the supported languages and their ISO 639-3 codes in the [MMS Language Coverage Overview](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html),
+and see all MMS-TTS checkpoints on the Hugging Face Hub: [facebook/mms-tts](https://huggingface.co/models?sort=trending&search=facebook%2Fmms-tts).
+    
+Coupled with the right data and the right training recipe, you can get an excellent finetuned version of every MMS checkpoints in **20 minutes** with as little as **80 to 150 samples**.            
+
+Training recipe available in this [github repository](https://github.com/ylacombe/finetune-hf-vits)!
+            """
+
+max_speakers = 15
+
+
+# Inference
+def generate_audio(text, model_id, language):
+
+    if pipe_dict["language"] != language:
+        gr.Warning(f"Language has changed - loading new default model: {default_model_per_language[language]}")
+        pipe_dict["language"] = language
+        pipe_dict["original_pipe"] = pipeline("text-to-speech", model=default_model_per_language[language], device=device)
+    
+    if pipe_dict["current_model"] != model_id:
+        gr.Warning("Model has changed - loading new model")
+        pipe_dict["pipe"] = pipeline("text-to-speech", model=model_id, device=device)
+        pipe_dict["current_model"] = model_id
+
+    num_speakers = pipe_dict["pipe"].model.config.num_speakers
+
+    out = []
+    # first generate original model result
+    output = pipe_dict["original_pipe"](text)
+    output =  gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label=f"Non finetuned model prediction {default_model_per_language[language]}", show_label=True,
+                               visible=True)
+    out.append(output)
+    
+    
+    if num_speakers>1:
+        for i in range(min(num_speakers, max_speakers - 1)):
+            forward_params = {"speaker_id": i}
+            output = pipe_dict["pipe"](text, forward_params=forward_params)
+            
+            output =  gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label=f"Generated Audio - speaker {i}", show_label=True,
+                               visible=True)
+            out.append(output)
+        out.extend([gr.Audio(visible=False)]*(max_speakers-num_speakers))
+    else:
+        output = pipe_dict["pipe"](text)
+        output =  gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label="Generated Audio - Mono speaker", show_label=True,
+                               visible=True)
+        out.append(output)
+        out.extend([gr.Audio(visible=False)]*(max_speakers-2))
+    return out
+
+
+css = """
+#container{
+    margin: 0 auto;
+    max-width: 80rem;
+}
+#intro{
+    max-width: 100%;
+    text-align: center;
+    margin: 0 auto;
+}
+"""
+# Gradio blocks demo    
+with gr.Blocks(css=css) as demo_blocks:
+    gr.Markdown(title, elem_id="intro")
+
+    with gr.Row():
+        with gr.Column():
+            inp_text = gr.Textbox(label="Input Text", info="What sentence would you like to synthesise?")
+            btn = gr.Button("Generate Audio!")
+            language = gr.Dropdown(
+                default_model_per_language.keys(),
+                value = "spanish",
+                label = "language",
+                info = "Language that you want to test"
+            )
+            
+            model_id = gr.Dropdown(
+                    models_per_language["spanish"],
+                    value="ylacombe/mms-spa-finetuned-chilean-monospeaker", 
+                    label="Model", 
+                    info="Model you want to test",
+                    )
+                
+        with gr.Column():
+            outputs = []
+            for i in range(max_speakers):
+                out_audio = gr.Audio(type="numpy", autoplay=False, label=f"Generated Audio - speaker {i}", show_label=True, visible=False)
+                outputs.append(out_audio)
+
+    with gr.Accordion("Datasets and models details", open=False):
+        gr.Markdown("""
+        
+For each language, we used 100 to 150 samples of a single speaker to finetune the model.
+
+### Spanish
+
+* **Model**: [Spanish MMS TTS](https://huggingface.co/facebook/mms-tts-spa).
+* **Datasets**:
+    - [Chilean Spanish TTS dataset](https://huggingface.co/datasets/ylacombe/google-chilean-spanish).
+
+### Tamil
+
+* **Model**: [Tamil MMS TTS](https://huggingface.co/facebook/mms-tts-tam).
+* **Datasets**:
+    - [Tamil TTS dataset](https://huggingface.co/datasets/ylacombe/google-tamil).
+
+### Gujarati
+
+* **Model**: [Gujarati MMS TTS](https://huggingface.co/facebook/mms-tts-guj).
+* **Datasets**:
+    - [Gujarati TTS dataset](https://huggingface.co/datasets/ylacombe/google-gujarati).
+
+### Marathi
+
+* **Model**: [Marathi MMS TTS](https://huggingface.co/facebook/mms-tts-mar).
+* **Datasets**:
+    - [Marathi TTS dataset](https://huggingface.co/datasets/ylacombe/google-chilean-marathi).
+
+### English
+
+* **Model**: [VITS-ljs](https://huggingface.co/kakao-enterprise/vits-ljs)
+* **Dataset**: [British Isles Accent](https://huggingface.co/datasets/ylacombe/english_dialects). For each accent, we used 100 to 150 samples of a single speaker to finetune [VITS-ljs](https://huggingface.co/kakao-enterprise/vits-ljs).
+
+                    
+                    """) 
+
+    with gr.Accordion("Run VITS and MMS with transformers", open=False):
+        gr.Markdown(
+            """
+        ```bash
+        pip install transformers
+        ```
+        ```py
+        from transformers import pipeline
+        import scipy
+        pipe = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs", device=0)
+        
+        results = pipe("A cinematic shot of a baby racoon wearing an intricate italian priest robe")
+
+        # write to a wav file
+        scipy.io.wavfile.write("audio_vits.wav", rate=results["sampling_rate"], data=results["audio"].squeeze())
+        ```
+        """
+        )
+
+
+    language.change(lambda language: gr.Dropdown(
+                    models_per_language[language],
+                    value=models_per_language[language][0], 
+                    label="Model", 
+                    info="Model you want to test",
+                    ),
+                    language,
+                    model_id
+                   )
+    
+    btn.click(generate_audio, [inp_text, model_id, language], outputs)
+    
+
+demo_blocks.queue().launch()

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

packages.txt

@@ -0,0 +1,3 @@
+festival
+espeak-ng
+mbrola

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+torch==2.0.1
+torchvision==0.15.2
+torchaudio==2.0.2
+phonemizer