Duplicate from Audio-AGI/WavJourney

Co-authored-by: Xubo Liu <Xubo-Liu@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+output/*
+__pycache__/*
+service_logs/*
+convert_script_to_audio_gen_code.py
+/cache/
+VoiceParser/hubert/*
+VoiceParser/__pycache__
+/services_logs/

APIs.py

@@ -0,0 +1,202 @@
+import os
+import numpy as np
+import requests
+import yaml
+import pyloudnorm as pyln
+from scipy.io.wavfile import write
+import torchaudio
+from retrying import retry
+from utils import get_service_port, get_service_url
+ 
+
+os.environ['OPENBLAS_NUM_THREADS'] = '1'
+SAMPLE_RATE = 32000
+
+
+with open('config.yaml', 'r') as file:
+    config = yaml.safe_load(file)
+    service_port = get_service_port()
+    localhost_addr = get_service_url()
+    enable_sr = config['Speech-Restoration']['Enable']
+
+def LOUDNESS_NORM(audio, sr=32000, volumn=-25):
+    # peak normalize audio to -1 dB
+    peak_normalized_audio = pyln.normalize.peak(audio, -10.0)
+    # measure the loudness first 
+    meter = pyln.Meter(sr) # create BS.1770 meter
+    loudness = meter.integrated_loudness(peak_normalized_audio)
+    # loudness normalize audio to -12 dB LUFS
+    normalized_audio = pyln.normalize.loudness(peak_normalized_audio, loudness, volumn)
+    return normalized_audio
+    
+
+def WRITE_AUDIO(wav, name=None, sr=SAMPLE_RATE):
+    """
+    function: write audio numpy to .wav file
+    @params:
+        wav: np.array [samples]
+    """   
+    if name is None:
+        name = 'output.wav' 
+    
+    if len(wav.shape) > 1:
+        wav = wav[0]
+
+    # declipping
+    
+    max_value = np.max(np.abs(wav))
+    if max_value > 1:
+        wav *= 0.9 / max_value
+    
+    # write audio
+    write(name, sr, np.round(wav*32767).astype(np.int16))
+
+
+def READ_AUDIO_NUMPY(wav, sr=SAMPLE_RATE):
+    """
+    function: read audio numpy 
+    return: np.array [samples]
+    """
+    waveform, sample_rate = torchaudio.load(wav)
+
+    if sample_rate != sr:
+        waveform = torchaudio.functional.resample(waveform, orig_freq=sample_rate, new_freq=sr)
+    
+    wav_numpy = waveform[0].numpy()
+
+    return wav_numpy
+
+
+def MIX(wavs=[['1.wav', 0.], ['2.wav', 10.]], out_wav='out.wav', sr=SAMPLE_RATE):
+    """
+    wavs:[[wav_name, absolute_offset], ...]
+    """
+
+    max_length = max([int(wav[1]*sr + len(READ_AUDIO_NUMPY(wav[0]))) for wav in wavs])
+    template_wav = np.zeros(max_length)
+
+    for wav in wavs:
+        cur_name, cur_offset = wav
+        cur_wav = READ_AUDIO_NUMPY(cur_name)
+        cur_len = len(cur_wav)
+        cur_offset = int(cur_offset * sr)
+        
+        # mix
+        template_wav[cur_offset:cur_offset+cur_len] += cur_wav
+    
+    WRITE_AUDIO(template_wav, name=out_wav)
+
+
+def CAT(wavs, out_wav='out.wav'):
+    """
+    wavs: List of wav file ['1.wav', '2.wav', ...]
+    """
+    wav_num = len(wavs)
+
+    segment0 = READ_AUDIO_NUMPY(wavs[0])
+
+    cat_wav = segment0
+
+    if wav_num > 1:
+        for i in range(1, wav_num):
+            next_wav = READ_AUDIO_NUMPY(wavs[i])
+            cat_wav = np.concatenate((cat_wav, next_wav), axis=-1)
+
+    WRITE_AUDIO(cat_wav, name=out_wav)
+
+
+def COMPUTE_LEN(wav):
+    wav= READ_AUDIO_NUMPY(wav)
+    return len(wav) / 32000
+
+
+@retry(stop_max_attempt_number=5, wait_fixed=2000)
+def TTM(text, length=10, volume=-28, out_wav='out.wav'):
+    url = f'http://{localhost_addr}:{service_port}/generate_music'
+    data = {
+        'text': f'{text}',
+        'length': f'{length}',
+        'volume': f'{volume}',
+        'output_wav': f'{out_wav}',
+    }
+    
+    response = requests.post(url, json=data)
+
+    if response.status_code == 200:
+        print('Success:', response.json()['message'])
+    else:
+        print('Error:', response.json()['API error'])
+        raise RuntimeError(response.json()['API error'])
+
+@retry(stop_max_attempt_number=5, wait_fixed=2000)
+def TTA(text, length=5, volume=-35, out_wav='out.wav'):
+    url = f'http://{localhost_addr}:{service_port}/generate_audio'
+    data = {
+        'text': f'{text}',
+        'length': f'{length}',
+        'volume': f'{volume}',
+        'output_wav': f'{out_wav}',
+    }
+
+    response = requests.post(url, json=data)
+
+    if response.status_code == 200:
+        print('Success:', response.json()['message'])
+    else:
+        print('Error:', response.json()['API error'])
+        raise RuntimeError(response.json()['API error'])
+
+
+@retry(stop_max_attempt_number=5, wait_fixed=2000)
+def TTS(text, volume=-20, out_wav='out.wav', enhanced=enable_sr, speaker_id='', speaker_npz=''):
+    url = f'http://{localhost_addr}:{service_port}/generate_speech'
+    data = {
+    'text': f'{text}',
+    'speaker_id': f'{speaker_id}',
+    'speaker_npz': f'{speaker_npz}',
+    'volume': f'{volume}',
+    'output_wav': f'{out_wav}',
+    }
+
+    response = requests.post(url, json=data)
+
+    if response.status_code == 200:
+        print('Success:', response.json()['message'])
+    else:
+        print('Error:', response.json()['API error'])
+        raise RuntimeError(response.json()['API error'])
+
+    if enhanced:
+        SR(processfile=out_wav)
+
+
+@retry(stop_max_attempt_number=5, wait_fixed=2000)
+def SR(processfile):
+    url = f'http://{localhost_addr}:{service_port}/fix_audio'
+    data = {'processfile': f'{processfile}'}
+
+    response = requests.post(url, json=data)
+
+    if response.status_code == 200:
+        print('Success:', response.json()['message'])
+    else:
+        print('Error:', response.json()['API error'])
+        raise RuntimeError(response.json()['API error'])
+
+
+@retry(stop_max_attempt_number=5, wait_fixed=2000)
+def VP(wav_path, out_dir):
+    url = f'http://{localhost_addr}:{service_port}/parse_voice'
+    data = {
+        'wav_path': f'{wav_path}', 
+        'out_dir':f'{out_dir}'
+    }
+
+    response = requests.post(url, json=data)
+
+    if response.status_code == 200:
+        print('Success:', response.json()['message'])
+    else:
+        print('Error:', response.json()['API error'])
+        raise RuntimeError(response.json()['API error'])
+

Dockerfile

@@ -0,0 +1,75 @@
+FROM python:3.11
+
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    git-lfs \
+    wget \
+    curl \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    # gradio dependencies \
+    ffmpeg \
+    # fairseq2 dependencies \
+    libsndfile-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Install miniconda
+RUN apt-get install -y wget && rm -rf /var/lib/apt/lists/*
+
+RUN wget \
+	https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+	&& bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+	&& rm -f Miniconda3-latest-Linux-x86_64.sh
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Add conda binary to PATH variable
+ENV HOME=/home/user \
+	PATH=/opt/miniconda3/bin:/home/user/.local/bin:$PATH \
+	CONDA_PREFIX=/opt/miniconda3/envs
+
+# Setup conda envs
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+
+# Conda envs setup 
+RUN bash ./scripts/EnvsSetup.sh
+
+# pre-download all models
+RUN conda run --live-stream -n WavJourney python scripts/download_models.py
+RUN mkdir $HOME/app/services_logs
+
+# Env settings to get docker images to work on HF Spaces
+ENV PYTHONPATH=${HOME}/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+
+# entrypoint
+ENTRYPOINT bash /home/user/app/scripts/start_service_and_ui.sh

Envs/AudioCraft.yml

@@ -0,0 +1,237 @@
+name: WavJourney
+channels:
+  - nvidia/label/cuda-11.8.0
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h7f98852_4
+  - ca-certificates=2023.05.30=h06a4308_0
+  - cuda-cccl=11.8.89=0
+  - cuda-command-line-tools=11.8.0=0
+  - cuda-compiler=11.8.0=0
+  - cuda-cudart=11.8.89=0
+  - cuda-cudart-dev=11.8.89=0
+  - cuda-cuobjdump=11.8.86=0
+  - cuda-cupti=11.8.87=0
+  - cuda-cuxxfilt=11.8.86=0
+  - cuda-documentation=11.8.86=0
+  - cuda-driver-dev=11.8.89=0
+  - cuda-gdb=11.8.86=0
+  - cuda-libraries=11.8.0=0
+  - cuda-libraries-dev=11.8.0=0
+  - cuda-memcheck=11.8.86=0
+  - cuda-nsight=11.8.86=0
+  - cuda-nsight-compute=11.8.0=0
+  - cuda-nvcc=11.8.89=0
+  - cuda-nvdisasm=11.8.86=0
+  - cuda-nvml-dev=11.8.86=0
+  - cuda-nvprof=11.8.87=0
+  - cuda-nvprune=11.8.86=0
+  - cuda-nvrtc=11.8.89=0
+  - cuda-nvrtc-dev=11.8.89=0
+  - cuda-nvtx=11.8.86=0
+  - cuda-nvvp=11.8.87=0
+  - cuda-profiler-api=11.8.86=0
+  - cuda-sanitizer-api=11.8.86=0
+  - cuda-toolkit=11.8.0=0
+  - cuda-tools=11.8.0=0
+  - cuda-visual-tools=11.8.0=0
+  - gds-tools=1.4.0.31=0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libcublas=11.11.3.6=0
+  - libcublas-dev=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufft-dev=10.9.0.58=0
+  - libcufile=1.4.0.31=0
+  - libcufile-dev=1.4.0.31=0
+  - libcurand=10.3.0.86=0
+  - libcurand-dev=10.3.0.86=0
+  - libcusolver=11.4.1.48=0
+  - libcusolver-dev=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libcusparse-dev=11.7.5.86=0
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libgomp=13.1.0=he5830b7_0
+  - libnpp=11.8.0.86=0
+  - libnpp-dev=11.8.0.86=0
+  - libnsl=2.0.0=h7f98852_0
+  - libnvjpeg=11.9.0.86=0
+  - libnvjpeg-dev=11.9.0.86=0
+  - libsqlite=3.42.0=h2797004_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.2.13=hd590300_5
+  - ncurses=6.4=hcb278e6_0
+  - nsight-compute=2022.3.0.22=0
+  - openssl=3.1.1=hd590300_1
+  - pip=23.1.2=pyhd8ed1ab_0
+  - python=3.8.17=he550d4f_0_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=68.0.0=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - wheel=0.40.0=pyhd8ed1ab_0
+  - xz=5.2.6=h166bdaf_0
+  - pip:
+      - aiofiles==23.1.0
+      - aiohttp==3.8.4
+      - aiosignal==1.3.1
+      - altair==5.0.1
+      - antlr4-python3-runtime==4.9.3
+      - anyio==3.7.1
+      - appdirs==1.4.4
+      - async-timeout==4.0.2
+      - attrs==23.1.0
+      - audioread==3.0.0
+      - av==10.0.0
+      - blinker==1.6.2
+      - blis==0.7.9
+      - catalogue==2.0.8
+      - certifi==2023.5.7
+      - cffi==1.15.1
+      - charset-normalizer==3.2.0
+      - click==8.1.5
+      - cloudpickle==2.2.1
+      - cmake==3.26.4
+      - colorlog==6.7.0
+      - confection==0.1.0
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - cymem==2.0.7
+      - cython==0.29.36
+      - decorator==5.1.1
+      - demucs==4.0.0
+      - diffq==0.2.4
+      - docopt==0.6.2
+      - dora-search==0.1.12
+      - einops==0.6.1
+      - encodec==0.1.1
+      - exceptiongroup==1.1.2
+      - fastapi==0.100.0
+      - ffmpy==0.3.0
+      - filelock==3.12.2
+      - flashy==0.0.2
+      - flask==2.3.2
+      - fonttools==4.41.0
+      - frozenlist==1.4.0
+      - fsspec==2023.6.0
+      - future==0.18.3
+      - gradio==3.36.1
+      - gradio-client==0.2.9
+      - h11==0.14.0
+      - httpcore==0.17.3
+      - httpx==0.24.1
+      - huggingface-hub==0.16.4
+      - hydra-colorlog==1.2.0
+      - hydra-core==1.3.2
+      - idna==3.4
+      - importlib-metadata==6.8.0
+      - importlib-resources==6.0.0
+      - itsdangerous==2.1.2
+      - jinja2==3.1.2
+      - joblib==1.3.1
+      - jsonschema==4.18.3
+      - jsonschema-specifications==2023.6.1
+      - julius==0.2.7
+      - kiwisolver==1.4.4
+      - lameenc==1.5.1
+      - langcodes==3.3.0
+      - lazy-loader==0.3
+      - librosa==0.10.0.post2
+      - lightning-utilities==0.9.0
+      - linkify-it-py==2.0.2
+      - lit==16.0.6
+      - llvmlite==0.40.1
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.3
+      - matplotlib==3.7.2
+      - mdit-py-plugins==0.3.3
+      - mdurl==0.1.2
+      - mpmath==1.3.0
+      - msgpack==1.0.5
+      - multidict==6.0.4
+      - murmurhash==1.0.9
+      - mypy-extensions==1.0.0
+      - networkx==3.1
+      - num2words==0.5.12
+      - numba==0.57.1
+      - numpy==1.24.4
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - omegaconf==2.3.0
+      - openunmix==1.2.1
+      - orjson==3.9.2
+      - packaging==23.1
+      - pandas==2.0.3
+      - pathy==0.10.2
+      - pillow==10.0.0
+      - pkgutil-resolve-name==1.3.10
+      - pooch==1.6.0
+      - preshed==3.0.8
+      - pycparser==2.21
+      - pydantic==1.10.11
+      - pydub==0.25.1
+      - pygments==2.15.1
+      - pyloudnorm==0.1.1
+      - pyparsing==3.0.9
+      - pyre-extensions==0.0.29
+      - python-dateutil==2.8.2
+      - python-multipart==0.0.6
+      - pytz==2023.3
+      - pyyaml==6.0
+      - referencing==0.29.1
+      - regex==2023.6.3
+      - requests==2.31.0
+      - retrying==1.3.4
+      - rpds-py==0.8.10
+      - safetensors==0.3.1
+      - scikit-learn==1.3.0
+      - scipy==1.10.1
+      - semantic-version==2.10.0
+      - sentencepiece==0.1.99
+      - six==1.16.0
+      - smart-open==6.3.0
+      - sniffio==1.3.0
+      - soundfile==0.12.1
+      - soxr==0.3.5
+      - spacy==3.5.2
+      - spacy-legacy==3.0.12
+      - spacy-loggers==1.0.4
+      - srsly==2.4.6
+      - starlette==0.27.0
+      - submitit==1.4.5
+      - sympy==1.12
+      - thinc==8.1.10
+      - threadpoolctl==3.2.0
+      - tokenizers==0.13.3
+      - toolz==0.12.0
+      - torch==2.0.1
+      - torchaudio==2.0.2
+      - torchmetrics==1.0.1
+      - tqdm==4.65.0
+      - transformers==4.31.0
+      - treetable==0.2.5
+      - triton==2.0.0
+      - typer==0.7.0
+      - typing-extensions==4.7.1
+      - typing-inspect==0.9.0
+      - tzdata==2023.3
+      - uc-micro-py==1.0.2
+      - urllib3==2.0.3
+      - uvicorn==0.22.0
+      - wasabi==1.1.2
+      - websockets==11.0.3
+      - werkzeug==2.3.6
+      - xformers==0.0.20
+      - yarl==1.9.2
+      - zipp==3.16.2

Envs/Bark.yml

@@ -0,0 +1,179 @@
+name: WavJourney
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h7f98852_4
+  - ca-certificates=2023.5.7=hbcca054_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libgomp=13.1.0=he5830b7_0
+  - libnsl=2.0.0=h7f98852_0
+  - libsqlite=3.42.0=h2797004_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.2.13=hd590300_5
+  - ncurses=6.4=hcb278e6_0
+  - openssl=3.1.1=hd590300_1
+  - pip=23.1.2=pyhd8ed1ab_0
+  - python=3.8.17=he550d4f_0_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=68.0.0=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - wheel=0.40.0=pyhd8ed1ab_0
+  - xz=5.2.6=h166bdaf_0
+  - pip:
+      - aiohttp==3.8.5
+      - aiosignal==1.3.1
+      - altair==5.0.1
+      - appdirs==1.4.4
+      - asttokens==2.2.1
+      - async-timeout==4.0.3
+      - attrs==23.1.0
+      - audioread==3.0.0
+      - backcall==0.2.0
+      - backports-zoneinfo==0.2.1
+      - blinker==1.6.2
+      - boto3==1.28.3
+      - botocore==1.31.3
+      - cachetools==5.3.1
+      - certifi==2023.5.7
+      - cffi==1.15.1
+      - charset-normalizer==3.2.0
+      - click==8.1.5
+      - cmake==3.26.4
+      - coloredlogs==15.0.1
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - datasets==2.14.4
+      - decorator==5.1.1
+      - dill==0.3.7
+      - einops==0.6.1
+      - encodec==0.1.1
+      - executing==1.2.0
+      - filelock==3.12.2
+      - fire==0.5.0
+      - flask==2.3.2
+      - fonttools==4.41.0
+      - frozenlist==1.4.0
+      - fsspec==2023.6.0
+      - funcy==2.0
+      - future==0.18.3
+      - gitdb==4.0.10
+      - gitpython==3.1.32
+      - huggingface-hub==0.16.4
+      - humanfriendly==10.0
+      - idna==3.4
+      - importlib-metadata==6.8.0
+      - importlib-resources==6.0.0
+      - ipdb==0.13.13
+      - ipython==8.12.2
+      - itsdangerous==2.1.2
+      - jedi==0.19.0
+      - jinja2==3.1.2
+      - jmespath==1.0.1
+      - joblib==1.3.1
+      - jsonschema==4.18.3
+      - jsonschema-specifications==2023.6.1
+      - kiwisolver==1.4.4
+      - lazy-loader==0.3
+      - librosa==0.10.0.post2
+      - lit==16.0.6
+      - llvmlite==0.40.1
+      - markdown-it-py==3.0.0
+      - markupsafe==2.1.3
+      - matplotlib==3.7.2
+      - matplotlib-inline==0.1.6
+      - mdurl==0.1.2
+      - mpmath==1.3.0
+      - msgpack==1.0.5
+      - multidict==6.0.4
+      - multiprocess==0.70.15
+      - networkx==3.1
+      - nltk==3.8.1
+      - numba==0.57.1
+      - numpy==1.24.4
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - optimum==1.11.1
+      - packaging==23.1
+      - pandas==2.0.3
+      - parso==0.8.3
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==9.5.0
+      - pkgutil-resolve-name==1.3.10
+      - pooch==1.6.0
+      - progressbar==2.5
+      - prompt-toolkit==3.0.39
+      - protobuf==4.23.4
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pyarrow==12.0.1
+      - pycparser==2.21
+      - pydeck==0.8.1b0
+      - pygments==2.15.1
+      - pyloudnorm==0.1.1
+      - pympler==1.0.1
+      - pyparsing==3.0.9
+      - python-dateutil==2.8.2
+      - pytz==2023.3
+      - pytz-deprecation-shim==0.1.0.post0
+      - pyyaml==6.0
+      - referencing==0.29.1
+      - regex==2023.6.3
+      - requests==2.31.0
+      - resampy==0.4.2
+      - retrying==1.3.4
+      - rich==13.4.2
+      - rpds-py==0.8.10
+      - s3transfer==0.6.1
+      - safetensors==0.3.1
+      - scikit-learn==1.3.0
+      - scipy==1.10.1
+      - sentencepiece==0.1.99
+      - six==1.16.0
+      - smmap==5.0.0
+      - soundfile==0.12.1
+      - soxr==0.3.5
+      - stack-data==0.6.2
+      - streamlit==1.24.1
+      - suno-bark==0.1.5
+      - sympy==1.12
+      - tenacity==8.2.2
+      - termcolor==2.3.0
+      - threadpoolctl==3.2.0
+      - tokenizers==0.13.3
+      - toml==0.10.2
+      - tomli==2.0.1
+      - toolz==0.12.0
+      - torch==2.0.1
+      - torchaudio==2.0.2
+      - torchlibrosa==0.0.7
+      - tornado==6.3.2
+      - tqdm==4.65.0
+      - traitlets==5.9.0
+      - transformers==4.31.0
+      - triton==2.0.0
+      - typing-extensions==4.7.1
+      - tzdata==2023.3
+      - tzlocal==4.3.1
+      - urllib3==1.26.16
+      - validators==0.20.0
+      - watchdog==3.0.0
+      - wcwidth==0.2.6
+      - werkzeug==2.3.6
+      - xxhash==3.3.0
+      - yarl==1.9.2
+      - zipp==3.16.1

Envs/VoiceFixer.yml

@@ -0,0 +1,123 @@
+name: VoiceFixer
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2023.05.30=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.9=h7f8727e_0
+  - pip=23.2.1=py38h06a4308_0
+  - python=3.8.17=h955ad1f_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.0.0=py38h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - tk=8.6.12=h1ccaba5_0
+  - wheel=0.38.4=py38h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - altair==5.0.1
+      - attrs==23.1.0
+      - audioread==3.0.0
+      - backports-zoneinfo==0.2.1
+      - blinker==1.6.2
+      - cachetools==5.3.1
+      - certifi==2023.5.7
+      - cffi==1.15.1
+      - charset-normalizer==3.2.0
+      - click==8.1.5
+      - cmake==3.27.0
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - decorator==5.1.1
+      - filelock==3.12.2
+      - flask==2.3.2
+      - fonttools==4.38.0
+      - gitdb==4.0.10
+      - gitpython==3.1.32
+      - idna==3.4
+      - importlib-metadata==6.7.0
+      - importlib-resources==5.12.0
+      - itsdangerous==2.1.2
+      - jinja2==3.1.2
+      - joblib==1.3.1
+      - jsonschema==4.17.3
+      - jsonschema-specifications==2023.7.1
+      - kiwisolver==1.4.4
+      - librosa==0.8.1
+      - lit==16.0.6
+      - llvmlite==0.39.1
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.3
+      - matplotlib==3.5.3
+      - mdurl==0.1.2
+      - mpmath==1.3.0
+      - networkx==3.1
+      - numba==0.56.4
+      - numpy==1.21.6
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - packaging==23.1
+      - pandas==1.3.5
+      - pillow==9.5.0
+      - pkgutil-resolve-name==1.3.10
+      - platformdirs==3.9.1
+      - pooch==1.7.0
+      - progressbar==2.5
+      - protobuf==4.23.4
+      - pyarrow==12.0.1
+      - pycparser==2.21
+      - pydeck==0.8.1b0
+      - pygments==2.15.1
+      - pympler==1.0.1
+      - pyparsing==3.1.0
+      - pyrsistent==0.19.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3
+      - pytz-deprecation-shim==0.1.0.post0
+      - pyyaml==6.0.1
+      - referencing==0.30.0
+      - requests==2.31.0
+      - resampy==0.4.2
+      - retrying==1.3.4
+      - rich==13.4.2
+      - rpds-py==0.9.2
+      - scikit-learn==1.0.2
+      - scipy==1.7.3
+      - six==1.16.0
+      - smmap==5.0.0
+      - soundfile==0.12.1
+      - streamlit==1.23.1
+      - sympy==1.12
+      - tenacity==8.2.2
+      - threadpoolctl==3.1.0
+      - toml==0.10.2
+      - toolz==0.12.0
+      - torch==1.13.1
+      - torchlibrosa==0.0.7
+      - tornado==6.2
+      - triton==2.0.0
+      - typing-extensions==4.7.1
+      - tzdata==2023.3
+      - tzlocal==4.3.1
+      - urllib3==2.0.3
+      - validators==0.20.0
+      - voicefixer==0.1.2
+      - watchdog==3.0.0
+      - werkzeug==2.3.6
+      - zipp==3.15.0

Envs/WavJourney.yml

@@ -0,0 +1,248 @@
+name: WavJourney
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - aom=3.5.0=h27087fc_0
+  - bzip2=1.0.8=h7f98852_4
+  - ca-certificates=2023.7.22=hbcca054_0
+  - cairo=1.16.0=hbbf8b49_1016
+  - dav1d=1.2.1=hd590300_0
+  - expat=2.5.0=hcb278e6_1
+  - ffmpeg=6.0.0=gpl_hdbbbd96_103
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=hab24e00_0
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - freetype=2.12.1=hca18f0e_1
+  - fribidi=1.0.10=h36c2ea0_0
+  - gettext=0.21.1=h27087fc_0
+  - gmp=6.2.1=h58526e2_0
+  - gnutls=3.7.8=hf3e180e_0
+  - graphite2=1.3.13=h58526e2_1001
+  - harfbuzz=7.3.0=hdb3a94d_0
+  - icu=72.1=hcb278e6_0
+  - lame=3.100=h166bdaf_1003
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libass=0.17.1=hc9aadba_0
+  - libdrm=2.4.114=h166bdaf_0
+  - libexpat=2.5.0=hcb278e6_1
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libglib=2.76.4=hebfc3b9_0
+  - libgomp=13.1.0=he5830b7_0
+  - libiconv=1.17=h166bdaf_0
+  - libidn2=2.3.4=h166bdaf_0
+  - libnsl=2.0.0=h7f98852_0
+  - libopus=1.3.1=h7f98852_1
+  - libpciaccess=0.17=h166bdaf_0
+  - libpng=1.6.39=h753d276_0
+  - libsqlite=3.42.0=h2797004_0
+  - libstdcxx-ng=13.1.0=hfd8a6a1_0
+  - libtasn1=4.19.0=h166bdaf_0
+  - libunistring=0.9.10=h7f98852_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libva=2.19.0=hd590300_0
+  - libvpx=1.13.0=hcb278e6_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxml2=2.11.5=h0d562d8_0
+  - libzlib=1.2.13=hd590300_5
+  - ncurses=6.4=hcb278e6_0
+  - nettle=3.8.1=hc379101_1
+  - openh264=2.3.1=hcb278e6_2
+  - openssl=3.1.2=hd590300_0
+  - p11-kit=0.24.1=hc5aa10d_0
+  - pcre2=10.40=hc3806b6_0
+  - pip=23.2=pyhd8ed1ab_0
+  - pixman=0.40.0=h36c2ea0_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - python=3.8.17=he550d4f_0_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=68.0.0=pyhd8ed1ab_0
+  - svt-av1=1.6.0=h59595ed_0
+  - tk=8.6.12=h27826a3_0
+  - wheel=0.40.0=pyhd8ed1ab_1
+  - x264=1!164.3095=h166bdaf_2
+  - x265=3.5=h924138e_3
+  - xorg-fixesproto=5.0=h7f98852_1002
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.6=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxfixes=5.0.3=h7f98852_1004
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - zlib=1.2.13=hd590300_5
+  - pip:
+      - accelerate==0.21.0
+      - aiofiles==23.1.0
+      - aiohttp==3.8.5
+      - aiosignal==1.3.1
+      - altair==5.0.1
+      - annotated-types==0.5.0
+      - antlr4-python3-runtime==4.8
+      - anyio==3.7.1
+      - appdirs==1.4.4
+      - asttokens==2.2.1
+      - async-timeout==4.0.2
+      - attrs==23.1.0
+      - audiolm-pytorch==1.1.4
+      - audioread==3.0.0
+      - backcall==0.2.0
+      - beartype==0.15.0
+      - bitarray==2.8.1
+      - blinker==1.6.2
+      - certifi==2023.5.7
+      - cffi==1.15.1
+      - charset-normalizer==3.2.0
+      - click==8.1.6
+      - cmake==3.26.4
+      - colorama==0.4.6
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - cython==3.0.0
+      - decorator==5.1.1
+      - einops==0.6.1
+      - ema-pytorch==0.2.3
+      - encodec==0.1.1
+      - exceptiongroup==1.1.2
+      - executing==1.2.0
+      - fairseq==0.12.2
+      - fastapi==0.100.1
+      - ffmpy==0.3.1
+      - filelock==3.12.2
+      - flask==2.3.2
+      - fonttools==4.42.0
+      - frozenlist==1.4.0
+      - fsspec==2023.6.0
+      - future==0.18.3
+      - gradio==3.39.0
+      - gradio-client==0.3.0
+      - h11==0.14.0
+      - httpcore==0.17.3
+      - httpx==0.24.1
+      - huggingface-hub==0.16.4
+      - hydra-core==1.0.7
+      - idna==3.4
+      - importlib-metadata==6.8.0
+      - importlib-resources==6.0.0
+      - ipdb==0.13.13
+      - ipython==8.12.2
+      - itsdangerous==2.1.2
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.3.1
+      - json5==0.9.14
+      - jsonschema==4.18.6
+      - jsonschema-specifications==2023.7.1
+      - kiwisolver==1.4.4
+      - lazy-loader==0.3
+      - librosa==0.10.0.post2
+      - linkify-it-py==2.0.2
+      - lion-pytorch==0.1.2
+      - lit==16.0.6
+      - llvmlite==0.40.1
+      - local-attention==1.8.6
+      - lxml==4.9.3
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.3
+      - matplotlib==3.7.2
+      - matplotlib-inline==0.1.6
+      - mdit-py-plugins==0.3.3
+      - mdurl==0.1.2
+      - mpmath==1.3.0
+      - msgpack==1.0.5
+      - multidict==6.0.4
+      - networkx==3.1
+      - nltk==3.8.1
+      - numba==0.57.1
+      - numpy==1.24.4
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - omegaconf==2.0.6
+      - openai==0.27.8
+      - orjson==3.9.2
+      - packaging==23.1
+      - pandas==2.0.3
+      - parso==0.8.3
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==10.0.0
+      - pkgutil-resolve-name==1.3.10
+      - pooch==1.6.0
+      - portalocker==2.7.0
+      - prompt-toolkit==3.0.39
+      - psutil==5.9.5
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pycparser==2.21
+      - pydantic==2.1.1
+      - pydantic-core==2.4.0
+      - pydub==0.25.1
+      - pygments==2.15.1
+      - pyloudnorm==0.1.1
+      - pyparsing==3.0.9
+      - python-dateutil==2.8.2
+      - python-multipart==0.0.6
+      - pytz==2023.3
+      - pyyaml==6.0.1
+      - referencing==0.30.1
+      - regex==2023.6.3
+      - requests==2.31.0
+      - retrying==1.3.4
+      - rpds-py==0.9.2
+      - sacrebleu==2.3.1
+      - safetensors==0.3.2
+      - scikit-learn==1.3.0
+      - scipy==1.10.1
+      - semantic-version==2.10.0
+      - sentencepiece==0.1.99
+      - six==1.16.0
+      - sniffio==1.3.0
+      - soundfile==0.12.1
+      - soxr==0.3.5
+      - stack-data==0.6.2
+      - starlette==0.27.0
+      - sympy==1.12
+      - tabulate==0.9.0
+      - threadpoolctl==3.2.0
+      - tokenizers==0.13.3
+      - tomli==2.0.1
+      - toolz==0.12.0
+      - torch==2.0.1
+      - torchaudio==2.0.2
+      - tqdm==4.65.0
+      - traitlets==5.9.0
+      - transformers==4.31.0
+      - triton==2.0.0
+      - typing-extensions==4.7.1
+      - tzdata==2023.3
+      - uc-micro-py==1.0.2
+      - urllib3==2.0.4
+      - uvicorn==0.23.2
+      - vector-quantize-pytorch==1.6.30
+      - wcwidth==0.2.6
+      - websockets==11.0.3
+      - werkzeug==2.3.6
+      - yarl==1.9.2
+      - zipp==3.16.2

LICENSE

@@ -0,0 +1,251 @@
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to be
+bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+("Public License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You such
+rights in consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+    a. Adapted Material means material subject to Copyright and Similar Rights
+       that is derived from or based upon the Licensed Material and in which
+       the Licensed Material is translated, altered, arranged, transformed, or
+       otherwise modified in a manner requiring permission under the Copyright
+       and Similar Rights held by the Licensor. For purposes of this Public
+       License, where the Licensed Material is a musical work, performance, or
+       sound recording, Adapted Material is always produced where the Licensed
+       Material is synched in timed relation with a moving image.
+    b. Copyright and Similar Rights means copyright and/or similar rights
+       closely related to copyright including, without limitation,
+       performance, broadcast, sound recording, and Sui Generis Database
+       Rights, without regard to how the rights are labeled or categorized.
+       For purposes of this Public License, the rights specified in Section
+       2(b)(1)-(2) are not Copyright and Similar Rights.
+    c. Effective Technological Measures means those measures that, in the
+       absence of proper authority, may not be circumvented under laws
+       fulfilling obligations under Article 11 of the WIPO Copyright Treaty
+       adopted on December 20, 1996, and/or similar international agreements.
+    d. Exceptions and Limitations means fair use, fair dealing, and/or any
+       other exception or limitation to Copyright and Similar Rights that
+       applies to Your use of the Licensed Material.
+    e. Licensed Material means the artistic or literary work, database, or
+       other material to which the Licensor applied this Public License.
+    f. Licensed Rights means the rights granted to You subject to the terms
+       and conditions of this Public License, which are limited to all
+       Copyright and Similar Rights that apply to Your use of the Licensed
+       Material and that the Licensor has authority to license.
+    g. Licensor means the individual(s) or entity(ies) granting rights under
+       this Public License.
+    h. NonCommercial means not primarily intended for or directed towards
+       commercial advantage or monetary compensation. For purposes of this
+       Public License, the exchange of the Licensed Material for other
+       material subject to Copyright and Similar Rights by digital
+       file-sharing or similar means is NonCommercial provided there is no
+       payment of monetary compensation in connection with the exchange.
+    i. Share means to provide material to the public by any means or process
+       that requires permission under the Licensed Rights, such as
+       reproduction, public display, public performance, distribution,
+       dissemination, communication, or importation, and to make material
+       available to the public including in ways that members of the public
+       may access the material from a place and at a time individually chosen
+       by them.
+    j. Sui Generis Database Rights means rights other than copyright resulting
+       from Directive 96/9/EC of the European Parliament and of the Council of
+       11 March 1996 on the legal protection of databases, as amended and/or
+       succeeded, as well as other essentially equivalent rights anywhere in
+       the world.
+    k. You means the individual or entity exercising the Licensed Rights under
+       this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+    a. License grant.
+        1. Subject to the terms and conditions of this Public License, the
+           Licensor hereby grants You a worldwide, royalty-free,
+           non-sublicensable, non-exclusive, irrevocable license to exercise
+           the Licensed Rights in the Licensed Material to:
+            A. reproduce and Share the Licensed Material, in whole or in part,
+               for NonCommercial purposes only; and
+            B. produce and reproduce, but not Share, Adapted Material for
+               NonCommercial purposes only.
+        2. Exceptions and Limitations. For the avoidance of doubt, where
+           Exceptions and Limitations apply to Your use, this Public License
+           does not apply, and You do not need to comply with its terms and
+           conditions.
+        3. Term. The term of this Public License is specified in Section 6(a).
+        4. Media and formats; technical modifications allowed. The Licensor
+           authorizes You to exercise the Licensed Rights in all media and
+           formats whether now known or hereafter created, and to make
+           technical modifications necessary to do so. The Licensor waives
+           and/or agrees not to assert any right or authority to forbid You
+           from making technical modifications necessary to exercise the
+           Licensed Rights, including technical modifications necessary to
+           circumvent Effective Technological Measures. For purposes of this
+           Public License, simply making modifications authorized by this
+           Section 2(a)(4) never produces Adapted Material.
+        5. Downstream recipients.
+            A. Offer from the Licensor – Licensed Material. Every recipient of
+               the Licensed Material automatically receives an offer from the
+               Licensor to exercise the Licensed Rights under the terms and
+               conditions of this Public License.
+            B. No downstream restrictions. You may not offer or impose any
+               additional or different terms or conditions on, or apply any
+               Effective Technological Measures to, the Licensed Material if
+               doing so restricts exercise of the Licensed Rights by any
+               recipient of the Licensed Material.
+        6. No endorsement. Nothing in this Public License constitutes or may
+           be construed as permission to assert or imply that You are, or that
+           Your use of the Licensed Material is, connected with, or sponsored,
+           endorsed, or granted official status by, the Licensor or others
+           designated to receive attribution as provided in Section
+           3(a)(1)(A)(i).
+
+    b. Other rights.
+        1. Moral rights, such as the right of integrity, are not licensed
+           under this Public License, nor are publicity, privacy, and/or other
+           similar personality rights; however, to the extent possible, the
+           Licensor waives and/or agrees not to assert any such rights held by
+           the Licensor to the limited extent necessary to allow You to
+           exercise the Licensed Rights, but not otherwise.
+        2. Patent and trademark rights are not licensed under this Public
+           License.
+        3. To the extent possible, the Licensor waives any right to collect
+           royalties from You for the exercise of the Licensed Rights, whether
+           directly or through a collecting society under any voluntary or
+           waivable statutory or compulsory licensing scheme. In all other
+           cases the Licensor expressly reserves any right to collect such
+           royalties, including when the Licensed Material is used other than
+           for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following conditions.
+
+    a. Attribution.
+
+        1. If You Share the Licensed Material, You must:
+            A. retain the following if it is supplied by the Licensor with the
+               Licensed Material:
+                i. identification of the creator(s) of the Licensed Material
+                   and any others designated to receive attribution, in any
+                   reasonable manner requested by the Licensor (including by
+                   pseudonym if designated);
+                ii. a copyright notice;
+                iii. a notice that refers to this Public License;
+                iv. a notice that refers to the disclaimer of warranties;
+                v. a URI or hyperlink to the Licensed Material to the extent
+                   reasonably practicable;
+            B. indicate if You modified the Licensed Material and retain an
+               indication of any previous modifications; and
+            C. indicate the Licensed Material is licensed under this Public
+               License, and include the text of, or the URI or hyperlink to,
+               this Public License.
+
+            For the avoidance of doubt, You do not have permission under this
+            Public License to Share Adapted Material.
+
+        2. You may satisfy the conditions in Section 3(a)(1) in any reasonable
+           manner based on the medium, means, and context in which You Share
+           the Licensed Material. For example, it may be reasonable to satisfy
+           the conditions by providing a URI or hyperlink to a resource that
+           includes the required information.
+        3. If requested by the Licensor, You must remove any of the
+           information required by Section 3(a)(1)(A) to the extent reasonably
+           practicable.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to
+Your use of the Licensed Material:
+
+    a. for the avoidance of doubt, Section 2(a)(1) grants You the right to
+       extract, reuse, reproduce, and Share all or a substantial portion of
+       the contents of the database for NonCommercial purposes only and
+       provided You do not Share Adapted Material;
+    b. if You include all or a substantial portion of the database contents in
+       a database in which You have Sui Generis Database Rights, then the
+       database in which You have Sui Generis Database Rights (but not its
+       individual contents) is Adapted Material; and
+    c. You must comply with the conditions in Section 3(a) if You Share all or
+       a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not replace
+Your obligations under this Public License where the Licensed Rights include
+other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+    a. Unless otherwise separately undertaken by the Licensor, to the extent
+       possible, the Licensor offers the Licensed Material as-is and
+       as-available, and makes no representations or warranties of any kind
+       concerning the Licensed Material, whether express, implied, statutory,
+       or other. This includes, without limitation, warranties of title,
+       merchantability, fitness for a particular purpose, non-infringement,
+       absence of latent or other defects, accuracy, or the presence or
+       absence of errors, whether or not known or discoverable. Where
+       disclaimers of warranties are not allowed in full or in part, this
+       disclaimer may not apply to You.
+    b. To the extent possible, in no event will the Licensor be liable to You
+       on any legal theory (including, without limitation, negligence) or
+       otherwise for any direct, special, indirect, incidental, consequential,
+       punitive, exemplary, or other losses, costs, expenses, or damages
+       arising out of this Public License or use of the Licensed Material,
+       even if the Licensor has been advised of the possibility of such
+       losses, costs, expenses, or damages. Where a limitation of liability is
+       not allowed in full or in part, this limitation may not apply to You.
+    c. The disclaimer of warranties and limitation of liability provided above
+       shall be interpreted in a manner that, to the extent possible, most
+       closely approximates an absolute disclaimer and waiver of all
+       liability.
+
+Section 6 – Term and Termination.
+
+    a. This Public License applies for the term of the Copyright and Similar
+       Rights licensed here. However, if You fail to comply with this Public
+       License, then Your rights under this Public License terminate
+       automatically.
+    b. Where Your right to use the Licensed Material has terminated under
+       Section 6(a), it reinstates:
+        1. automatically as of the date the violation is cured, provided it is
+           cured within 30 days of Your discovery of the violation; or
+        2. upon express reinstatement by the Licensor.
+
+      For the avoidance of doubt, this Section 6(b) does not affect any right
+      the Licensor may have to seek remedies for Your violations of this
+      Public License.
+
+    c. For the avoidance of doubt, the Licensor may also offer the Licensed
+       Material under separate terms or conditions or stop distributing the
+       Licensed Material at any time; however, doing so will not terminate
+       this Public License.
+    d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 – Other Terms and Conditions.
+
+    a. The Licensor shall not be bound by any additional or different terms or
+       conditions communicated by You unless expressly agreed.
+    b. Any arrangements, understandings, or agreements regarding the Licensed
+       Material not stated herein are separate from and independent of the
+       terms and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+    a. For the avoidance of doubt, this Public License does not, and shall not
+       be interpreted to, reduce, limit, restrict, or impose conditions on any
+       use of the Licensed Material that could lawfully be made without
+       permission under this Public License.
+    b. To the extent possible, if any provision of this Public License is
+       deemed unenforceable, it shall be automatically reformed to the minimum
+       extent necessary to make it enforceable. If the provision cannot be
+       reformed, it shall be severed from this Public License without
+       affecting the enforceability of the remaining terms and conditions.
+    c. No term or condition of this Public License will be waived and no
+       failure to comply consented to unless expressly agreed to by the Licensor.
+    d. Nothing in this Public License constitutes or may be interpreted as a
+       limitation upon, or waiver of, any privileges and immunities that apply
+       to the Licensor or You, including from the legal processes of any
+       jurisdiction or authority.

README.md

@@ -0,0 +1,112 @@
+---
+title: WavJourney
+emoji: 🔥
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+license: cc-by-nc-4.0
+duplicated_from: Audio-AGI/WavJourney
+---
+# <span style="color: blue;">🎵</span> WavJourney: Compositional Audio Creation with LLMs
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2307.14335) [![GitHub Stars](https://img.shields.io/github/stars/Audio-AGI/WavJourney?style=social)](https://github.com/Audio-AGI/WavJourney/) [![githubio](https://img.shields.io/badge/GitHub.io-Demo_Page-blue?logo=Github&style=flat-square)](https://audio-agi.github.io/WavJourney_demopage/) [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/Audio-AGI/WavJourney)  
+
+
+This repository contains the official implementation of ["WavJourney: Compositional Audio Creation with Large Language Models"](https://audio-agi.github.io/WavJourney_demopage/WavJourney_arXiv.pdf).
+
+Starting with a text prompt, WavJourney can create audio content with engaging storylines encompassing personalized speakers, lifelike speech in context, emotionally resonant music compositions, and impactful sound effects that enhance the auditory experience. Check the audio examples in the [Project Page](https://audio-agi.github.io/WavJourney_demopage/)!
+
+<!-- <p align="center">
+  <img align="middle" width="800" src="assets/WavJourney.png"/>
+</p> -->
+
+<hr>
+
+
+## Preliminaries
+1. Install the environment:
+```bash
+bash ./scripts/EnvsSetup.sh
+```
+2. Activate the conda environment:
+```bash
+conda activate WavJourney
+```
+
+3. (Optional) You can modify the default configuration in `config.yaml`, check the details described in the configuration file. 
+4. Pre-download the models (might take some time):
+```bash
+python scripts/download_models.py
+```
+
+5. Set the WAVJOURNEY_OPENAI_KEY in the environment variable for accessing [GPT-4 API](https://platform.openai.com/account/api-keys) [[Guidance](https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4)]
+```bash
+export WAVJOURNEY_OPENAI_KEY=your_openai_key_here
+```
+
+6. Set environment variables for using API services
+```bash
+# Set the port for the WAVJOURNEY service to 8021
+export WAVJOURNEY_SERVICE_PORT=8021
+
+# Set the URL for the WAVJOURNEY service to 127.0.0.1
+export WAVJOURNEY_SERVICE_URL=127.0.0.1
+
+# Limit the maximum script lines for WAVJOURNEY to 999
+export WAVJOURNEY_MAX_SCRIPT_LINES=999
+```
+
+
+7. Start Python API services (e.g., Text-to-Speech, Text-to-Audio)
+```bash
+bash scripts/start_services.sh
+```
+
+## Web APP
+ ```bash
+bash scripts/start_ui.sh
+  ```
+
+## Commandline Usage
+ ```bash
+ python wavjourney_cli.py -f --input-text "Generate a one-minute introduction to quantum mechanics" 
+ ```
+
+
+## Kill the services
+You can kill the running services via this command:
+ ```bash
+python scripts/kill_services.py
+  ```
+  
+## (Advanced features) Speaker customization 
+You can add voice presets to WavJourney to customize the voice actors. Simply provide the voice id, the description and a sample wav file, and WavJourney will pick the voice automatically based on the audio script. Predefined system voice presets are in `data/voice_presets`.
+
+You can manage voice presets via UI. Specifically, if you want to add voice to voice presets. Run the script via command line below:
+```bash
+python add_voice_preset.py --id "id" --desc "description" --wav-path path/to/wav --session-id ''
+```
+What makes for good voice prompt? See detailed instructions <a href="https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer">here</a>. 
+## Hardware requirement
+- The VRAM of the GPU in the default configuration should be greater than 16 GB.
+- Operation system: Linux.
+
+## Citation
+If you find this work useful, you can cite the paper below:
+
+    @article{liu2023wavjourney,
+        title   = {WavJourney: Compositional Audio Creation with Large Language Models},
+        author  = {Liu, Xubo and Zhu, Zhongkai and Liu, Haohe and Yuan, Yi and Huang, Qiushi and Liang, Jinhua and Cao, Yin and Kong, Qiuqiang and Plumbley, Mark D and Wang, Wenwu},
+        journal = {arXiv preprint arXiv:2307.14335},
+        year    = {2023}
+    }
+
+[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/liuxubo)
+
+## Appreciation
+- [Bark](https://github.com/suno-ai/bark) for a zero-shot text-to-speech synthesis model.
+- [AudioCraft](https://github.com/facebookresearch/audiocraft) for state-of-the-art audio generation models.
+
+## Disclaimer
+We are not responsible for audio generated using semantics created by this model. Just don't use it for illegal purposes.
+

VoiceParser/customtokenizer.py

@@ -0,0 +1,202 @@
+"""
+Custom tokenizer model.
+Author: https://www.github.com/gitmylo/
+License: MIT
+"""
+
+import json
+import os.path
+from zipfile import ZipFile
+from typing import Union
+
+
+import numpy
+import torch
+from torch import nn, optim
+from torch.serialization import MAP_LOCATION
+
+
+class CustomTokenizer(nn.Module):
+    def __init__(self, hidden_size=1024, input_size=768, output_size=10000, version=0):
+        super(CustomTokenizer, self).__init__()
+        next_size = input_size
+        if version == 0:
+            self.lstm = nn.LSTM(input_size, hidden_size, 2, batch_first=True)
+            next_size = hidden_size
+        if version == 1:
+            self.lstm = nn.LSTM(input_size, hidden_size, 2, batch_first=True)
+            self.intermediate = nn.Linear(hidden_size, 4096)
+            next_size = 4096
+
+        self.fc = nn.Linear(next_size, output_size)
+        self.softmax = nn.LogSoftmax(dim=1)
+        self.optimizer: optim.Optimizer = None
+        self.lossfunc = nn.CrossEntropyLoss()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.version = version
+
+    def forward(self, x):
+        x, _ = self.lstm(x)
+        if self.version == 1:
+            x = self.intermediate(x)
+        x = self.fc(x)
+        x = self.softmax(x)
+        return x
+
+    @torch.no_grad()
+    def get_token(self, x):
+        """
+        Used to get the token for the first
+        :param x: An array with shape (N, input_size) where N is a whole number greater or equal to 1, and input_size is the input size used when creating the model.
+        :return: An array with shape (N,) where N is the same as N from the input. Every number in the array is a whole number in range 0...output_size - 1 where output_size is the output size used when creating the model.
+        """
+        return torch.argmax(self(x), dim=1)
+
+    def prepare_training(self):
+        self.optimizer = optim.Adam(self.parameters(), 0.001)
+
+    def train_step(self, x_train, y_train, log_loss=False):
+        # y_train = y_train[:-1]
+        # y_train = y_train[1:]
+
+        optimizer = self.optimizer
+        lossfunc = self.lossfunc
+        # Zero the gradients
+        self.zero_grad()
+
+        # Forward pass
+        y_pred = self(x_train)
+
+        y_train_len = len(y_train)
+        y_pred_len = y_pred.shape[0]
+
+        if y_train_len > y_pred_len:
+            diff = y_train_len - y_pred_len
+            y_train = y_train[diff:]
+        elif y_train_len < y_pred_len:
+            diff = y_pred_len - y_train_len
+            y_pred = y_pred[:-diff, :]
+
+        y_train_hot = torch.zeros(len(y_train), self.output_size)
+        y_train_hot[range(len(y_train)), y_train] = 1
+        y_train_hot = y_train_hot.to('cuda')
+
+        # Calculate the loss
+        loss = lossfunc(y_pred, y_train_hot)
+
+        # Print loss
+        if log_loss:
+            print('Loss', loss.item())
+
+        # Backward pass
+        loss.backward()
+
+        # Update the weights
+        optimizer.step()
+
+    def save(self, path):
+        info_path = '.'.join(os.path.basename(path).split('.')[:-1]) + '/.info'
+        torch.save(self.state_dict(), path)
+        data_from_model = Data(self.input_size, self.hidden_size, self.output_size, self.version)
+        with ZipFile(path, 'a') as model_zip:
+            model_zip.writestr(info_path, data_from_model.save())
+            model_zip.close()
+
+    @staticmethod
+    def load_from_checkpoint(path, map_location: MAP_LOCATION = None):
+        old = True
+        with ZipFile(path) as model_zip:
+            filesMatch = [file for file in model_zip.namelist() if file.endswith('/.info')]
+            file = filesMatch[0] if filesMatch else None
+            if file:
+                old = False
+                data_from_model = Data.load(model_zip.read(file).decode('utf-8'))
+            model_zip.close()
+        if old:
+            model = CustomTokenizer()
+        else:
+            model = CustomTokenizer(data_from_model.hidden_size, data_from_model.input_size, data_from_model.output_size, data_from_model.version)
+        model.load_state_dict(torch.load(path, map_location=map_location))
+        if map_location:
+            model = model.to(map_location)
+        return model
+
+
+
+class Data:
+    input_size: int
+    hidden_size: int
+    output_size: int
+    version: int
+
+    def __init__(self, input_size=768, hidden_size=1024, output_size=10000, version=0):
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.version = version
+
+    @staticmethod
+    def load(string):
+        data = json.loads(string)
+        return Data(data['input_size'], data['hidden_size'], data['output_size'], data['version'])
+
+    def save(self):
+        data = {
+            'input_size': self.input_size,
+            'hidden_size': self.hidden_size,
+            'output_size': self.output_size,
+            'version': self.version,
+        }
+        return json.dumps(data)
+
+
+def auto_train(data_path, save_path='model.pth', lload_model: Union[str, None] = None, save_epochs=1):
+    data_x, data_y = {}, {}
+
+    if load_model and os.path.isfile(load_model):
+        print('Loading model from', load_model)
+        model_training = CustomTokenizer.load_from_checkpoint(load_model, 'cuda')
+    else:
+        print('Creating new model.')
+        model_training = CustomTokenizer(version=1).to('cuda')
+    save_path = os.path.join(data_path, save_path)
+    base_save_path = '.'.join(save_path.split('.')[:-1])
+
+    sem_string = '_semantic.npy'
+    feat_string = '_semantic_features.npy'
+
+    ready = os.path.join(data_path, 'ready')
+    for input_file in os.listdir(ready):
+        full_path = os.path.join(ready, input_file)        
+        try:
+            prefix = input_file.split("_")[0]
+            number = int(prefix)
+        except ValueError as e:            
+            raise e
+        if input_file.endswith(sem_string):
+            data_y[number] = numpy.load(full_path)
+        elif input_file.endswith(feat_string):
+            data_x[number] = numpy.load(full_path)
+    
+    model_training.prepare_training()
+    epoch = 1
+
+    while 1:
+        for i in range(save_epochs):
+            j = 0
+            for i in range(max(len(data_x), len(data_y))):
+                x = data_x.get(i)
+                y = data_y.get(i)
+                if x is None or y is None:
+                    print(f'The training data does not match. key={i}')
+                    continue
+                model_training.train_step(torch.tensor(x).to('cuda'), torch.tensor(y).to('cuda'), j % 50 == 0)  # Print loss every 50 steps
+                j += 1
+        save_p = save_path
+        save_p_2 = f'{base_save_path}_epoch_{epoch}.pth'
+        model_training.save(save_p)
+        model_training.save(save_p_2)
+        print(f'Epoch {epoch} completed')
+        epoch += 1

VoiceParser/hubert_manager.py

@@ -0,0 +1,33 @@
+import os.path
+import shutil
+import urllib.request
+
+import huggingface_hub
+
+
+class HuBERTManager:
+    @staticmethod
+    def make_sure_hubert_installed(download_url: str = 'https://dl.fbaipublicfiles.com/hubert/hubert_base_ls960.pt', file_name: str = 'hubert.pt'):
+        install_dir = os.path.join('VoiceParser', 'hubert')
+        if not os.path.isdir(install_dir):
+            os.makedirs(install_dir, exist_ok=True)
+        install_file = os.path.join(install_dir, file_name)
+        if not os.path.isfile(install_file):
+            print('Downloading HuBERT base model')
+            urllib.request.urlretrieve(download_url, install_file)
+            print('Downloaded HuBERT')
+        return install_file
+
+
+    @staticmethod
+    def make_sure_tokenizer_installed(model: str = 'quantifier_hubert_base_ls960_14.pth', repo: str = 'GitMylo/bark-voice-cloning', local_file: str = 'tokenizer.pth'):
+        install_dir = os.path.join('VoiceParser', 'hubert')
+        if not os.path.isdir(install_dir):
+            os.makedirs(install_dir, exist_ok=True)
+        install_file = os.path.join(install_dir, local_file)
+        if not os.path.isfile(install_file):
+            print('Downloading HuBERT custom tokenizer')
+            huggingface_hub.hf_hub_download(repo, model, local_dir=install_dir, local_dir_use_symlinks=False)
+            shutil.move(os.path.join(install_dir, model), install_file)
+            print('Downloaded tokenizer')
+        return install_file

VoiceParser/model.py

@@ -0,0 +1,102 @@
+import os
+import json
+import numpy as np
+
+import torch
+import torchaudio
+torchaudio.set_audio_backend("soundfile")  # Use 'soundfile' backend
+
+from encodec import EncodecModel
+from encodec.utils import convert_audio
+from .hubert_manager import HuBERTManager
+from .pre_kmeans_hubert import CustomHubert
+from .customtokenizer import CustomTokenizer
+
+class VoiceParser():
+    def __init__(self, device='cpu'):
+        model = ('quantifier_hubert_base_ls960_14.pth', 'tokenizer.pth')
+
+        hubert_model = CustomHubert(HuBERTManager.make_sure_hubert_installed(), device=device)
+        quant_model = CustomTokenizer.load_from_checkpoint(HuBERTManager.make_sure_tokenizer_installed(model=model[0], local_file=model[1]), device)
+        encodec_model = EncodecModel.encodec_model_24khz()
+        encodec_model.set_target_bandwidth(6.0)
+        
+        self.hubert_model = hubert_model
+        self.quant_model = quant_model
+        self.encodec_model = encodec_model.to(device)
+        self.device = device
+        print('Loaded VoiceParser models!')
+
+
+    def extract_acoustic_embed(self, wav_path, npz_dir):
+        wav, sr = torchaudio.load(wav_path)
+
+        wav_hubert = wav.to(self.device)
+
+        if wav_hubert.shape[0] == 2:  # Stereo to mono if needed
+            wav_hubert = wav_hubert.mean(0, keepdim=True)
+        
+        semantic_vectors = self.hubert_model.forward(wav_hubert, input_sample_hz=sr)
+        semantic_tokens = self.quant_model.get_token(semantic_vectors)
+        wav = convert_audio(wav, sr, self.encodec_model.sample_rate, 1).unsqueeze(0)
+
+        wav = wav.to(self.device)
+        
+        with torch.no_grad():
+            encoded_frames = self.encodec_model.encode(wav)
+
+        codes = torch.cat([encoded[0] for encoded in encoded_frames], dim=-1).squeeze()
+        
+        codes = codes.cpu()
+        semantic_tokens = semantic_tokens.cpu()
+
+        wav_name = os.path.split(wav_path)[1]
+        npz_name = wav_name[:-4] + '.npz'
+        npz_path = os.path.join(npz_dir, npz_name)
+
+        np.savez(
+         npz_path,
+         semantic_prompt=semantic_tokens,
+         fine_prompt=codes,
+         coarse_prompt=codes[:2, :]
+        )
+
+        return npz_path
+
+
+    def read_json_file(self, json_path):
+        with open(json_path, 'r') as file:
+            data = json.load(file)
+        return data
+
+
+    def parse_voice_json(self, voice_json, output_dir):
+        """
+        Parse a voice json file, generate the corresponding output json and npz files
+        Params:
+        voice_json: path of a json file or List of json nodes
+        output_dir: output dir for new json and npz files
+        """
+        if isinstance(voice_json, list):
+            voice_json = voice_json
+        else:
+            # If voice_json is a file path (str), read the JSON file
+            voice_json = self.read_json_file(voice_json)
+        for item in voice_json:
+            wav_path = item['wav']
+            npz_path = self.extract_acoustic_embed(wav_path=wav_path, npz_dir=output_dir)
+            item['npz'] = npz_path
+            del item['wav']
+
+            output_json = os.path.join(output_dir, 'metadata.json')
+        
+        with open(output_json, 'w') as file:
+            json.dump(voice_json, file, indent=4)
+
+
+
+
+
+
+
+

VoiceParser/pre_kmeans_hubert.py

@@ -0,0 +1,106 @@
+"""
+Modified HuBERT model without kmeans.
+Original author: https://github.com/lucidrains/
+Modified by: https://www.github.com/gitmylo/
+License: MIT
+"""
+
+# Modified code from https://github.com/lucidrains/audiolm-pytorch/blob/main/audiolm_pytorch/hubert_kmeans.py
+
+from pathlib import Path
+
+import torch
+from torch import nn
+from einops import pack, unpack
+
+import fairseq
+
+from torchaudio.functional import resample
+
+from audiolm_pytorch.utils import curtail_to_multiple
+
+import logging
+logging.root.setLevel(logging.ERROR)
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+class CustomHubert(nn.Module):
+    """
+    checkpoint and kmeans can be downloaded at https://github.com/facebookresearch/fairseq/tree/main/examples/hubert
+    or you can train your own
+    """
+
+    def __init__(
+        self,
+        checkpoint_path,
+        target_sample_hz=16000,
+        seq_len_multiple_of=None,
+        output_layer=9,
+        device=None
+    ):
+        super().__init__()
+        self.target_sample_hz = target_sample_hz
+        self.seq_len_multiple_of = seq_len_multiple_of
+        self.output_layer = output_layer
+
+        if device is not None:
+            self.to(device)
+
+        model_path = Path(checkpoint_path)
+
+        assert model_path.exists(), f'path {checkpoint_path} does not exist'
+
+        checkpoint = torch.load(checkpoint_path, map_location=device)
+        load_model_input = {checkpoint_path: checkpoint}
+        model, *_ = fairseq.checkpoint_utils.load_model_ensemble_and_task(load_model_input)
+
+        if device is not None:
+            model[0].to(device)
+
+        self.model = model[0]
+        self.model.eval()
+
+    @property
+    def groups(self):
+        return 1
+
+    @torch.no_grad()
+    def forward(
+        self,
+        wav_input,
+        flatten=True,
+        input_sample_hz=None
+    ):
+        device = wav_input.device
+
+        if exists(input_sample_hz):
+            wav_input = resample(wav_input, input_sample_hz, self.target_sample_hz)
+
+        if exists(self.seq_len_multiple_of):
+            wav_input = curtail_to_multiple(wav_input, self.seq_len_multiple_of)
+
+        embed = self.model(
+            wav_input,
+            features_only=True,
+            mask=False,  # thanks to @maitycyrus for noticing that mask is defaulted to True in the fairseq code
+            output_layer=self.output_layer
+        )
+
+        embed, packed_shape = pack([embed['x']], '* d')
+
+        # codebook_indices = self.kmeans.predict(embed.cpu().detach().numpy())
+
+        codebook_indices = torch.from_numpy(embed.cpu().detach().numpy()).to(device)  # .long()
+
+        if flatten:
+            return codebook_indices
+
+        codebook_indices, = unpack(codebook_indices, packed_shape, '*')
+        return codebook_indices

add_voice_preset.py

@@ -0,0 +1,21 @@
+import argparse
+import voice_presets
+
+def main():
+    # Argument Parsing
+    parser = argparse.ArgumentParser(description="Add Voice Preset")
+    parser.add_argument("--id", required=True, help="ID of the voice")
+    parser.add_argument("--desc", required=True, help="Description of the voice")
+    parser.add_argument("--wav-path", required=True, help="Path to the .wav file")
+    parser.add_argument("--session-id", required=True, help="session_id, if set to '' then it's system voice presets")
+    args = parser.parse_args()
+
+    if args.session_id:
+        print(voice_presets.add_session_voice_preset(args.id, args.desc, args.wav_path, args.session_id))
+    else:
+        print(voice_presets.add_system_voice_preset(args.id, args.desc, args.wav_path))
+    
+    
+
+if __name__ == "__main__":
+    main()

code_generator.py

@@ -0,0 +1,188 @@
+import os
+import json5
+import utils
+
+
+def check_json_script(data):
+    foreground_mandatory_attrs_map = {
+        'music': ['vol', 'len', 'desc'],
+        'sound_effect': ['vol', 'len', 'desc'],
+        'speech': ['vol', 'text']
+    }
+    background_mandatory_attrs_map = {
+        'music': ['vol', 'desc'],
+        'sound_effect': ['vol', 'desc'],
+    }
+
+    def check_by_audio_type(audio, mandatory_attrs_map, audio_str):
+        if audio['audio_type'] not in mandatory_attrs_map:
+            raise ValueError('audio_type is not allowed in this layout, audio={audio_str}')
+        for attr_name in mandatory_attrs_map[audio['audio_type']]:
+            if attr_name not in audio:
+                raise ValueError(f'{attr_name} does not exist, audio={audio_str}')
+
+    # Check json's format
+    for audio in data:
+        audio_str = json5.dumps(audio, indent=None)
+        if 'layout' not in audio:
+            raise ValueError(f'layout missing, audio={audio_str}')
+        elif 'audio_type' not in audio:
+            raise ValueError(f'audio_type missing, audio={audio_str}')
+        elif audio['layout'] == 'foreground':
+            check_by_audio_type(audio, foreground_mandatory_attrs_map, audio_str)
+        elif audio['layout'] == 'background':
+            if 'id' not in audio:
+                raise ValueError(f'id not in background audio, audio={audio_str}')
+            if 'action' not in audio:
+                raise ValueError(f'action not in background audio, audio={audio_str}')
+            if audio['action'] == 'begin':
+                check_by_audio_type(audio, background_mandatory_attrs_map, audio_str)
+            else:
+                if audio['action'] != 'end':
+                    raise ValueError(f'Unknown action, audio={audio_str}')
+        else:
+            raise ValueError(f'Unknown layout, audio={audio_str}')
+        #except Exception as err:
+        #    sys.stderr.write(f'PARSING ERROR: {err}, audio={json5.dumps(audio, indent=None)}\n')
+        #    all_clear = False
+
+
+def collect_and_check_audio_data(data):
+    fg_audio_id = 0
+    fg_audios = []
+    bg_audios = []
+    # Collect all the foreground and background audio ids used to calculate background audio length later
+    for audio in data:
+        if audio['layout'] == 'foreground':
+            audio['id'] = fg_audio_id
+            fg_audios.append(audio)
+            fg_audio_id += 1
+        else:   # background
+            if audio['action'] == 'begin':
+                audio['begin_fg_audio_id'] = fg_audio_id
+                bg_audios.append(audio)
+            else:   # ends
+                # find the backgound with the id, and update its 'end_fg_audio_id'
+                for bg_audio in bg_audios:
+                    if bg_audio['id'] == audio['id'] and bg_audio['audio_type'] == audio['audio_type']:
+                        bg_audio['end_fg_audio_id'] = fg_audio_id
+                        break
+    
+    # check if all background audios are valid
+    for bg_audio in bg_audios:
+        if 'begin_fg_audio_id' not in bg_audio:
+            raise ValueError(f'begin of background missing, audio={bg_audio}')
+        elif 'end_fg_audio_id' not in bg_audio:
+            raise ValueError(f'end of background missing, audio={bg_audio}')
+
+        if bg_audio['begin_fg_audio_id'] > bg_audio['end_fg_audio_id']:
+            raise ValueError(f'background audio ends before start, audio={bg_audio}')
+        elif bg_audio['begin_fg_audio_id'] == bg_audio['end_fg_audio_id']:
+            raise ValueError(f'background audio contains no foreground audio, audio={bg_audio}')
+        #except Exception as err:
+        #    sys.stderr.write(f'ALIGNMENT ERROR: {err}, audio={bg_audio}\n')
+        #    return None, None
+
+    return fg_audios, bg_audios
+
+
+class AudioCodeGenerator:
+    def __init__(self):
+        self.wav_counters = {
+            'bg_sound_effect': 0,
+            'bg_music': 0,
+            'idle': 0,
+            'fg_sound_effect': 0,
+            'fg_music': 0,
+            'fg_speech': 0,
+        }
+        self.code = ''
+    
+    def append_code(self, content):
+        self.code = f'{self.code}{content}\n'
+
+    def generate_code(self, fg_audios, bg_audios, output_path, result_filename):
+        def get_wav_name(audio):
+            audio_type = audio['audio_type']
+            layout = 'fg' if audio['layout'] == 'foreground' else 'bg'
+            wav_type = f'{layout}_{audio_type}' if layout else audio_type
+            desc = audio['text'] if 'text' in audio else audio['desc']
+            desc = utils.text_to_abbrev_prompt(desc)
+            wav_filename = f'{wav_type}_{self.wav_counters[wav_type]}_{desc}.wav'
+            self.wav_counters[wav_type] += 1
+            return wav_filename
+
+        header = f'''
+import os
+import sys
+import datetime
+
+from APIs import TTM, TTS, TTA, MIX, CAT, COMPUTE_LEN
+
+
+fg_audio_lens = []
+wav_path = \"{output_path.absolute()}/audio\"
+os.makedirs(wav_path, exist_ok=True)
+
+'''
+        self.append_code(header)
+
+        fg_audio_wavs = []
+        for fg_audio in fg_audios:
+            wav_name = get_wav_name(fg_audio)
+            if fg_audio['audio_type'] == 'sound_effect':
+                self.append_code(f'TTA(text=\"{fg_audio["desc"]}\", length={fg_audio["len"]}, volume={fg_audio["vol"]}, out_wav=os.path.join(wav_path, \"{wav_name}\"))')
+            elif fg_audio['audio_type'] == 'music':
+                self.append_code(f'TTM(text=\"{fg_audio["desc"]}\", length={fg_audio["len"]}, volume={fg_audio["vol"]}, out_wav=os.path.join(wav_path, \"{wav_name}\"))')
+            elif fg_audio['audio_type'] == 'speech':
+                npz_path = self.char_to_voice_map[fg_audio["character"]]["npz_path"]
+                npz_full_path = os.path.abspath(npz_path) if os.path.exists(npz_path) else npz_path
+                self.append_code(f'TTS(text=\"{fg_audio["text"]}\", speaker_id=\"{self.char_to_voice_map[fg_audio["character"]]["id"]}\", volume={fg_audio["vol"]}, out_wav=os.path.join(wav_path, \"{wav_name}\"), speaker_npz=\"{npz_full_path}\")')
+            fg_audio_wavs.append(wav_name)
+            self.append_code(f'fg_audio_lens.append(COMPUTE_LEN(os.path.join(wav_path, \"{wav_name}\")))\n')
+        
+        # cat all foreground audio together
+        self.append_code(f'fg_audio_wavs = []')
+        for wav_filename in fg_audio_wavs:
+            self.append_code(f'fg_audio_wavs.append(os.path.join(wav_path, \"{wav_filename}\"))')
+        self.append_code(f'CAT(wavs=fg_audio_wavs, out_wav=os.path.join(wav_path, \"foreground.wav\"))')
+
+        bg_audio_wavs = []
+        self.append_code(f'\nbg_audio_offsets = []')
+        for bg_audio in bg_audios:
+            wav_name = get_wav_name(bg_audio)
+            self.append_code(f'bg_audio_len = sum(fg_audio_lens[{bg_audio["begin_fg_audio_id"]}:{bg_audio["end_fg_audio_id"]}])')
+            self.append_code(f'bg_audio_offset = sum(fg_audio_lens[:{bg_audio["begin_fg_audio_id"]}])')
+            if bg_audio['audio_type'] == 'sound_effect':
+                self.append_code(f'TTA(text=\"{bg_audio["desc"]}\", volume={bg_audio["vol"]}, length=bg_audio_len, out_wav=os.path.join(wav_path, \"{wav_name}\"))')
+            elif bg_audio['audio_type'] == 'music':
+                self.append_code(f'TTM(text=\"{bg_audio["desc"]}\", volume={bg_audio["vol"]}, length=bg_audio_len, out_wav=os.path.join(wav_path, \"{wav_name}\"))')
+            else:
+                raise ValueError()
+            bg_audio_wavs.append(wav_name)
+            self.append_code(f'bg_audio_offsets.append(bg_audio_offset)\n')
+        self.append_code(f'bg_audio_wavs = []')
+        for wav_filename in bg_audio_wavs:
+            self.append_code(f'bg_audio_wavs.append(os.path.join(wav_path, \"{wav_filename}\"))')
+
+        self.append_code(f'bg_audio_wav_offset_pairs = list(zip(bg_audio_wavs, bg_audio_offsets))')
+        self.append_code(f'bg_audio_wav_offset_pairs.append((os.path.join(wav_path, \"foreground.wav\"), 0))')
+        self.append_code(f'MIX(wavs=bg_audio_wav_offset_pairs, out_wav=os.path.join(wav_path, \"{result_filename}.wav\"))')
+
+
+    def init_char_to_voice_map(self, filename):
+        with open(filename, 'r') as file:
+            self.char_to_voice_map = json5.load(file)
+
+
+    def parse_and_generate(self, script_filename, char_to_voice_map_filename, output_path, result_filename='result'):
+        self.code = ''
+        self.init_char_to_voice_map(char_to_voice_map_filename)
+
+        with open(script_filename, 'r') as file:
+            data = json5.load(file)
+
+        check_json_script(data)
+        fg_audios, bg_audios = collect_and_check_audio_data(data)
+        self.generate_code(fg_audios, bg_audios, output_path, result_filename)
+        return self.code

config.yaml

@@ -0,0 +1,17 @@
+AudioCraft:
+  # MusicGen 
+  ttm_model_size: small # [small, medium, large]
+  # AudioGen 
+  tta_model_size: medium # [medium]
+
+Text-to-Speech:
+  # Bark
+  speed: 1.05
+
+Speech-Restoration:
+  # VoiceFixer
+  Enable: True
+
+Voice-Parser:
+  # HuBERT
+  device: 'cpu'

convert_json_to_audio_gen_code.py

@@ -0,0 +1,30 @@
+import argparse
+import os
+import json5
+from pathlib import Path
+from code_generator import AudioCodeGenerator
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--script", help="Path to the json script file")
+    parser.add_argument("--character-to-voice-map", help="Path to the character-to-voice mapping CSV file")
+    parser.add_argument(
+        "--path",
+        type=str,
+        default=".",
+        help="Path of all the output wav files to be created by the generated code, default: current path"
+    )
+    args = parser.parse_args()
+
+    if not os.path.isfile(args.script):
+        print(f"File {args.script} does not exist.")
+        return
+
+    output_path = Path(args.path)
+    audio_code_generator = AudioCodeGenerator()
+    code = audio_code_generator.parse_and_generate(args.script, args.character_to_voice_map, output_path)
+    print(code)
+
+if __name__ == "__main__":
+    main()

data/voice_presets/metadata.json

@@ -0,0 +1,272 @@
+{
+    "Male1_En": {
+        "id": "Male1_En",
+        "desc": "A normal male adult voice, British accent; Language: English.",
+        "npz_path": "v2/en_speaker_1"
+    },
+    "Male2_En": {
+        "id": "Male2_En",
+        "desc": "A normal male adult voice, American accent; Language: English.",
+        "npz_path": "v2/en_speaker_6"
+    },
+    "Female1_En": {
+        "id": "Female1_En",
+        "desc": "A normal female adult voice, British accent; Language: English.",
+        "npz_path": "v2/en_speaker_9"
+    },
+    "Female2_En": {
+        "id": "Female2_En",
+        "desc": "A normal female adult voice, American accent; Language: English.",
+        "npz_path": "v2/de_speaker_3"
+    },
+    "News_Male_En": {
+        "id": "News_Male_En",
+        "desc": "A male voice of a news anchor, suitable for news scenarios; Language: English.",
+        "npz_path": "data/voice_presets/npz/news_male_speaker.npz"
+    },
+    "News_Female_En": {
+        "id": "News_Female_En",
+        "desc": "A female voice of a news anchor, suitable for news scenarios; Language: English.",
+        "npz_path": "data/voice_presets/npz/news_male_speaker.npz"
+    },
+    "News_Female_Out_En": {
+        "id": "News_Female_Out_En",
+        "desc": "A female voice of a off-site news reporter, suitable for news scenario; Language: English.",
+        "npz_path": "data/voice_presets/npz/news_female_speaker_outside.npz"
+    },
+    "Child_En": {
+        "id": "Child_En",
+        "desc": "A small young boy voice; Language: English.",
+        "npz_path": "data/voice_presets/npz/child_boy.npz"
+    },
+    "Old_Man_En": {
+        "id": "Old_Man_En",
+        "desc": "A voice of an old man; Language: English.",
+        "npz_path": "data/voice_presets/npz/elder_morgen.npz"
+    },
+    "Male1_Zh": {
+        "id": "Male1_Zh",
+        "desc": "A normal male adult voice; Language: Chinese.",
+        "npz_path": "v2/zh_speaker_0"
+    },
+    "Male2_Zh": {
+        "id": "Male2_Zh",
+        "desc": "A normal male adult voice; Language: Chinese.",
+        "npz_path": "v2/zh_speaker_1"
+    },
+    "Female1_Zh": {
+        "id": "Female1_Zh",
+        "desc": "A normal female adult voice; Language: Chinese.",
+        "npz_path": "v2/zh_speaker_9"
+    },
+    "Female2_Zh": {
+        "id": "Female2_Zh",
+        "desc": "A normal female adult voice; Language: Chinese.",
+        "npz_path": "v2/zh_speaker_4"
+    },
+    "Male1_Fr": {
+        "id": "Male1_Fr",
+        "desc": "A normal male adult voice; Language: French.",
+        "npz_path": "v2/fr_speaker_0"
+    },
+    "Male2_Fr": {
+        "id": "Male2_Fr",
+        "desc": "A normal male adult voice; Language: French.",
+        "npz_path": "v2/fr_speaker_8"
+    },
+    "Female1_Fr": {
+        "id": "Female1_Fr",
+        "desc": "A normal female adult voice; Language: French.",
+        "npz_path": "v2/fr_speaker_5"
+    },
+    "Female2_Fr": {
+        "id": "Female2_Fr",
+        "desc": "A normal female adult voice; Language: French.",
+        "npz_path": "v2/fr_speaker_1"
+    },
+    "Male1_De": {
+        "id": "Male1_De",
+        "desc": "A normal male adult voice; Language: German.",
+        "npz_path": "v2/de_speaker_0"
+    },
+    "Male2_De": {
+        "id": "Male2_De",
+        "desc": "A normal male adult voice; Language: German.",
+        "npz_path": "v2/de_speaker_1"
+    },
+    "Female1_De": {
+        "id": "Female1_De",
+        "desc": "A normal female adult voice; Language: German.",
+        "npz_path": "v2/de_speaker_3"
+    },
+    "Female2_De": {
+        "id": "Female2_De",
+        "desc": "A normal female adult voice; Language: German.",
+        "npz_path": "v2/de_speaker_8"
+    },
+    "Male1_Hi": {
+        "id": "Male1_Hi",
+        "desc": "A normal male adult voice; Language: Hindi.",
+        "npz_path": "v2/hi_speaker_5"
+    },
+    "Male2_Hi": {
+        "id": "Male2_Hi",
+        "desc": "A normal male adult voice; Language: Hindi.",
+        "npz_path": "v2/hi_speaker_8"
+    },
+    "Female1_Hi": {
+        "id": "Female1_Hi",
+        "desc": "A normal female adult voice; Language: Hindi.",
+        "npz_path": "v2/hi_speaker_0"
+    },
+    "Female2_Hi": {
+        "id": "Female2_Hi",
+        "desc": "A normal female adult voice; Language: Hindi.",
+        "npz_path": "v2/hi_speaker_3"
+    },
+    "Male1_It": {
+        "id": "Male1_It",
+        "desc": "A normal male adult voice; Language: Italian.",
+        "npz_path": "v2/it_speaker_4"
+    },
+    "Male2_It": {
+        "id": "Male2_It",
+        "desc": "A normal male adult voice; Language: Italian.",
+        "npz_path": "v2/it_speaker_5"
+    },
+    "Female1_It": {
+        "id": "Female1_It",
+        "desc": "A normal female adult voice; Language: Italian.",
+        "npz_path": "v2/it_speaker_7"
+    },
+    "Female2_It": {
+        "id": "Female2_It",
+        "desc": "A normal female adult voice; Language: Italian.",
+        "npz_path": "v2/it_speaker_9"
+    },
+    "Male1_Ja": {
+        "id": "Male1_Ja",
+        "desc": "A normal male adult voice; Language: Japanese.",
+        "npz_path": "v2/ja_speaker_2"
+    },
+    "Male2_Ja": {
+        "id": "Male2_Ja",
+        "desc": "A normal male adult voice; Language: Japanese.",
+        "npz_path": "v2/ja_speaker_6"
+    },
+    "Female1_Ja": {
+        "id": "Female1_Ja",
+        "desc": "A normal female adult voice; Language: Japanese.",
+        "npz_path": "v2/ja_speaker_4"
+    },
+    "Female2_Ja": {
+        "id": "Female2_Ja",
+        "desc": "A normal female adult voice; Language: Japanese.",
+        "npz_path": "v2/ja_speaker_5"
+    },
+    "Male1_Ko": {
+        "id": "Male1_Ko",
+        "desc": "A normal male adult voice; Language: Korean.",
+        "npz_path": "v2/ko_speaker_1"
+    },
+    "Male2_Ko": {
+        "id": "Male2_Ko",
+        "desc": "A normal male adult voice; Language: Korean.",
+        "npz_path": "v2/ko_speaker_2"
+    },
+    "Female1_Ko": {
+        "id": "Female1_Ko",
+        "desc": "A normal female adult voice; Language: Korean.",
+        "npz_path": "v2/ko_speaker_0"
+    },
+    "Female1_Ru": {
+        "id": "Female1_Ru",
+        "desc": "A normal female adult voice; Language: Russian.",
+        "npz_path": "v2/ru_speaker_5"
+    },
+    "Female2_Ru": {
+        "id": "Female2_Ru",
+        "desc": "A normal female adult voice; Language: Russian.",
+        "npz_path": "v2/ru_speaker_6"
+    },
+    "Male1_Ru": {
+        "id": "Male1_Ru",
+        "desc": "A normal male adult voice; Language: Russian.",
+        "npz_path": "v2/ru_speaker_3"
+    },
+    "Male2_Ru": {
+        "id": "Male2_Ru",
+        "desc": "A normal male adult voice; Language: Russian.",
+        "npz_path": "v2/ru_speaker_4"
+    },
+    "Female1_Es": {
+        "id": "Female1_Es",
+        "desc": "A normal female adult voice; Language: Spanish.",
+        "npz_path": "v2/es_speaker_8"
+    },
+    "Female2_Es": {
+        "id": "Female2_Es",
+        "desc": "A normal female adult voice; Language: Spanish.",
+        "npz_path": "v2/es_speaker_9"
+    },
+    "Male1_Es": {
+        "id": "Male1_Es",
+        "desc": "A normal male adult voice; Language: Spanish.",
+        "npz_path": "v2/es_speaker_6"
+    },
+    "Male2_Es": {
+        "id": "Male2_Es",
+        "desc": "A normal male adult voice; Language: Spanish.",
+        "npz_path": "v2/es_speaker_7"
+    },
+    "Female1_Tr": {
+        "id": "Female1_Tr",
+        "desc": "A normal female adult voice; Language: Turkish.",
+        "npz_path": "v2/tr_speaker_4"
+    },
+    "Female2_Tr": {
+        "id": "Female2_Tr",
+        "desc": "A normal female adult voice; Language: Turkish.",
+        "npz_path": "v2/tr_speaker_5"
+    },
+    "Male1_Tr": {
+        "id": "Male1_Tr",
+        "desc": "A normal male adult voice; Language: Turkish.",
+        "npz_path": "v2/tr_speaker_2"
+    },
+    "Male2_Tr": {
+        "id": "Male2_Tr",
+        "desc": "A normal male adult voice; Language: Turkish.",
+        "npz_path": "v2/tr_speaker_3"
+    },
+    "Male1_Pt": {
+        "id": "Male1_Pt",
+        "desc": "A normal male adult voice; Language: Purtuguese.",
+        "npz_path": "v2/pt_speaker_0"
+    },
+    "Male2_Pt": {
+        "id": "Male2_Pt",
+        "desc": "A normal male adult voice; Language: Purtuguese.",
+        "npz_path": "v2/pt_speaker_1"
+    },
+    "Female1_Pl": {
+        "id": "Female1_Pl",
+        "desc": "A normal female adult voice; Language: Polish.",
+        "npz_path": "v2/pl_speaker_4"
+    },
+    "Female2_Pl": {
+        "id": "Female2_Pl",
+        "desc": "A normal female adult voice; Language: Polish.",
+        "npz_path": "v2/pl_speaker_6"
+    },
+    "Male1_Pl": {
+        "id": "Male1_Pl",
+        "desc": "A normal male adult voice; Language: Polish.",
+        "npz_path": "v2/pl_speaker_5"
+    },
+    "Male2_Pl": {
+        "id": "Male2_Pl",
+        "desc": "A normal male adult voice; Language: Polish.",
+        "npz_path": "v2/pl_speaker_7"
+    }
+}

examples/examples.py

@@ -0,0 +1,87 @@
+
+example1 = {
+    'text': "An introduction to AI-assisted audio content creation.",
+    'table_script': """
+| Audio Type   | Layout     | ID | Character | Action | Volume | Description                                                      | Length |
+|--------------|------------|----|-----------|--------|--------|------------------------------------------------------------------|--------|
+| music        | background | 1  | N/A       | begin  | -35    | Inspirational technology-themed music                            | Auto   |
+| speech       | foreground | N/A| Narrator  | N/A    | -15    | Welcome to the future of audio content creation.                 | Auto   |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Digital startup sound                                            | 2      |
+| speech       | foreground | N/A| Narrator  | N/A    | -15    | With evolving technology, we are introducing AI-assisted tools for pristine audio production. | Auto |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Keyboard typing noise                                            | 3      |
+| speech       | foreground | N/A| Narrator  | N/A    | -15    | Imagine crafting audio content with the power of AI at your fingertips. | Auto |
+| sound_effect | background | 2  | N/A       | begin  | -35    | Ambiance of a busy control room                                   | Auto   |
+| speech       | foreground | N/A| Narrator  | N/A    | -15    | Enhanced quality, efficient production and limitless creativity, all under one roof. | Auto |
+| sound_effect | background | 2  | N/A       | end    | N/A    | N/A                                                              | Auto   |
+| speech       | foreground | N/A| Narrator  | N/A    | -15    | Unleash your potential with AI-assisted audio content creation.  | Auto   |
+| music        | background | 1  | N/A       | end    | N/A    | N/A                                                              | Auto   |
+
+""",
+    'table_voice': """
+| Character   | Voice     |
+|-------------|-----------|
+| Narrator    | News_Male_En |
+
+""",
+    'wav_file': 'examples/1.mp4',
+}
+
+example2 = {
+    'text': "A couple dating in a cafe.",
+    'table_script': """
+| Audio Type   | Layout     | ID | Character | Action | Volume | Description                                   | Length |
+|--------------|------------|----|-----------|--------|--------|-----------------------------------------------|--------|
+| sound_effect | background | 1  | N/A       | begin  | -35    | Soft chattering in a cafe                     | Auto   |
+| sound_effect | background | 2  | N/A       | begin  | -38    | Coffee brewing noises                         | Auto   |
+| music        | background | 3  | N/A       | begin  | -35    | Soft jazz playing in the background           | Auto   |
+| speech       | foreground | N/A| Man       | N/A    | -15    | It’s really nice to finally get out and relax a little, isn’t it? | Auto |
+| speech       | foreground | N/A| Woman     | N/A    | -15    | I know, right? We should do this more often.  | Auto   |
+| sound_effect | background | 2  | N/A       | end    | N/A    | N/A                                           | Auto   |
+| speech       | foreground | N/A| Man       | N/A    | -15    | Here’s your coffee, just as you like it.      | Auto   |
+| speech       | foreground | N/A| Woman     | N/A    | -15    | Thank you, it smells wonderful.               | Auto   |
+| music        | background | 3  | N/A       | end    | N/A    | N/A                                           | Auto   |
+| sound_effect | background | 1  | N/A       | end    | N/A    | N/A                                           | Auto   |
+
+""",
+    'table_voice': """
+| Character   | Voice     |
+|-------------|-----------|
+| Man         | Male1_En    |
+| Woman       | Female1_En   |
+
+""",
+    'wav_file': 'examples/2.mp4',
+}
+
+
+example3 = {
+    'text': "A child is participating in a farting contest.",
+    'table_script': """
+| Audio Type   | Layout     | ID | Character | Action | Volume | Description                                          | Length |
+|--------------|------------|----|-----------|--------|--------|------------------------------------------------------|--------|
+| sound_effect | background | 1  | N/A       | begin  | -35    | Outdoor park ambiance, people chattering             | Auto   |
+| music        | background | 2  | N/A       | begin  | -35    | Light comedy theme music, quirky                     | Auto   |
+| speech       | foreground | N/A| Host      | N/A    | -15    | Welcome to the annual Fart Competition.              | Auto   |
+| speech       | foreground | N/A| Host      | N/A    | -15    | Now, let’s welcome our youngest participant.         | Auto   |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Clapping sound                                       | 2      |
+| speech       | foreground | N/A| Child     | N/A    | -15    | Hi, I’m excited to be here.                          | Auto   |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Short, cartoonish duration of a fart sound           | 4      |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Audience laughing and applauding                     | 2      |
+| speech       | foreground | N/A| Host      | N/A    | -15    | Wow, that was impressive! Let’s give another round of applause! | Auto |
+| sound_effect | foreground | N/A| N/A       | N/A    | -35    | Audience clapping and cheering                       | 3      |
+| music        | background | 2  | N/A       | end    | N/A    | N/A                                                  | Auto   |
+| sound_effect | background | 1  | N/A       | end    | N/A    | N/A                                                  | Auto   |
+""",
+    'table_voice': """
+| Character   | Voice     |
+|-------------|-----------|
+| Host        | Male1_En     |
+| Child       | Child_En     |
+
+""",
+    'wav_file': 'examples/3.mp4',
+}
+
+
+
+examples = [example1, example2, example3]

parse_voice.py

@@ -0,0 +1,31 @@
+import os
+import argparse
+from VoiceParser.model import VoiceParser
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--wav-path', type=str, help="Path of a wav file")
+    parser.add_argument('--wav-dir', type=str, help="Directory of wav files")
+    parser.add_argument('--out-dir', type=str, help="Directory of output npz files")
+    args = parser.parse_args()
+
+    if (args.wav_path is None and args.wav_dir is None) or (args.wav_path is not None and args.wav_dir is not None):
+        parser.error("Please provide either '--wav-path' or '--wav-dir', but not both.")
+
+    out_dir = args.out_dir
+
+    model = VoiceParser(device='cpu')
+
+    if args.wav_path is not None:
+        model.extract_acoustic_embed(args.wav_path, out_dir)
+        print(f'Sucessfully parsed {args.wav_path}')
+    else:
+        wav_name_list = os.listdir(args.wav_dir)
+        for wav_name in wav_name_list:
+            wav_path = os.path.join(args.wav_dir, wav_name)
+            model.extract_acoustic_embed(wav_path, out_dir)
+            print(f'Sucessfully parsed {wav_path}')
+
+
+if __name__ == '__main__':
+    main()

pipeline.py

@@ -0,0 +1,229 @@
+import datetime
+import os
+from string import Template
+import openai
+import re
+import glob
+import pickle
+import time
+import json5
+from retrying import retry
+from code_generator import check_json_script, collect_and_check_audio_data
+import random
+import string
+
+import utils
+import voice_presets
+from code_generator import AudioCodeGenerator
+
+# Enable this for debugging
+USE_OPENAI_CACHE = False
+openai_cache = []
+if USE_OPENAI_CACHE:
+    os.makedirs('cache', exist_ok=True)
+    for cache_file in glob.glob('cache/*.pkl'):
+        with open(cache_file, 'rb') as file:
+            openai_cache.append(pickle.load(file))
+
+def chat_with_gpt(prompt, api_key):
+    if USE_OPENAI_CACHE:
+        filtered_object = list(filter(lambda x: x['prompt'] == prompt, openai_cache))
+        if len(filtered_object) > 0:
+            response = filtered_object[0]['response']
+            return response
+    
+    try:
+        openai.api_key = api_key
+        chat = openai.ChatCompletion.create(
+            # model="gpt-3.5-turbo",
+            model="gpt-4",
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant."
+                },
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ]
+        )
+    finally:
+        openai.api_key = ''
+
+    if USE_OPENAI_CACHE:
+        cache_obj = {
+            'prompt': prompt,
+            'response': chat['choices'][0]['message']['content']
+        }
+        with open(f'cache/{time.time()}.pkl', 'wb') as _openai_cache:
+            pickle.dump(cache_obj, _openai_cache)
+            openai_cache.append(cache_obj)
+
+    return chat['choices'][0]['message']['content']
+
+
+def get_file_content(filename):
+    with open(filename, 'r') as file:
+        return file.read().strip()
+
+
+def write_to_file(filename, content):
+    with open(filename, 'w') as file:
+        file.write(content)
+
+
+def extract_substring_with_quotes(input_string, quotes="'''"):
+    pattern = f"{quotes}(.*?){quotes}"
+    matches = re.findall(pattern, input_string, re.DOTALL)
+    return matches
+
+
+def try_extract_content_from_quotes(content):
+    if "'''" in content:
+        return extract_substring_with_quotes(content)[0]
+    elif "```" in content:
+        return extract_substring_with_quotes(content, quotes="```")[0]
+    else:
+        return content
+
+def maybe_get_content_from_file(content_or_filename):
+    if os.path.exists(content_or_filename):
+        with open(content_or_filename, 'r') as file:
+            return file.read().strip()
+    return content_or_filename
+
+
+
+# Pipeline Interface Guidelines:
+#
+# Init calls:
+# - Init calls must be called before running the actual steps
+#   - init_session() is called every time a gradio webpage is loaded
+#
+# Single Step:
+# - takes input (file or content) and output path as input
+# - most of time just returns output content
+# 
+# Compositional Step:
+# - takes session_id as input (you have session_id, you have all the paths)
+# - run a series of steps
+
+# This is called for every new gradio webpage
+
+def init_session(session_id=''):
+    def uid8():
+        return ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))
+
+    if session_id == '':
+        session_id = f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}_{uid8()}'
+    # create the paths
+    os.makedirs(utils.get_session_voice_preset_path(session_id))
+    os.makedirs(utils.get_session_audio_path(session_id))
+    print(f'New session created, session_id={session_id}')
+    return session_id
+
+@retry(stop_max_attempt_number=3)
+def input_text_to_json_script_with_retry(complete_prompt_path, api_key):
+    print("    trying ...")
+    complete_prompt = get_file_content(complete_prompt_path)
+    json_response = try_extract_content_from_quotes(chat_with_gpt(complete_prompt, api_key))
+    json_data = json5.loads(json_response)
+
+    try:
+        check_json_script(json_data)
+        collect_and_check_audio_data(json_data)
+    except Exception as err:
+        print(f'JSON ERROR: {err}')
+        retry_complete_prompt = f'{complete_prompt}\n```\n{json_response}```\nThe script above has format error(s). Return the fixed script.\n\nScript:\n'
+        write_to_file(complete_prompt_path, retry_complete_prompt)
+        raise err
+
+    return json_response
+
+# Step 1: input_text to json
+def input_text_to_json_script(input_text, output_path, api_key):
+    input_text = maybe_get_content_from_file(input_text)
+    text_to_audio_script_prompt = get_file_content('prompts/text_to_json.prompt')
+    prompt = f'{text_to_audio_script_prompt}\n\nInput text: {input_text}\n\nScript:\n'
+    complete_prompt_path = output_path / 'complete_input_text_to_audio_script.prompt'
+    write_to_file(complete_prompt_path, prompt)
+    audio_script_response = input_text_to_json_script_with_retry(complete_prompt_path, api_key)
+    generated_audio_script_filename = output_path / 'audio_script.json'
+    write_to_file(generated_audio_script_filename, audio_script_response)
+    return audio_script_response
+
+# Step 2: json to char-voice map
+def json_script_to_char_voice_map(json_script, voices, output_path, api_key):
+    json_script_content = maybe_get_content_from_file(json_script)
+    prompt = get_file_content('prompts/audio_script_to_character_voice_map.prompt')
+    presets_str = '\n'.join(f"{preset['id']}: {preset['desc']}" for preset in voices.values())
+    prompt = Template(prompt).substitute(voice_and_desc=presets_str)
+    prompt = f"{prompt}\n\nAudio script:\n'''\n{json_script_content}\n'''\n\noutput:\n"
+    write_to_file(output_path / 'complete_audio_script_to_char_voice_map.prompt', prompt)
+    char_voice_map_response = try_extract_content_from_quotes(chat_with_gpt(prompt, api_key))
+    char_voice_map = json5.loads(char_voice_map_response)
+    # enrich char_voice_map with voice preset metadata
+    complete_char_voice_map = {c: voices[char_voice_map[c]] for c in char_voice_map}
+    char_voice_map_filename = output_path / 'character_voice_map.json'
+    write_to_file(char_voice_map_filename, json5.dumps(complete_char_voice_map))
+    return complete_char_voice_map
+    
+# Step 3: json to py code
+def json_script_and_char_voice_map_to_audio_gen_code(json_script_filename, char_voice_map_filename, output_path, result_filename):
+    audio_code_generator = AudioCodeGenerator()
+    code = audio_code_generator.parse_and_generate(
+        json_script_filename,
+        char_voice_map_filename,
+        output_path,
+        result_filename
+    )
+    write_to_file(output_path / 'audio_generation.py', code)
+
+# Step 4: py code to final wav
+def audio_code_gen_to_result(audio_gen_code_path):
+    audio_gen_code_filename = audio_gen_code_path / 'audio_generation.py'
+    os.system(f'PYTHONPATH=. python {audio_gen_code_filename}')
+
+# Function call used by Gradio: input_text to json
+def generate_json_file(session_id, input_text, api_key):
+    output_path = utils.get_session_path(session_id)
+    # Step 1
+    print(f'session_id={session_id}, Step 1: Writing audio script based on text: {input_text} ...')
+    return input_text_to_json_script(input_text, output_path, api_key)
+
+# Function call used by Gradio: json to result wav
+def generate_audio(session_id, json_script, api_key):
+    def count_lines(content):
+        # Split the string using the newline character and count the non-empty lines
+        return sum(1 for line in content.split('\n') if line.strip())
+
+    max_lines = utils.get_max_script_lines()
+    if count_lines(json_script) > max_lines:
+        raise ValueError(f'The number of lines of the JSON script has exceeded {max_lines}!')
+
+    output_path = utils.get_session_path(session_id)
+    output_audio_path = utils.get_session_audio_path(session_id)
+    voices = voice_presets.get_merged_voice_presets(session_id)
+
+    # Step 2
+    print(f'session_id={session_id}, Step 2: Parsing character voice with LLM...')
+    char_voice_map = json_script_to_char_voice_map(json_script, voices, output_path, api_key)
+    # Step 3
+    json_script_filename = output_path / 'audio_script.json'
+    char_voice_map_filename = output_path / 'character_voice_map.json'
+    result_wav_basename = f'res_{session_id}'
+    print(f'session_id={session_id}, Step 3: Compiling audio script to Python program ...')
+    json_script_and_char_voice_map_to_audio_gen_code(json_script_filename, char_voice_map_filename, output_path, result_wav_basename)
+    # Step 4
+    print(f'session_id={session_id}, Step 4: Start running Python program ...')
+    audio_code_gen_to_result(output_path)
+
+    result_wav_filename = output_audio_path / f'{result_wav_basename}.wav'
+    print(f'Done all processes, result: {result_wav_filename}')
+    return result_wav_filename, char_voice_map
+
+# Convenient function call used by wavjourney_cli
+def full_steps(session_id, input_text, api_key):
+    json_script = generate_json_file(session_id, input_text, api_key)
+    return generate_audio(session_id, json_script, api_key)

prompts/audio_script_to_character_voice_map.prompt

@@ -0,0 +1,11 @@
+Given an audio script in json format, for each character appeared in the "character" attribute, you should map the character to a "voice type" according to the his/her lines and the voice type's features. Each character must be mapped to a different voice type, and each voice type must be from one of the following(each line in the format of "[voice_type_id]: [voice_type_description]"):
+$voice_and_desc
+
+Output should be in the format of json, like:
+'''
+{
+    "character_1": "voice_type_1",
+    "character_2": "voice_type_2",
+    ...
+}
+'''

prompts/audio_script_to_json.prompt

@@ -0,0 +1,74 @@
+Given an audio script, adapt it into a json file. You must go through each line of the script, and try your best to convert it to a json object or multiple json objects.
+
+Each json object represents an audio. There are three types of audios: sound effect, music, and speech. For each audio, there are two types of layouts: foreground and background. Foreground audios are played sequentially, and background audios are environmental sounds or music which are played while the foreground audios are being played.
+
+While going through each line of the script, you have choices as below:
+- For character lines, you need to convert it to a speech audio. Note that a speech audio can only be foreground. Example:
+From
+```
+News Anchor: Good evening, this is BBC News.
+```
+To
+```
+{"audio_type": "speech", "layout": "foreground", "character": "News Anchor", "vol": -15, "text": "Good evening, this is BBC News."},
+```
+- For sound effects, you need to convert it to a sound_effect audio. Especially, you need to figure out its length according to the script's context, and put it into "len". Example:
+From
+```
+(SFX: Airport beeping sound)
+```
+to
+```
+{"audio_type": "sound_effect", "layout": "foreground", "vol": -35, "len": 2, "desc": "Airport beeping sound"},
+```
+- For music, you need to convert it to a music audio. Especially, you need to figure out its length according to the script's context, and put it into "len". Example:
+From
+```
+(SFX: Uplifting newsroom music)
+```
+to
+```
+{"audio_type": "music", "layout": "foreground", "vol": -35, "len": 10, "desc": "Uplifting newsroom music"},
+```
+
+When a sound effect or music is environmental played in the background, you should set their layout to "background". You must give the background audio an unique id, and you must figure out the end of the background audio according to the context and indicate it explicitly. Example:
+From
+```
+...
+(SFX: Airport ambiance, people walking)
+Airport Announcer: Lades and Gentlemen, attentions please!
+...
+```
+to
+```
+...
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "begin", "vol": -35, "desc": "Airport ambiance, people walking"},
+[foreground audio]
+...
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "end"},
+...
+```
+
+When a line contains multiple sound effects and musics, you need to decompose it into multiple audios. Example:
+From
+```
+...
+(SFX: A classy restaurant, low chatter, clinking silverware, jazz music playing)
+...
+```
+to
+```
+...
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "begin", "vol": -35, "desc": "low chatter"},
+{"audio_type": "sound_effect", "layout": "background", "id":2, "action": "begin", "vol": -35, "desc": "clinking silverware"},
+{"audio_type": "music", "layout": "background", "id":3, "action": "begin", "vol": -35, "desc": "jazz music"},
+...
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "end"},
+{"audio_type": "sound_effect", "layout": "background", "id":2, "action": "end"},
+{"audio_type": "music", "layout": "background", "id":3, "action": "end"},
+...
+```
+
+The final json object contains a list of all the audio objects.
+
+Script:

prompts/script_to_json.prompt

@@ -0,0 +1,58 @@
+Convert an audio script line to another format. Each line will be converted to a simple json format. Below are the examples of conversion of each line.
+
+Example line 1:
+'''
+[Background music 1 begins, -35dB: Uplifting newsroom music]
+'''
+convert to:
+'''
+{"voice_type": "back_ground_music", "id": 1, "state": "begin", "volume": -35, "desc": "Uplifting newsroom music"},
+'''
+Example line 2:
+'''
+[Background music 1 ends]
+'''
+convert to:
+'''
+{"voice_type": "back_ground_music", "id": 1, "state": "end"},
+'''
+Example line 3:
+'''
+[Background sound effect 2 begins, -35dB: Crowds cheering and arcade ambiance]
+'''
+convert to:
+'''
+{"voice_type": "back_ground_sound_effect", "id": 2, "state": "begin", "volume": -35, "desc": "Crowds cheering and arcade ambiance"},
+'''
+Example line 4:
+'''
+[Background sound effect 2 ends]
+'''
+convert to:
+'''
+{"voice_type": "back_ground_sound_effect", "id": 2, "state": "end"},
+'''
+Example line 5:
+'''
+News Anchor, -15dB: Good evening, this is BBC News.
+'''
+convert to:
+'''
+{"voice_type": "speech", "character": "News Anchor", "volume": -15, "desc": "Good evening, this is BBC News."},
+'''
+Example line 6:
+'''
+[Sound effect, 3s, -15dB: Keyboard typing and mouse clicking]
+'''
+convert to:
+'''
+{"voice_type": "sound_effect", "length": 3, "volume": -15, "desc": "Keyboard typing and mouse clicking"},
+'''
+Example line 7:
+'''
+[Sound music, 10s, -15dB: Uplifting newsroom music]
+'''
+convert to:
+'''
+{"voice_type": "music", "length": 10, "volume": -15, "desc": "Uplifting newsroom music"},
+'''

prompts/text_to_audio_script.prompt

@@ -0,0 +1,34 @@
+I want you to act as a audio script writer. I'll give you an instruction which is a general idea and you will make it a short audio script.
+
+The script should follow the rules below:
+- For dialogs, each line must contain the character's name, its volume in decibel (human voices are usually around -15dB) and the line, example:
+'''
+Darth Vader, -16dB: Luke, I'm your father.
+'''
+- For foreground sound effect, you must wrap the line with brackets and start with "Sound effect, ", and you should give the duration of the sound effect in seconds, and you should specify the volume you want in decibel(For foreground sound effects it's usually around -15dB), and you should give very detailed description of the sound effect, example:
+'''
+[Sound effect, 2s, -15dB: Airport beeping sound]
+'''
+- For foreground music, you must wrap the line with brackets and start with "Music, ", and you should give the duration of the music in seconds, and you should specify the volume you want in decibel(for foreground music it's usually around -15dB), and you should give very detailed description of the music, example:
+'''
+[Music, 10s, -15dB: 80's Rock and Roll music]
+'''
+- For background sound effects, you must wrap the line with brackets and start with "Background sound effect" followed by its id, and you must always explicitly indicate the start and end of the sound effects, and you should specify the volume you want in decibel(for background sound effect it's usually around -35dB), and you should give very detailed description of the sound effect, example:
+'''
+[Background sound effect 1 begins, -34dB: Airport ambiance, including footsteps, luggage rolling, and distant airplane engine]
+...
+[Background sound effect 1 ends]
+'''
+- For background music, you must wrap the line with brackets and start with "Background music" followed by its id, and you must always explicitly indicate the start and end of the music, and you should specify the volume you want in decibel(for background sound effect it's usually around -35dB), and you should give very detailed description of the music, example:
+'''
+[Background music 1 begins, -35dB: Uplifting newsroom music]
+...
+[Background music 1 ends]
+'''
+- For music and sound effect, you can not name the element outside these: 
+["Sound effect, ", 
+"Music, ", 
+"Background sound effect" followed by its id, 
+"Background music" followed by its id]
+such as "Foreground sound effect", "Foreground music" is forbidden
+

prompts/text_to_json.prompt

@@ -0,0 +1,33 @@
+I want you to act as an audio script writer. I'll give you input text which is a general idea and you will make it an audio script in json format. Instructions:
+- Each line represents an audio. There are three types of audio: sound effects, music, and speech. For each audio, there are only two types of layouts: foreground and background. Foreground audios are played sequentially, and background audios are environmental sounds or music which are played while the foreground audios are being played. 
+- Sound effects can be either foreground or background. For sound effects, you must provide its layout, volume, length (in seconds), and detailed description of the real-world sound effect. Example:
+'''
+- The description of sound effects should not contain a specific person.
+{"audio_type": "sound_effect", "layout": "foreground", "vol": -35, "len": 2, "desc": "Airport beeping sound"},
+'''
+- Music can be either foreground or background. For music, you must provide its layout, volume, length (in seconds), and detailed description of the music. Example:
+'''
+{"audio_type": "music", "layout": "foreground", "vol": -35, "len": 10, "desc": "Uplifting newsroom music"},
+'''
+- Speech can only be foreground. For speech, you must provide the character, volume, and the character's line. You do not need to specify the length of the speech. Example:
+'''
+{"audio_type": "speech", "layout": "foreground", "character": "News Anchor", "vol": -15, "text": "Good evening, this is BBC News. In today's breaking news, we have an unexpected turn of events in the political arena"},
+'''
+- The description of speech should not contain anything other than the lines, such as actions, expressions, emotions etc.
+- For background sound audio, you must specify the beginning and the end of background audio in separate lines to indicate when the audio begins and when it ends. Example for background sound effect (for background music it's similar):
+'''
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "begin", "vol": -35, "desc": "Airport ambiance, people walking"},
+[foreground audio 1]
+[foreground audio 2]
+...
+{"audio_type": "sound_effect", "layout": "background", "id":1, "action": "end"},
+'''
+- Each background audio must have a unique id.
+- You do not specify the length of the background audio.
+- A background audio must be wrapped around at least one foreground audio.
+- If a background sound effect has multiple sounds, please decompose it into multiple background sound effects.
+- The description of speech can be multilingual, default is English. 
+- The description of sound effects and music must be in English.
+- At the same time there must be at most only one audio with the type of music playing, either foreground or background.
+- The volume of background sound effects/music is usually around -35 ~ -40 dB
+- The output json must be a list as the root node containing all the audio nodes, and must be wrapped with triple quotes '''.

scripts/EnvsSetup.sh

@@ -0,0 +1,7 @@
+conda env create -f Envs/WavJourney.yml && \
+conda env update -f Envs/Bark.yml && \
+conda env update -f Envs/AudioCraft.yml && \
+conda run --live-stream -n WavJourney pip install -U git+https://git@github.com/facebookresearch/audiocraft@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft && \
+conda run --live-stream -n WavJourney pip install -U --no-deps voicefixer==0.1.2 && \
+conda run --live-stream -n WavJourney pip install -U --no-deps numpy==1.21 && \
+conda run --live-stream -n WavJourney pip install -U --no-deps librosa==0.8.1

scripts/download_models.py

@@ -0,0 +1,32 @@
+import yaml
+import os
+
+# Read the YAML file
+with open('config.yaml', 'r') as file:
+    config = yaml.safe_load(file)
+
+# Extract values for each application
+ttm_model_size = config['AudioCraft']['ttm_model_size']
+tta_model_size = config['AudioCraft']['tta_model_size']
+
+# Download nltk
+import nltk
+nltk.download('punkt')
+
+# Downloading the TTS models
+print('Step 1: Downloading TTS model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from transformers import BarkModel; BarkModel.from_pretrained("suno/bark")\'')
+
+print('Step 2: Downloading TTA model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from audiocraft.models import AudioGen; tta_model = AudioGen.get_pretrained("facebook/audiogen-{tta_model_size}")\'')
+
+print('Step 3: Downloading TTM model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from audiocraft.models import MusicGen; tta_model = MusicGen.get_pretrained("facebook/musicgen-{ttm_model_size}")\'')
+
+print('Step 4: Downloading SR model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from voicefixer import VoiceFixer; vf = VoiceFixer()\'')
+
+print('Step 5: Downloading VP model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from VoiceParser.model import VoiceParser; vp = VoiceParser(device="cpu")\'')
+
+print('All models successfully downloaded!')

scripts/kill_services.py

@@ -0,0 +1,11 @@
+import os
+
+# Extract values for each application
+service_port = os.environ.get('WAVJOURNEY_SERVICE_PORT')
+
+# Execute the commands 
+os.system(f'kill $(lsof -t -i :{service_port})')
+
+
+
+

scripts/start_service_and_ui.sh

@@ -0,0 +1,2 @@
+conda run --live-stream -n WavJourney python -u services.py 2>&1 | tee services_logs/service.out &
+conda run --live-stream -n WavJourney python -u ui_client.py 2>&1 | tee services_logs/wavejourney.out

scripts/start_services.sh

@@ -0,0 +1 @@
+nohup conda run --live-stream -n WavJourney python services.py > services_logs/service.out 2>&1 &

scripts/start_ui.sh

@@ -0,0 +1 @@
+conda run --live-stream -n WavJourney python -u ui_client.py 2>&1 | stdbuf -oL tee services_logs/wavejourney.out

services.py

@@ -0,0 +1,231 @@
+import os
+import yaml
+import logging
+import nltk
+import torch
+import torchaudio
+from torchaudio.transforms import SpeedPerturbation
+from APIs import WRITE_AUDIO, LOUDNESS_NORM
+from utils import fade, get_service_port
+from flask import Flask, request, jsonify
+
+with open('config.yaml', 'r') as file:
+    config = yaml.safe_load(file)
+
+# Configure the logging format and level
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+# Create a FileHandler for the log file
+os.makedirs('services_logs', exist_ok=True)
+log_filename = 'services_logs/Wav-API.log'
+file_handler = logging.FileHandler(log_filename, mode='w')
+file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+
+# Add the FileHandler to the root logger
+logging.getLogger('').addHandler(file_handler)
+
+
+"""
+Initialize the AudioCraft models here
+"""
+from audiocraft.models import AudioGen, MusicGen
+tta_model_size = config['AudioCraft']['tta_model_size']
+tta_model = AudioGen.get_pretrained(f'facebook/audiogen-{tta_model_size}')
+logging.info(f'AudioGen ({tta_model_size}) is loaded ...')
+
+ttm_model_size = config['AudioCraft']['ttm_model_size']
+ttm_model = MusicGen.get_pretrained(f'facebook/musicgen-{ttm_model_size}')
+logging.info(f'MusicGen ({ttm_model_size}) is loaded ...')
+
+
+"""
+Initialize the BarkModel here
+"""
+from transformers import BarkModel, AutoProcessor
+SPEED = float(config['Text-to-Speech']['speed'])
+speed_perturb = SpeedPerturbation(32000, [SPEED])
+tts_model = BarkModel.from_pretrained("suno/bark")
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+tts_model = tts_model.to(device)
+tts_model = tts_model.to_bettertransformer()    # Flash attention
+SAMPLE_RATE = tts_model.generation_config.sample_rate
+SEMANTIC_TEMPERATURE = 0.9
+COARSE_TEMPERATURE = 0.5
+FINE_TEMPERATURE = 0.5
+processor = AutoProcessor.from_pretrained("suno/bark")
+logging.info('Bark model is loaded ...')
+
+
+""" 
+Initialize the VoiceFixer model here
+"""
+from voicefixer import VoiceFixer
+vf = VoiceFixer()
+logging.info('VoiceFixer is loaded ...')
+
+
+"""
+Initalize the VoiceParser model here
+"""
+from VoiceParser.model import VoiceParser
+vp_device = config['Voice-Parser']['device']
+vp = VoiceParser(device=vp_device)
+logging.info('VoiceParser is loaded ...')
+
+
+app = Flask(__name__)
+
+
+@app.route('/generate_audio', methods=['POST'])
+def generate_audio():
+    # Receive the text from the POST request
+    data = request.json
+    text = data['text']
+    length = float(data.get('length', 5.0))
+    volume = float(data.get('volume', -35))
+    output_wav = data.get('output_wav', 'out.wav')
+
+    logging.info(f'TTA (AudioGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
+    
+    try:
+        tta_model.set_generation_params(duration=length)  
+        wav = tta_model.generate([text])  
+        wav = torchaudio.functional.resample(wav, orig_freq=16000, new_freq=32000)
+
+        wav = wav.squeeze().cpu().detach().numpy()
+        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
+        WRITE_AUDIO(wav, name=output_wav)
+
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Text-to-Audio generated successfully | {text}', 'file': output_wav})
+
+    except Exception as e:
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/generate_music', methods=['POST'])
+def generate_music():
+    # Receive the text from the POST request
+    data = request.json
+    text = data['text']
+    length = float(data.get('length', 5.0))
+    volume = float(data.get('volume', -35))
+    output_wav = data.get('output_wav', 'out.wav')
+
+    logging.info(f'TTM (MusicGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
+
+
+    try:
+        ttm_model.set_generation_params(duration=length)  
+        wav = ttm_model.generate([text])  
+        wav = wav[0][0].cpu().detach().numpy()
+        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
+        WRITE_AUDIO(wav, name=output_wav)
+
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Text-to-Music generated successfully | {text}', 'file': output_wav})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/generate_speech', methods=['POST'])
+def generate_speech():
+    # Receive the text from the POST request
+    data = request.json
+    text = data['text']
+    speaker_id = data['speaker_id']
+    speaker_npz = data['speaker_npz']
+    volume = float(data.get('volume', -35))
+    output_wav = data.get('output_wav', 'out.wav')
+    
+    logging.info(f'TTS (Bark): Speaker: {speaker_id}, Volume: {volume} dB, Prompt: {text}')
+
+    try:   
+        # Generate audio using the global pipe object
+        text = text.replace('\n', ' ').strip()
+        sentences = nltk.sent_tokenize(text)
+        silence = torch.zeros(int(0.1 * SAMPLE_RATE), device=device).unsqueeze(0)  # 0.1 second of silence
+
+        pieces = []
+        for sentence in sentences:
+            inputs = processor(sentence, voice_preset=speaker_npz).to(device)
+            # NOTE: you must run the line below, otherwise you will see the runtime error
+            # RuntimeError: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead.
+            inputs['history_prompt']['coarse_prompt'] = inputs['history_prompt']['coarse_prompt'].transpose(0, 1).contiguous().transpose(0, 1)
+
+            with torch.inference_mode():
+                # TODO: min_eos_p?
+                output = tts_model.generate(
+                    **inputs,
+                    do_sample = True,
+                    semantic_temperature = SEMANTIC_TEMPERATURE,
+                    coarse_temperature = COARSE_TEMPERATURE,
+                    fine_temperature = FINE_TEMPERATURE
+                )
+
+            pieces += [output, silence]
+
+        result_audio = torch.cat(pieces, dim=1)
+        wav_tensor = result_audio.to(dtype=torch.float32).cpu()
+        wav = torchaudio.functional.resample(wav_tensor, orig_freq=SAMPLE_RATE, new_freq=32000)
+        wav = speed_perturb(wav.float())[0].squeeze(0)
+        wav = wav.numpy()
+        wav = LOUDNESS_NORM(wav, volumn=volume)
+        WRITE_AUDIO(wav, name=output_wav)
+
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Text-to-Speech generated successfully | {speaker_id}: {text}', 'file': output_wav})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/fix_audio', methods=['POST'])
+def fix_audio():
+    # Receive the text from the POST request
+    data = request.json
+    processfile = data['processfile']
+
+    logging.info(f'Fixing {processfile} ...')
+
+    try:
+        vf.restore(input=processfile, output=processfile, cuda=True, mode=0)
+        
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': 'Speech restored successfully', 'file': processfile})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/parse_voice', methods=['POST'])
+def parse_voice():
+    # Receive the text from the POST request
+    data = request.json
+    wav_path = data['wav_path']
+    out_dir = data['out_dir']
+
+    logging.info(f'Parsing {wav_path} ...')
+
+    try:
+        vp.extract_acoustic_embed(wav_path, out_dir)
+        
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Sucessfully parsed {wav_path}'})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+if __name__ == '__main__':
+    service_port = get_service_port()
+    # We disable multithreading to force services to process one request at a time and avoid CUDA OOM
+    app.run(debug=False, threaded=False, port=service_port)

share_btn.py

@@ -0,0 +1,74 @@
+community_icon_html = """<svg id="share-btn-share-icon" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32">
+    <path d="M20.6081 3C21.7684 3 22.8053 3.49196 23.5284 4.38415C23.9756 4.93678 24.4428 5.82749 24.4808 7.16133C24.9674 7.01707 25.4353 6.93643 25.8725 6.93643C26.9833 6.93643 27.9865 7.37587 28.696 8.17411C29.6075 9.19872 30.0124 10.4579 29.8361 11.7177C29.7523 12.3177 29.5581 12.8555 29.2678 13.3534C29.8798 13.8646 30.3306 14.5763 30.5485 15.4322C30.719 16.1032 30.8939 17.5006 29.9808 18.9403C30.0389 19.0342 30.0934 19.1319 30.1442 19.2318C30.6932 20.3074 30.7283 21.5229 30.2439 22.6548C29.5093 24.3704 27.6841 25.7219 24.1397 27.1727C21.9347 28.0753 19.9174 28.6523 19.8994 28.6575C16.9842 29.4379 14.3477 29.8345 12.0653 29.8345C7.87017 29.8345 4.8668 28.508 3.13831 25.8921C0.356375 21.6797 0.754104 17.8269 4.35369 14.1131C6.34591 12.058 7.67023 9.02782 7.94613 8.36275C8.50224 6.39343 9.97271 4.20438 12.4172 4.20438H12.4179C12.6236 4.20438 12.8314 4.2214 13.0364 4.25468C14.107 4.42854 15.0428 5.06476 15.7115 6.02205C16.4331 5.09583 17.134 4.359 17.7682 3.94323C18.7242 3.31737 19.6794 3 20.6081 3ZM20.6081 5.95917C20.2427 5.95917 19.7963 6.1197 19.3039 6.44225C17.7754 7.44319 14.8258 12.6772 13.7458 14.7131C13.3839 15.3952 12.7655 15.6837 12.2086 15.6837C11.1036 15.6837 10.2408 14.5497 12.1076 13.1085C14.9146 10.9402 13.9299 7.39584 12.5898 7.1776C12.5311 7.16799 12.4731 7.16355 12.4172 7.16355C11.1989 7.16355 10.6615 9.33114 10.6615 9.33114C10.6615 9.33114 9.0863 13.4148 6.38031 16.206C3.67434 18.998 3.5346 21.2388 5.50675 24.2246C6.85185 26.2606 9.42666 26.8753 12.0653 26.8753C14.8021 26.8753 17.6077 26.2139 19.1799 25.793C19.2574 25.7723 28.8193 22.984 27.6081 20.6107C27.4046 20.212 27.0693 20.0522 26.6471 20.0522C24.9416 20.0522 21.8393 22.6726 20.5057 22.6726C20.2076 22.6726 19.9976 22.5416 19.9116 22.222C19.3433 20.1173 28.552 19.2325 27.7758 16.1839C27.639 15.6445 27.2677 15.4256 26.746 15.4263C24.4923 15.4263 19.4358 19.5181 18.3759 19.5181C18.2949 19.5181 18.2368 19.4937 18.2053 19.4419C17.6743 18.557 17.9653 17.9394 21.7082 15.6009C25.4511 13.2617 28.0783 11.8545 26.5841 10.1752C26.4121 9.98141 26.1684 9.8956 25.8725 9.8956C23.6001 9.89634 18.2311 14.9403 18.2311 14.9403C18.2311 14.9403 16.7821 16.496 15.9057 16.496C15.7043 16.496 15.533 16.4139 15.4169 16.2112C14.7956 15.1296 21.1879 10.1286 21.5484 8.06535C21.7928 6.66715 21.3771 5.95917 20.6081 5.95917Z" fill="#FF9D00"></path>
+    <path d="M5.50686 24.2246C3.53472 21.2387 3.67446 18.9979 6.38043 16.206C9.08641 13.4147 10.6615 9.33111 10.6615 9.33111C10.6615 9.33111 11.2499 6.95933 12.59 7.17757C13.93 7.39581 14.9139 10.9401 12.1069 13.1084C9.29997 15.276 12.6659 16.7489 13.7459 14.713C14.8258 12.6772 17.7747 7.44316 19.304 6.44221C20.8326 5.44128 21.9089 6.00204 21.5484 8.06532C21.188 10.1286 14.795 15.1295 15.4171 16.2118C16.0391 17.2934 18.2312 14.9402 18.2312 14.9402C18.2312 14.9402 25.0907 8.49588 26.5842 10.1752C28.0776 11.8545 25.4512 13.2616 21.7082 15.6008C17.9646 17.9393 17.6744 18.557 18.2054 19.4418C18.7372 20.3266 26.9998 13.1351 27.7759 16.1838C28.5513 19.2324 19.3434 20.1173 19.9117 22.2219C20.48 24.3274 26.3979 18.2382 27.6082 20.6107C28.8193 22.9839 19.2574 25.7722 19.18 25.7929C16.0914 26.62 8.24723 28.3726 5.50686 24.2246Z" fill="#FFD21E"></path>
+</svg>"""
+
+loading_icon_html = """<svg id="share-btn-loading-icon" style="display:none;" class="animate-spin"
+   style="color: #ffffff; 
+"
+   xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" fill="none" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><circle style="opacity: 0.25;" cx="12" cy="12" r="10" stroke="white" stroke-width="4"></circle><path style="opacity: 0.75;" fill="white" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg>"""
+
+share_js = """async () => {
+	async function uploadFile(file){
+		const UPLOAD_URL = 'https://huggingface.co/uploads';
+		const response = await fetch(UPLOAD_URL, {
+			method: 'POST',
+			headers: {
+				'Content-Type': file.type,
+				'X-Requested-With': 'XMLHttpRequest',
+			},
+			body: file, /// <- File inherits from Blob
+		});
+		const url = await response.text();
+		return url;
+	}
+    async function getInputVideoFile(videoEl){
+        const res = await fetch(videoEl.src);
+        const blob = await res.blob();
+        const videoId = Date.now() % 200;
+        const fileName = `sd-perception-${videoId}.mp4`;
+        return new File([blob], fileName, { type: 'video/mp4' }); 
+	}
+    
+    async function audioToBase64(audioFile) {
+        return new Promise((resolve, reject) => {
+            let reader = new FileReader();
+            reader.readAsDataURL(audioFile);
+            reader.onload = () => resolve(reader.result);
+            reader.onerror = error => reject(error);
+          
+        });
+      }
+    const gradioEl = document.querySelector("gradio-app").shadowRoot || document.querySelector('body > gradio-app');
+    const inputPromptEl = gradioEl.querySelector('#prompt-in textarea').value;
+    const outputVideoEl = gradioEl.querySelector('#output-video video');
+    
+    let titleTxt = `WavJourney: ${inputPromptEl}`;
+
+    const shareBtnEl = gradioEl.querySelector('#share-btn');
+    const shareIconEl = gradioEl.querySelector('#share-btn-share-icon');
+    const loadingIconEl = gradioEl.querySelector('#share-btn-loading-icon');
+    if(!outputVideoEl){
+        return;
+    };
+    shareBtnEl.style.pointerEvents = 'none';
+    shareIconEl.style.display = 'none';
+    loadingIconEl.style.removeProperty('display');
+    const outputVideo = await getInputVideoFile(outputVideoEl);
+    const urlOutputVideo = await uploadFile(outputVideo);
+    
+    const descriptionMd = `
+##### ${inputPromptEl}
+    
+${urlOutputVideo}
+`;
+    const params = new URLSearchParams({
+        title: titleTxt,
+        description: descriptionMd,
+    });
+	const paramsStr = params.toString();
+	window.open(`https://huggingface.co/spaces/Audio-AGI/WavJourney/discussions/new?${paramsStr}`, '_blank');
+    shareBtnEl.style.removeProperty('pointer-events');
+    shareIconEl.style.removeProperty('display');
+    loadingIconEl.style.display = 'none';
+}"""

ui_client.py

@@ -0,0 +1,632 @@
+import shutil
+import json5
+import traceback
+
+import gradio as gr
+from tabulate import tabulate
+
+import utils
+import pipeline
+from pipeline import generate_json_file, generate_audio
+from voice_presets import load_voice_presets_metadata, add_session_voice_preset, \
+    remove_session_voice_preset
+from share_btn import community_icon_html, loading_icon_html, share_js
+
+
+
+VOICE_PRESETS_HEADERS = ['ID', 'Description']
+DELETE_FILE_WHEN_DO_CLEAR = False
+DEBUG = False
+
+
+def convert_json_to_md(audio_script_response):
+    audio_json_data = json5.loads(audio_script_response)
+    table = [[node.get(field, 'N/A') for field in ["audio_type", "layout", "id", "character", "action", 'vol']] +
+             [node.get("desc", "N/A") if node.get("audio_type") != "speech" else node.get("text", "N/A")] +
+             [node.get("len", "Auto") if "len" in node else "Auto"]
+             for i, node in enumerate(audio_json_data)]
+
+    headers = ["Audio Type", "Layout", "ID", "Character", "Action", 'Volume', "Description", "Length" ]
+
+    # Tabulate
+    table_txt = tabulate(table, headers, tablefmt="github")
+    return table_txt
+
+
+def convert_char_voice_map_to_md(char_voice_map):
+    table =[[character, char_voice_map[character]["id"]] for character in char_voice_map]
+    headers = ["Character", "Voice"]
+    # Tabulate
+    table_txt = tabulate(table, headers, tablefmt="github")
+    return table_txt
+
+
+def get_or_create_session_from_state(ui_state):
+    if 'session_id' not in ui_state:
+        ui_state['session_id'] = pipeline.init_session()
+    return ui_state['session_id']
+
+
+def generate_script_fn(instruction, _state: gr.State):
+    try:
+        session_id = get_or_create_session_from_state(_state)
+        api_key = utils.get_api_key()
+        json_script = generate_json_file(session_id, instruction, api_key)
+        table_text = convert_json_to_md(json_script)
+    except Exception as e:
+        gr.Warning(str(e))
+        print(f"Generating script error: {str(e)}")
+        traceback.print_exc()
+        return [
+            None,
+            _state,
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=True),
+            gr.Button.update(interactive=True),
+            gr.Button.update(interactive=True),
+        ]
+
+    _state = {
+        **_state,
+        'session_id': session_id,
+        'json_script': json_script
+    }
+    return [
+        table_text, 
+        _state,
+        gr.Button.update(interactive=True),
+        gr.Button.update(interactive=True),
+        gr.Button.update(interactive=True),
+        gr.Button.update(interactive=True),
+    ]
+
+
+def generate_audio_fn(state):
+    btn_state = gr.Button.update(interactive=True)
+    try:
+        api_key = utils.get_api_key()
+        audio_path, char_voice_map = generate_audio(**state, api_key=api_key)
+        table_text = convert_char_voice_map_to_md(char_voice_map)
+        # TODO: output char_voice_map to a table
+        return [
+            table_text,
+            gr.make_waveform(str(audio_path)),
+            btn_state,
+            btn_state,
+            btn_state,
+            btn_state,
+        ]
+    except Exception as e:
+        print(f"Generation audio error: {str(e)}")
+        traceback.print_exc()
+        gr.Warning(str(e))
+
+    return [
+        None,
+        None,
+        btn_state,
+        btn_state,
+        btn_state,
+        btn_state,
+    ]
+
+
+def clear_fn(state):
+    if DELETE_FILE_WHEN_DO_CLEAR:
+        shutil.rmtree('output', ignore_errors=True)
+    state = {'session_id': pipeline.init_session()}
+    return [gr.Markdown.update(value=''),
+            gr.Textbox.update(value=''), 
+            gr.Video.update(value=None),
+            gr.Markdown.update(value=''), 
+            gr.Button.update(interactive=False), 
+            gr.Button.update(interactive=False),
+            state, gr.Dataframe.update(visible=False), 
+            gr.Button.update(visible=False),
+            gr.Textbox.update(value=''), 
+            gr.Textbox.update(value=''), 
+            gr.File.update(value=None)]
+
+
+def textbox_listener(textbox_input):
+    if len(textbox_input) > 0:
+        return gr.Button.update(interactive=True)
+    else:
+        return gr.Button.update(interactive=False)
+
+
+def get_voice_preset_to_list(state: gr.State):
+    if state.__class__ == gr.State:
+        state = state.value
+    if 'session_id' in state:
+        path = utils.get_session_voice_preset_path(state['session_id'])
+    else:
+        path = ''
+    voice_presets = load_voice_presets_metadata(
+        path,
+        safe_if_metadata_not_exist=True
+    )
+    dataframe = []
+    for key in voice_presets.keys():
+        row = [key, voice_presets[key]['desc']]
+        dataframe.append(row)
+    return dataframe
+
+
+def df_on_select(evt: gr.SelectData):
+    print(f"You selected {evt.value} at {evt.index} from {evt.target}")
+    return {'selected_voice_preset': evt.index}
+
+
+def del_voice_preset(selected_voice_presets, ui_state, dataframe):
+    gr_visible = gr.Dataframe.update(visible=True)
+    btn_visible = gr.Button.update(visible=True)
+    current_presets = get_voice_preset_to_list(ui_state)
+    if selected_voice_presets['selected_voice_preset'] is None or \
+            selected_voice_presets['selected_voice_preset'][0] > len(current_presets) - 1:
+        gr.Warning('None row is selected')
+        return [current_presets, gr_visible, btn_visible, selected_voice_presets]
+    # Do the real file deletion
+    index = selected_voice_presets['selected_voice_preset'][0]
+    vp_id = dataframe['ID'][index]
+    remove_session_voice_preset(vp_id, ui_state['session_id'])
+    current_presets = get_voice_preset_to_list(ui_state)
+    gr.Dataframe.update(value=current_presets)
+    if len(current_presets) == 0:
+        gr_visible = gr.Dataframe.update(visible=False)
+        btn_visible = gr.Button.update(visible=False)
+    selected_voice_presets['selected_voice_preset'] = None
+    return [current_presets, gr_visible, btn_visible, selected_voice_presets]
+
+
+def get_system_voice_presets():
+    system_presets = load_voice_presets_metadata(utils.get_system_voice_preset_path())
+    data = []
+    for k, v in system_presets.items():
+        data.append([k, v['desc']])
+    # headers = ['id', 'description']
+    # table_txt = tabulate(data, headers, tablefmt="github")
+    return data
+
+
+def set_openai_key(key, _state):
+    _state['api_key'] = key
+    return key
+
+
+def add_voice_preset(vp_id, vp_desc, file, ui_state, added_voice_preset):
+    if vp_id is None or vp_desc is None or file is None or vp_id.strip() == '' or vp_desc.strip() == '':
+        gr.Warning('please complete all three fields')
+    else:
+        count: int = added_voice_preset['count']
+        # check if greater than 3
+        session_id = get_or_create_session_from_state(ui_state)
+        file_path = file.name
+        print(f'session {session_id}, id {id}, desc {vp_desc}, file {file_path}')
+        # Do adding ...
+        try:
+            add_session_voice_preset(vp_id, vp_desc, file_path, session_id)
+            added_voice_preset['count'] = count + 1
+        except Exception as exception:
+            print(exception)
+            traceback.print_exc()
+            gr.Warning(str(exception))
+
+    # After added
+    dataframe = get_voice_preset_to_list(ui_state)
+    df_visible = gr.Dataframe.update(visible=True)
+    del_visible = gr.Button.update(visible=True)
+    if len(dataframe) == 0:
+        df_visible = gr.Dataframe.update(visible=False)
+        del_visible = gr.Button.update(visible=False)
+    return [gr.Textbox.update(value=''), gr.Textbox.update(value=''), gr.File.update(value=None),
+            ui_state, added_voice_preset, dataframe, gr.Button.update(interactive=True),
+            df_visible, del_visible]
+
+
+css = """
+        a {
+            color: inherit;
+            text-decoration: underline;
+        }
+        .gradio-container {
+            font-family: 'IBM Plex Sans', sans-serif;
+        }
+        .gr-button {
+            color: white;
+            border-color: #000000;
+            background: #000000;
+        }
+        input[type='range'] {
+            accent-color: #000000;
+        }
+        .dark input[type='range'] {
+            accent-color: #dfdfdf;
+        }
+        .container {
+            max-width: 730px;
+            margin: auto;
+            padding-top: 1.5rem;
+        }
+        #gallery {
+            min-height: 22rem;
+            margin-bottom: 15px;
+            margin-left: auto;
+            margin-right: auto;
+            border-bottom-right-radius: .5rem !important;
+            border-bottom-left-radius: .5rem !important;
+        }
+        #gallery>div>.h-full {
+            min-height: 20rem;
+        }
+        .details:hover {
+            text-decoration: underline;
+        }
+        .gr-button {
+            white-space: nowrap;
+        }
+        .gr-button:focus {
+            border-color: rgb(147 197 253 / var(--tw-border-opacity));
+            outline: none;
+            box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
+            --tw-border-opacity: 1;
+            --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
+            --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
+            --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
+            --tw-ring-opacity: .5;
+        }
+        #advanced-btn {
+            font-size: .7rem !important;
+            line-height: 19px;
+            margin-top: 12px;
+            margin-bottom: 12px;
+            padding: 2px 8px;
+            border-radius: 14px !important;
+        }
+        #advanced-options {
+            margin-bottom: 20px;
+        }
+        .footer {
+            margin-bottom: 45px;
+            margin-top: 35px;
+            text-align: center;
+            border-bottom: 1px solid #e5e5e5;
+        }
+        .footer>p {
+            font-size: .8rem;
+            display: inline-block;
+            padding: 0 10px;
+            transform: translateY(10px);
+            background: white;
+        }
+        .dark .footer {
+            border-color: #303030;
+        }
+        .dark .footer>p {
+            background: #0b0f19;
+        }
+        .acknowledgments h4{
+            margin: 1.25em 0 .25em 0;
+            font-weight: bold;
+            font-size: 115%;
+        }
+        #container-advanced-btns{
+            display: flex;
+            flex-wrap: wrap;
+            justify-content: space-between;
+            align-items: center;
+        }
+        .animate-spin {
+            animation: spin 1s linear infinite;
+        }
+        @keyframes spin {
+            from {
+                transform: rotate(0deg);
+            }
+            to {
+                transform: rotate(360deg);
+            }
+        }
+        #share-btn-container {
+            display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem;
+            margin-top: 10px;
+            margin-left: auto;
+        }
+        #share-btn {
+            all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0;
+        }
+        #share-btn * {
+            all: unset;
+        }
+        #share-btn-container div:nth-child(-n+2){
+            width: auto !important;
+            min-height: 0px !important;
+        }
+        #share-btn-container .wrap {
+            display: none !important;
+        }
+        .gr-form{
+            flex: 1 1 50%; border-top-right-radius: 0; border-bottom-right-radius: 0;
+        }
+        #prompt-container{
+            gap: 0;
+        }
+        #generated_id{
+            min-height: 700px
+        }
+        #setting_id{
+          margin-bottom: 12px;
+          text-align: center;
+          font-weight: 900;
+        }
+"""
+
+with gr.Blocks(css=css) as interface:
+
+    gr.HTML(
+            """
+                <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+                <div
+                    style="
+                    display: inline-flex;
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1.75rem;
+                    "
+                >
+                    <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
+                    WavJourney: Compositional Audio Creation with LLMs
+                    </h1>
+                </div>
+                <p style="margin-bottom: 10px; margin-top: 10px; font-size: 94%">
+                    <a href="https://arxiv.org/abs/2307.14335">[Paper]</a> <a href="https://audio-agi.github.io/WavJourney_demopage/">[Demo Page]</a> <a href="https://github.com/Audio-AGI/WavJourney">[GitHub]</a> <a href="https://discord.com/invite/5Hqu9NmA8V">[Join Discord]</a>
+                </p>
+                </div>
+            """
+        )
+    gr.HTML(
+        """
+        <p>Due to the high user demand we are facing from our community, we will be offering free access to WavJourney for a few more days. You can also access WavJourney in this space later by providing your OPENAI_KEY.<p/>
+        <p>For faster inference without waiting in the queue, you can duplicate the space and upgrade to GPU (VRAM>16G) and provide OPENAI_KEY to access GPT-4 in settings.
+        <br>
+        <a href="https://huggingface.co/spaces/Audio-AGI/WavJourney?duplicate=true">
+        <img style="margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>
+        <p/>
+    """
+    )
+
+    # gr.HTML(
+    #     """
+    #     <p>Begin with a text prompt, and let WavJourney transform it into captivating audio content. Experience engaging audio storylines, personalized voices, lifelike speech, emotionally resonant musical compositions, and immersive sound effects!
+    #     <p/>
+    # """
+    # )
+    
+    gr.Markdown(value='## WavJourney Pipeline:')
+
+    gr.Markdown(value='Begin with a text prompt, and let WavJourney transform it into captivating audio content. Experience engaging audio storylines, personalized voices, lifelike speech, emotionally resonant musical compositions, and immersive sound effects!')
+
+    gr.HTML(
+        """
+        <ul>
+        <li>Stage 0: (optional) add your customized voice preset for a more personalized audio creation experience. User also often shares presets in <a href="https://discord.com/invite/5Hqu9NmA8V">Discord</a>.</li>
+        <li>Stage 1: generate the audio script based on the input text instruction (the default language is English, but you can actually type in your own language).</li>
+        <li>Stage 2: Select the suitable voice in the multilingual voice preset for each character in the audio script & generate audio.</li>
+        </ul>
+
+
+    """
+    )
+
+
+
+    system_voice_presets = get_system_voice_presets()
+    # State
+    ui_state = gr.State({})
+    selected_voice_presets = gr.State(value={'selected_voice_preset': None})
+    added_voice_preset_state = gr.State(value={'added_file': None, 'count': 0})
+    # UI Component
+    # gr.Markdown(
+    # """
+    # How can I access GPT-4? <a href="https://platform.openai.com/account/api-keys">[Ref1]</a><a href="https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4">[Ref2]</a>
+    # """
+    # )
+    # key_text_input = gr.Textbox(label='Please Enter OPENAI Key for accessing GPT-4 API', lines=1, placeholder="OPENAI Key here.",
+    #                         value=utils.get_key())
+    text_input_value = '' if DEBUG is False else "an audio introduction to quantum mechanics"
+    
+    text_input = gr.Textbox(
+        label='Input Text Instruction', 
+        lines=2, 
+        placeholder="Input instruction here (e.g., An introduction to AI-assisted audio content creation).",
+        value=text_input_value,
+        elem_id="prompt-in",)
+
+    gr.Markdown(
+    """
+    Clicking 'Generate Script' button, the generated audio script will be displayed below.
+    """
+    )
+    audio_script_markdown = gr.Markdown(label='Audio Script')
+    generate_script_btn = gr.Button(value='Generate Script', interactive=False)
+    
+    gr.Markdown(
+    """
+    Clicking 'Generate Audio' button, the voice mapping results & generated audio will be displayed below (might take some time).
+    """
+    )
+    char_voice_map_markdown = gr.Markdown(label='Character-to-voice Map')
+
+    audio_output = gr.Video(elem_id="output-video")
+
+    generate_audio_btn = gr.Button(value='Generate Audio', interactive=False)
+    
+    # share to community
+    with gr.Group(elem_id="share-btn-container", visible=False):
+        community_icon = gr.HTML(community_icon_html)
+        loading_icon = gr.HTML(loading_icon_html)
+        share_button = gr.Button(value="Share to community", elem_id="share-btn")
+    
+    gr.Markdown(value='### Share your creation with the community!')
+    gr.HTML(
+        """
+        <ul> 
+        <li> You can share with the HuggingFace community by clicking the "Share to community" button.</li>
+        <li> You can share your generations to our <a href="https://discord.com/invite/5Hqu9NmA8V">Discord</a> channel!</li>
+        <li> You can also share the voice presets (along with descriptions) you found in <a href="https://discord.com/invite/5Hqu9NmA8V">Discord</a>.</li>
+        </ul>
+    """
+    )
+    
+    
+    
+    
+    gr.Markdown(value='### Useful tips for prompting WavJourney:')
+
+    gr.HTML(
+        """
+        <ul> 
+        <li>You can use vague or specific descriptions or a combination of them. For example: "male speech about pizza" or "a man is saying: I love pizza!"</li>
+        <li> You can control the length of the audio script by simply adding the restriction. For example: "generate an audio script around 10-15 lines (max length has been set to 30)"</li>
+        <li> You can specify the language of the speaker. For example: "a boy is playing with a girl, boy's speech is in Chinese while girl's speech in Japanese"</li>
+        <li> Explore more prompting techniques by yourself! 🤗</li>
+        </ul>
+
+    """
+    )
+
+    # add examples
+    from examples.examples import examples as WJExamples
+    def example_fn(idx, _text_input):
+        print('from example', idx, _text_input)
+        example = WJExamples[int(idx)-1]
+        print(example['table_script'], example['table_voice'], gr.make_waveform(example['wav_file']))
+        return example['table_script'], example['table_voice'], gr.make_waveform(example['wav_file'])
+
+    _idx_input = gr.Textbox(label='Example No.')
+    _idx_input.visible=False
+    gr.Examples(
+            [[idx+1, x['text']] for idx, x in enumerate(WJExamples)],
+            fn=example_fn,
+            inputs=[_idx_input, text_input],
+            outputs=[audio_script_markdown, char_voice_map_markdown, audio_output],
+            cache_examples=True,
+        )
+
+    # System Voice Presets
+    gr.Markdown(label='System Voice Presets', value='### System Voice Presets')
+    with gr.Accordion("Click to display system speakers", open=False):
+        gr.Markdown('Supported Language: English, Chinese, French, German, Hindi, Italian, Japanese, Korean, Russian, Spanish, Turkish, Polish, Portuguese')
+
+        system_markdown_voice_presets = gr.Dataframe(label='System Voice Presets', headers=VOICE_PRESETS_HEADERS,
+                                                    value=system_voice_presets)
+    # User Voice Preset Related
+    gr.Markdown('## (Optional) Speaker Customization ')
+    with gr.Accordion("Click to add speakers", open=False):
+        gr.Markdown(label='User Voice Presets', value='### User Voice Presets')
+        get_voice_preset_to_list(ui_state)
+        voice_presets_df = gr.Dataframe(headers=VOICE_PRESETS_HEADERS, col_count=len(VOICE_PRESETS_HEADERS),
+                                    value=get_voice_preset_to_list(ui_state), interactive=False, visible=False)
+    # voice_presets_ds = gr.Dataset(components=[gr.Dataframe(visible=True)], samples=get_voice_preset_to_list(ui_state))
+        del_voice_btn = gr.Button(value='Delete Selected Voice Preset', visible=False)
+        gr.Markdown(label='Add Voice Preset', value='### Add Voice Preset')
+        gr.Markdown(
+        """
+        What makes for good voice prompt? See detailed instructions <a href="https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer">here</a>. 
+        """
+        )
+        vp_text_id = gr.Textbox(label='Id', lines=1, placeholder="Input voice preset id here.")
+        vp_text_desc = gr.Textbox(label='Desc', lines=1, placeholder="Input description here.")
+        vp_file = gr.File(label='Wav File', type='file', file_types=['.wav'],
+                        interactive=True)
+        vp_submit = gr.Button(label='Upload Voice Preset', value="Upload Voice Preset")
+    
+    # clear btn, will re-new a session
+    clear_btn = gr.ClearButton(value='Clear All')
+
+    # disclaimer
+    gr.Markdown(
+        """
+    ## Disclaimer
+    We are not responsible for audio generated using semantics created by WavJourney. Just don't use it for illegal purposes.
+    """
+    )
+
+    # events
+    # key_text_input.change(fn=set_openai_key, inputs=[key_text_input, ui_state], outputs=[key_text_input])
+    text_input.change(fn=textbox_listener, inputs=[text_input], outputs=[generate_script_btn])
+    generate_audio_btn.click(
+        fn=generate_audio_fn,
+        inputs=[ui_state],
+        outputs=[
+            char_voice_map_markdown,
+            audio_output,
+            generate_audio_btn,
+            generate_script_btn,
+            clear_btn,
+            vp_submit,
+        ],
+        api_name='audio_journey',
+    )
+    generate_audio_btn.click(
+        fn=lambda: [
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+        ],
+        outputs=[
+            generate_audio_btn,
+            generate_script_btn,
+            clear_btn,
+            vp_submit,
+        ]
+    )
+    clear_btn.click(fn=clear_fn, inputs=ui_state,
+                    outputs=[char_voice_map_markdown, text_input, audio_output, audio_script_markdown, generate_audio_btn, generate_script_btn,
+                             ui_state, voice_presets_df, del_voice_btn,
+                             vp_text_id, vp_text_desc, vp_file])
+    generate_script_btn.click(
+        fn=generate_script_fn, inputs=[text_input, ui_state],
+        outputs=[
+            audio_script_markdown,
+            ui_state,
+            generate_audio_btn,
+            generate_script_btn,
+            clear_btn,
+            vp_submit,
+        ]
+    )
+    generate_script_btn.click(
+        fn=lambda: [
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+        ],
+        outputs=[
+            generate_audio_btn,
+            generate_script_btn,
+            clear_btn,
+            vp_submit,
+        ]
+    )
+    voice_presets_df.select(df_on_select, outputs=[selected_voice_presets])
+    voice_presets_df.update(lambda x: print(x))
+    del_voice_btn.click(del_voice_preset, inputs=[selected_voice_presets, ui_state, voice_presets_df],
+                        outputs=[voice_presets_df, voice_presets_df, del_voice_btn, selected_voice_presets])
+    # user voice preset upload
+    vp_submit.click(add_voice_preset, inputs=[vp_text_id, vp_text_desc, vp_file, ui_state, added_voice_preset_state],
+                    outputs=[vp_text_id, vp_text_desc, vp_file, ui_state, added_voice_preset_state, voice_presets_df,
+                             vp_submit,
+                             voice_presets_df, del_voice_btn])
+    vp_submit.click(lambda _: gr.Button.update(interactive=False), inputs=[vp_submit])
+    
+    # share to HF community
+    share_button.click(None, [], [], _js=share_js)
+
+    # debug only
+    # print_state_btn = gr.Button(value='Print State')
+    # print_state_btn.click(fn=lambda state, state2: print(state, state2), inputs=[ui_state, selected_voice_presets])
+interface.queue(concurrency_count=2, max_size=20)
+interface.launch()

utils.py

@@ -0,0 +1,82 @@
+import os
+import re
+import torch
+import  numpy as np
+import yaml
+from pathlib import Path
+
+
+#### path related code BEGIN ####
+def get_session_path(session_id):
+    return Path(f'output/sessions/{session_id}')
+
+def get_system_voice_preset_path():
+    return Path('data/voice_presets')
+    
+def get_session_voice_preset_path(session_id):
+    return Path(f'{get_session_path(session_id)}/voice_presets')
+    
+def get_session_audio_path(session_id):
+    return Path(f'{get_session_path(session_id)}/audio')
+
+def rescale_to_match_energy(segment1, segment2):
+    ratio = get_energy_ratio(segment1, segment2)
+    recaled_segment1 = segment1 / ratio
+    return recaled_segment1.numpy()
+#### path related code END ####
+
+def text_to_abbrev_prompt(input_text):
+    return re.sub(r'[^a-zA-Z_]', '', '_'.join(input_text.split()[:5]))
+
+def get_energy(x):
+    return np.mean(x ** 2)
+
+
+def get_energy_ratio(segment1, segment2):
+    energy1 = get_energy(segment1)
+    energy2 = max(get_energy(segment2), 1e-10)
+    ratio = (energy1 / energy2) ** 0.5
+    ratio = torch.tensor(ratio)
+    ratio = torch.clamp(ratio, 0.02, 50)
+    return ratio
+
+def fade(audio_data, fade_duration=2, sr=32000):
+    audio_duration = audio_data.shape[0] / sr
+
+    # automated choose fade duration
+    if audio_duration >=8:
+         # keep fade_duration 2
+        pass
+    else:
+        fade_duration = audio_duration / 5
+
+    fade_sampels = int(sr * fade_duration)
+    fade_in = np.linspace(0, 1, fade_sampels)
+    fade_out = np.linspace(1, 0, fade_sampels)
+
+    audio_data_fade_in = audio_data[:fade_sampels] * fade_in
+    audio_data_fade_out = audio_data[-fade_sampels:] * fade_out
+
+    audio_data_faded = np.concatenate((audio_data_fade_in, audio_data[len(fade_in):-len(fade_out)], audio_data_fade_out))
+    return audio_data_faded
+
+# def get_key(config='config.yaml'):
+#     with open('config.yaml', 'r') as file:
+#         config = yaml.safe_load(file)
+#         return config['OpenAI-Key'] if 'OpenAI-Key' in config else None
+
+def get_service_port():
+    service_port = os.environ.get('WAVJOURNEY_SERVICE_PORT')
+    return service_port
+
+def get_service_url():
+    service_url = os.environ.get('WAVJOURNEY_SERVICE_URL')
+    return service_url 
+
+def get_api_key():
+    api_key = os.environ.get('WAVJOURNEY_OPENAI_KEY')
+    return api_key       
+
+def get_max_script_lines():
+    max_lines = int(os.environ.get('WAVJOURNEY_MAX_SCRIPT_LINES', 999))
+    return max_lines

voice_presets.py

@@ -0,0 +1,96 @@
+import os
+import json, json5
+from pathlib import Path
+
+import utils
+from APIs import VP
+
+
+def save_voice_presets_metadata(voice_presets_path, metadata):
+    with open(voice_presets_path / 'metadata.json', 'w') as f:
+        json.dump(metadata, f, indent=4)
+
+def load_voice_presets_metadata(voice_presets_path, safe_if_metadata_not_exist=False):
+    metadata_full_path = Path(voice_presets_path) / 'metadata.json'
+
+    if safe_if_metadata_not_exist:
+        if not os.path.exists(metadata_full_path):
+            return {}
+
+    with open(metadata_full_path, 'r') as f:
+        presets = json5.load(f)
+
+    return presets
+
+# return system voice presets and session voice presets individually, each in a list
+def get_voice_presets(session_id):
+    system_presets, session_presets = [], []
+
+    # Load system presets
+    system_presets = load_voice_presets_metadata(utils.get_system_voice_preset_path())
+
+    # Load session presets
+    session_presets = load_voice_presets_metadata(
+        utils.get_session_voice_preset_path(session_id),
+        safe_if_metadata_not_exist=True
+    )
+
+    return system_presets, session_presets
+
+# return merged voice presets in a {voice_preset_name: voice_preset} dict
+def get_merged_voice_presets(session_id):
+    system_presets, session_presets = get_voice_presets(session_id)
+    res = {}
+    for preset in list(system_presets.values()) + list(session_presets.values()):
+        res[preset['id']] = preset  # session presets with the same id will cover that of system presets
+    return res
+
+def add_voice_preset(voice_presets_path, presets, id, desc, wav_file_path):
+    if id in presets:
+        raise KeyError(f'{id} already in voice preset, path={voice_presets_path}!')
+
+    # Convert wav to npz
+    npz_path = voice_presets_path / 'npz'
+    VP(wav_file_path, npz_path)
+    npz_file_path = npz_path / f'{Path(wav_file_path).stem}.npz'
+
+    presets[id]  = {
+        'id': id,
+        'desc': desc,
+        'npz_path': str(npz_file_path)
+    }
+    save_voice_presets_metadata(voice_presets_path, presets)
+    return presets[id]
+
+def add_session_voice_preset(id, desc, wav_file_path, session_id):
+    voice_presets_path = utils.get_session_voice_preset_path(session_id)
+    os.makedirs(voice_presets_path / 'npz', exist_ok=True)
+    presets = load_voice_presets_metadata(voice_presets_path, safe_if_metadata_not_exist=True)
+    if len(presets) >= 3:
+        raise ValueError(f'session voice presets size exceed 3')
+    if id in presets:
+        raise KeyError(f'{id} already in voice preset, path={voice_presets_path}!')
+
+    return add_voice_preset(voice_presets_path, presets, id, desc, wav_file_path)
+
+def add_system_voice_preset(id, desc, wav_file_path):
+    voice_presets_path = utils.get_system_voice_preset_path()
+    presets = load_voice_presets_metadata(voice_presets_path)
+    return add_voice_preset(voice_presets_path, presets, id, desc, wav_file_path)
+
+# if session_id set to '', we are removing system voice presets
+def remove_session_voice_preset(id, session_id):
+    voice_presets_path = utils.get_session_voice_preset_path(session_id)
+    presets = load_voice_presets_metadata(
+        voice_presets_path,
+        safe_if_metadata_not_exist=True
+    )
+    preset = presets.pop(id)
+    npz_path = preset['npz_path']
+
+    try:
+        os.remove(npz_path)
+    except FileNotFoundError:
+        print(f"INFO: trying to delete {npz_path} which does not exist, path={voice_presets_path}.")
+
+    save_voice_presets_metadata(voice_presets_path, presets)