Spaces:
Running
Running
""" | |
Copyright (c) Meta Platforms, Inc. and affiliates. | |
All rights reserved. | |
This source code is licensed under the license found in the | |
LICENSE file in the root directory of this source tree. | |
""" | |
from tempfile import NamedTemporaryFile | |
import argparse | |
import torch | |
import torchaudio | |
import gradio as gr | |
import os | |
from audiocraft.models import MusicGen | |
from audiocraft.data.audio import audio_write | |
from share_btn import community_icon_html, loading_icon_html, share_js, css | |
MODEL = None | |
def load_model(version): | |
print("Loading model", version) | |
return MusicGen.get_pretrained(version) | |
def predict( | |
text, | |
melody_input, | |
duration=30, | |
continuation_start=0, | |
continuation_end=30, | |
topk=250, | |
topp=0, | |
temperature=1, | |
cfg_coef=3, | |
): | |
global MODEL | |
topk = int(topk) | |
if MODEL is None: | |
MODEL = load_model("melody") | |
if melody_input is None: | |
raise gr.Error("Please upload a melody to continue!") | |
if duration > MODEL.lm.cfg.dataset.segment_duration: | |
raise gr.Error("MusicGen currently supports durations of up to 30 seconds!") | |
if continuation_end < continuation_start: | |
raise gr.Error("The end time must be greater than the start time!") | |
MODEL.set_generation_params( | |
use_sampling=True, | |
top_k=topk, | |
top_p=topp, | |
temperature=temperature, | |
cfg_coef=cfg_coef, | |
duration=duration, | |
) | |
if melody_input: | |
melody, sr = torchaudio.load(melody_input) | |
# sr, melody = melody_input[0], torch.from_numpy(melody_input[1]).to(MODEL.device).float().t().unsqueeze(0) | |
if melody.dim() == 2: | |
melody = melody[None] | |
print("\nGenerating continuation\n") | |
melody_wavform = melody[ | |
..., int(sr * continuation_start) : int(sr * continuation_end) | |
] | |
melody_duration = melody_wavform.shape[-1] / sr | |
if duration + melody_duration > MODEL.lm.cfg.dataset.segment_duration: | |
raise gr.Error("Duration + continuation duration must be <= 30 seconds") | |
output = MODEL.generate_continuation( | |
prompt=melody_wavform, | |
prompt_sample_rate=sr, | |
descriptions=[text], | |
progress=True, | |
) | |
output = output.detach().cpu().float()[0] | |
with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file: | |
audio_write( | |
file.name, | |
output, | |
MODEL.sample_rate, | |
strategy="loudness", | |
loudness_headroom_db=16, | |
loudness_compressor=True, | |
add_suffix=False, | |
) | |
waveform_video = gr.make_waveform(file.name) | |
return ( | |
waveform_video, | |
(sr, melody_wavform.unsqueeze(0).numpy()) if melody_input else None, | |
) | |
def ui(**kwargs): | |
def toggle(choice): | |
if choice == "mic": | |
return gr.update(source="microphone", value=None, label="Microphone") | |
else: | |
return gr.update(source="upload", value=None, label="File") | |
def check_melody_length(melody_input): | |
if not melody_input: | |
return gr.update(maximum=0, value=0), gr.update(maximum=0, value=0) | |
melody, sr = torchaudio.load(melody_input) | |
audio_length = melody.shape[-1] / sr | |
if melody.dim() == 2: | |
melody = melody[None] | |
return gr.update(maximum=audio_length, value=0), gr.update( | |
maximum=audio_length, value=audio_length | |
) | |
def preview_melody_cut(melody_input, continuation_start, continuation_end): | |
if not melody_input: | |
return gr.update(maximum=0, value=0), gr.update(maximum=0, value=0) | |
melody, sr = torchaudio.load(melody_input) | |
audio_length = melody.shape[-1] / sr | |
if melody.dim() == 2: | |
melody = melody[None] | |
if continuation_end < continuation_start: | |
raise gr.Error("The end time must be greater than the start time!") | |
if continuation_start < 0 or continuation_end > audio_length: | |
raise gr.Error("The continuation settings must be within the audio length!") | |
print("cutting", int(sr * continuation_start), int(sr * continuation_end)) | |
prompt_waveform = melody[ | |
..., int(sr * continuation_start) : int(sr * continuation_end) | |
] | |
return (sr, prompt_waveform.unsqueeze(0).numpy()) | |
with gr.Blocks(css=css) as interface: | |
gr.Markdown( | |
""" | |
# MusicGen Continuation | |
This a [MusicGen](https://github.com/facebookresearch/audiocraft), a simple and controllable model for music generation | |
presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284) | |
This Spaces only does melody continuation, you can try other features [here](https://huggingface.co/spaces/facebook/MusicGen) | |
""" | |
) | |
gr.Markdown( | |
""" | |
<a href="https://huggingface.co/spaces/radames/MusicGen-Continuation?duplicate=true" style="display: inline-block;margin-top: .5em;margin-right: .25em;" target="_blank"> | |
<img style="margin-bottom: 0em;display: inline;margin-top: -.25em;" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a> | |
to use it privately | |
""" | |
) | |
with gr.Row(): | |
with gr.Column(): | |
with gr.Row(): | |
text = gr.Text( | |
label="Describe your music", | |
lines=2, | |
interactive=True, | |
elem_id="text-input", | |
) | |
with gr.Column(): | |
radio = gr.Radio( | |
["file", "mic"], | |
value="file", | |
label="Melody Inital Condition File or Mic", | |
info="Make sure the audio is no longer than total generation duration which is max 30 seconds, you can trim the audio in the next section", | |
) | |
melody = gr.Audio( | |
source="upload", | |
type="filepath", | |
label="File", | |
interactive=True, | |
elem_id="melody-input", | |
) | |
with gr.Row(): | |
submit = gr.Button("Submit") | |
with gr.Row(): | |
duration = gr.Slider( | |
minimum=1, | |
maximum=30, | |
value=10, | |
label="Total Generation Duration", | |
interactive=True, | |
) | |
with gr.Accordion(label="Input Melody Trimming (optional)", open=False): | |
with gr.Row(): | |
continuation_start = gr.Slider( | |
minimum=0, | |
maximum=30, | |
step=0.01, | |
value=0, | |
label="melody cut start", | |
interactive=True, | |
) | |
continuation_end = gr.Slider( | |
minimum=0, | |
maximum=30, | |
step=0.01, | |
value=0, | |
label="melody cut end", | |
interactive=True, | |
) | |
cut_btn = gr.Button("Cut Melody").style(full_width=False) | |
with gr.Row(): | |
preview_cut = gr.Audio( | |
type="numpy", | |
label="Cut Preview", | |
) | |
with gr.Accordion(label="Advanced Settings", open=False): | |
with gr.Row(): | |
topk = gr.Number(label="Top-k", value=250, interactive=True) | |
topp = gr.Number(label="Top-p", value=0, interactive=True) | |
temperature = gr.Number( | |
label="Temperature", value=1.0, interactive=True | |
) | |
cfg_coef = gr.Number( | |
label="Classifier Free Guidance", | |
value=3.0, | |
interactive=True, | |
) | |
with gr.Column(): | |
output = gr.Video(label="Generated Music", elem_id="generated-video") | |
output_melody = gr.Audio(label="Melody ", elem_id="melody-output") | |
with gr.Row(visible=False) as share_row: | |
with gr.Group(elem_id="share-btn-container"): | |
community_icon = gr.HTML(community_icon_html) | |
loading_icon = gr.HTML(loading_icon_html) | |
share_button = gr.Button( | |
"Share to community", elem_id="share-btn" | |
) | |
share_button.click(None, [], [], _js=share_js) | |
melody.change( | |
check_melody_length, | |
melody, | |
[continuation_start, continuation_end], | |
queue=False, | |
) | |
cut_btn.click( | |
preview_melody_cut, | |
[melody, continuation_start, continuation_end], | |
preview_cut, | |
queue=False, | |
) | |
submit.click( | |
lambda x: gr.update(visible=False), | |
None, | |
[share_row], | |
queue=False, | |
show_progress=False, | |
).then( | |
predict, | |
inputs=[ | |
text, | |
melody, | |
duration, | |
continuation_start, | |
continuation_end, | |
topk, | |
topp, | |
temperature, | |
cfg_coef, | |
], | |
outputs=[output, output_melody], | |
).then( | |
lambda x: gr.update(visible=True), | |
None, | |
[share_row], | |
queue=False, | |
show_progress=False, | |
) | |
radio.change(toggle, radio, [melody], queue=False, show_progress=False) | |
examples = gr.Examples( | |
fn=predict, | |
examples=[ | |
[ | |
"An 80s driving pop song with heavy drums and synth pads in the background", | |
"./assets/bach.mp3", | |
25, | |
0, | |
5, | |
], | |
[ | |
"A cheerful country song with acoustic guitars", | |
"./assets/bolero_ravel.mp3", | |
25, | |
0, | |
5, | |
], | |
[ | |
"90s rock song with electric guitar and heavy drums", | |
"./assets/bach.mp3", | |
25, | |
0, | |
5, | |
], | |
[ | |
"a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions", | |
"./assets/bach.mp3", | |
25, | |
0, | |
5, | |
], | |
[ | |
"lofi slow bpm electro chill with organic samples", | |
"./assets/bolero_ravel.mp3", | |
25, | |
0, | |
5, | |
], | |
], | |
inputs=[text, melody, duration, continuation_start, continuation_end], | |
outputs=[output], | |
) | |
gr.Markdown( | |
""" | |
### More details | |
The model will generate a short music extract based on the description you provided. | |
You can generate up to 30 seconds of audio. | |
We present 4 model variations: | |
1. Melody -- a music generation model capable of generating music condition on text and melody inputs. **Note**, you can also use text only. | |
2. Small -- a 300M transformer decoder conditioned on text only. | |
3. Medium -- a 1.5B transformer decoder conditioned on text only. | |
4. Large -- a 3.3B transformer decoder conditioned on text only (might OOM for the longest sequences.) | |
When using `melody`, ou can optionaly provide a reference audio from | |
which a broad melody will be extracted. The model will then try to follow both the description and melody provided. | |
You can also use your own GPU or a Google Colab by following the instructions on our repo. | |
See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft) | |
for more details. | |
""" | |
) | |
# Show the interface | |
launch_kwargs = {} | |
username = kwargs.get("username") | |
password = kwargs.get("password") | |
server_port = kwargs.get("server_port", 0) | |
inbrowser = kwargs.get("inbrowser", False) | |
share = kwargs.get("share", False) | |
server_name = kwargs.get("listen") | |
launch_kwargs["server_name"] = server_name | |
if username and password: | |
launch_kwargs["auth"] = (username, password) | |
if server_port > 0: | |
launch_kwargs["server_port"] = server_port | |
if inbrowser: | |
launch_kwargs["inbrowser"] = inbrowser | |
if share: | |
launch_kwargs["share"] = share | |
interface.queue().launch(**launch_kwargs, max_threads=1) | |
if __name__ == "__main__": | |
parser = argparse.ArgumentParser() | |
parser.add_argument( | |
"--listen", | |
type=str, | |
default="0.0.0.0", | |
help="IP to listen on for connections to Gradio", | |
) | |
parser.add_argument( | |
"--username", type=str, default="", help="Username for authentication" | |
) | |
parser.add_argument( | |
"--password", type=str, default="", help="Password for authentication" | |
) | |
parser.add_argument( | |
"--server_port", | |
type=int, | |
default=7860, | |
help="Port to run the server listener on", | |
) | |
parser.add_argument("--inbrowser", action="store_true", help="Open in browser") | |
parser.add_argument("--share", action="store_true", help="Share the gradio UI") | |
args = parser.parse_args() | |
ui( | |
username=args.username, | |
password=args.password, | |
inbrowser=args.inbrowser, | |
server_port=args.server_port, | |
share=args.share, | |
listen=args.listen, | |
) | |