import binascii import os import gradio as gr import librosa import numpy as np import pretty_midi import torch import yt_dlp from transformers import Pop2PianoForConditionalGeneration, Pop2PianoProcessor from utils import cli_to_api, mp3_write, normalize yt_video_dir = "./yt_dir" outputs_dir = "./midi_wav_outputs" os.makedirs(outputs_dir, exist_ok=True) os.makedirs(yt_video_dir, exist_ok=True) device = "cuda" if torch.cuda.is_available() else "cpu" model = Pop2PianoForConditionalGeneration.from_pretrained("sweetcocoa/pop2piano").to(device) processor = Pop2PianoProcessor.from_pretrained("sweetcocoa/pop2piano") composers = model.generation_config.composer_to_feature_token.keys() def get_audio_from_yt_video(yt_link: str): filename = binascii.hexlify(os.urandom(8)).decode() + ".mp3" filename = os.path.join(yt_video_dir, filename) yt_opt = cli_to_api( [ "--extract-audio", "--audio-format", "mp3", "--restrict-filenames", "-o", filename, ] ) with yt_dlp.YoutubeDL(yt_opt) as ydl: ydl.download([yt_link]) return filename, filename def inference(file_uploaded, composer): # to save the native sampling rate of the file, sr=None is used, but this can cause some silent errors where the # generated output will not be upto the desired quality. If that happens please consider switching sr to 44100 Hz. pop_y, sr = librosa.load(file_uploaded, sr=None) inputs = processor(audio=pop_y, sampling_rate=sr, return_tensors="pt").to(device) model_output = model.generate(input_features=inputs["input_features"], composer=composer) tokenizer_output = processor.batch_decode( token_ids=model_output.to("cpu"), feature_extractor_output=inputs.to("cpu") )["pretty_midi_objects"] return prepare_output_file(tokenizer_output, sr, pop_y) def prepare_output_file(tokenizer_output: pretty_midi.PrettyMIDI, sr: int, pop_y: np.ndarray): # Add some random values so that no two file names are same output_file_name = "p2p_" + binascii.hexlify(os.urandom(8)).decode() midi_output = os.path.join(outputs_dir, output_file_name + ".mid") # write the .mid and its wav files tokenizer_output[0].write(midi_output) midi_y: np.ndarray = tokenizer_output[0].fluidsynth(sr) midi_y_path: str = midi_output.replace(".mid", ".mp3") mp3_write(midi_y_path, sr, normalize(midi_y), normalized=True) # stack stereo audio if len(pop_y) > len(midi_y): midi_y = np.pad(midi_y, (0, len(pop_y) - len(midi_y))) elif len(pop_y) < len(midi_y): pop_y = np.pad(pop_y, (0, -len(pop_y) + len(midi_y))) stereo = np.stack((midi_y, pop_y * 0.5)) # write stereo audio stereo_path = midi_output.replace(".mid", ".mix.mp3") mp3_write(stereo_path, sr, normalize(stereo.T), normalized=True) return midi_y_path, midi_y_path, midi_output, stereo_path, stereo_path block = gr.Blocks() with block: gr.HTML( """
A demo for Pop2Piano:Pop Audio-based Piano Cover Generation.
Please select the composer(Arranger) and upload the pop audio or enter the YouTube link and then click Generate.