import gradio as gr import os hf_token = os.environ.get('HF_TOKEN') lpmc_client = gr.load("seungheondoh/LP-Music-Caps-demo", src="spaces") from gradio_client import Client client = Client("https://fffiloni-test-llama-api.hf.space/", hf_token=hf_token) lyrics_client = Client("https://fffiloni-music-to-lyrics.hf.space/") from share_btn import community_icon_html, loading_icon_html, share_js from compel import Compel, ReturnedEmbeddingsType from diffusers import DiffusionPipeline import torch pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, use_safetensors=True, variant="fp16") pipe.to("cuda") compel = Compel( tokenizer=[pipe.tokenizer, pipe.tokenizer_2], text_encoder=[pipe.text_encoder, pipe.text_encoder_2], returned_embeddings_type=ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED, requires_pooled=[False, True] ) #pipe.enable_model_cpu_offload() # if using torch < 2.0 # pipe.enable_xformers_memory_efficient_attention() from pydub import AudioSegment def cut_audio(input_path, output_path, max_duration=30000): audio = AudioSegment.from_file(input_path) if len(audio) > max_duration: audio = audio[:max_duration] audio.export(output_path, format="mp3") return output_path def get_text_after_colon(input_text): # Find the first occurrence of ":" colon_index = input_text.find(":") # Check if ":" exists in the input_text if colon_index != -1: # Extract the text after the colon result_text = input_text[colon_index + 1:].strip() return result_text else: # Return the original text if ":" is not found return input_text def solo_xd(prompt): images = pipe(prompt=prompt).images[0] return images def infer(audio_file, has_lyrics): print("NEW INFERENCE ...") truncated_audio = cut_audio(audio_file, "trunc_audio.mp3") print("Calling LP Music Caps...") cap_result = lpmc_client( truncated_audio, # str (filepath or URL to file) in 'audio_path' Audio component api_name="predict" ) print(f"MUSIC DESC: {cap_result}") if has_lyrics == "Yes" : print("""——— Getting Lyrics ... """) lyrics_result = lyrics_client.predict( audio_file, # str (filepath or URL to file) in 'Song input' Audio component fn_index=0 ) print(f"LYRICS: {lyrics_result}") llama_q = f""" I'll give you a music description + the lyrics of the song. Give me an image description that would fit well with the music description, reflecting the lyrics too. Be creative, do not do list, just an image description as required. Try to think about human characters first. Your image description must fit well for a stable diffusion prompt. Here's the music description : « {cap_result} » And here are the lyrics : « {lyrics_result} » """ elif has_lyrics == "No" : llama_q = f""" I'll give you a music description. Give me an image description that would fit well with the music description. Be creative, do not do list, just an image description as required. Try to think about human characters first. Your image description must fit well for a stable diffusion prompt. Here's the music description : « {cap_result} » """ print("""——— Calling Llama2 ... """) result = client.predict( llama_q, # str in 'Message' Textbox component api_name="/predict" ) result = get_text_after_colon(result) print(f"Llama2 result: {result}") # ——— print("""——— Calling SD-XL ... """) prompt = result conditioning, pooled = compel(prompt) images = pipe(prompt_embeds=conditioning, pooled_prompt_embeds=pooled).images[0] print("Finished") #return cap_result, result, images return images, result, gr.update(visible=True), gr.Group.update(visible=True) css = """ #col-container {max-width: 780px; margin-left: auto; margin-right: auto;} a {text-decoration-line: underline; font-weight: 600;} .animate-spin { animation: spin 1s linear infinite; } @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } #share-btn-container { display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; max-width: 13rem; } div#share-btn-container > div { flex-direction: row; background: black; align-items: center; } #share-btn-container:hover { background-color: #060606; } #share-btn { all: initial; color: #ffffff; font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.5rem !important; padding-bottom: 0.5rem !important; right:0; } #share-btn * { all: unset; } #share-btn-container div:nth-child(-n+2){ width: auto !important; min-height: 0px !important; } #share-btn-container .wrap { display: none !important; } #share-btn-container.hidden { display: none!important; } .footer { margin-bottom: 45px; margin-top: 10px; text-align: center; border-bottom: 1px solid #e5e5e5; } .footer>p { font-size: .8rem; display: inline-block; padding: 0 10px; transform: translateY(10px); background: white; } .dark .footer { border-color: #303030; } .dark .footer>p { background: #0b0f19; } """ with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.HTML("""

Music To Image

Sends an audio into LP-Music-Caps to generate a audio caption which is then translated to an illustrative image description with Llama2, and finally run through Stable Diffusion XL to generate an image from the audio !

Note: Only the first 30 seconds of your audio will be used for inference.

""") audio_input = gr.Audio(label="Music input", type="filepath", source="upload") with gr.Row(): has_lyrics = gr.Radio(label="Does your audio has lyrics ?", choices=["Yes", "No"], value="No", info="If yes, the image should reflect the lyrics, but be aware that because we add a step (getting lyrics), inference will take more time.") song_title = gr.Textbox(label="Song Title", value="Title: ", interactive=True, info="If you want to share your result, please provide the title of your audio sample :)", elem_id="song-title") infer_btn = gr.Button("Generate Image from Music") #lpmc_cap = gr.Textbox(label="Lp Music Caps caption") with gr.Row(): llama_trans_cap = gr.Textbox(label="Llama Image Suggestion", placeholder="Llama2 image prompt suggestion will be displayed here ;)", visible=True, lines=12, elem_id="llama-prompt") img_result = gr.Image(label="Image Result", elem_id="image-out") with gr.Row(): tryagain_btn = gr.Button("Try another image ?", visible=False) with gr.Group(elem_id="share-btn-container", visible=False) as share_group: community_icon = gr.HTML(community_icon_html) loading_icon = gr.HTML(loading_icon_html) share_button = gr.Button("Share to community", elem_id="share-btn") gr.Examples(examples=[["./examples/electronic.mp3", "No"],["./examples/folk.wav", "No"], ["./examples/orchestra.wav", "No"]], fn=infer, inputs=[audio_input, has_lyrics], outputs=[img_result, llama_trans_cap, tryagain_btn, share_group], cache_examples=True ) gr.HTML("""

Music to Image Demo by 🤗 Sylvain Filoni

You may also like:

""") #infer_btn.click(fn=infer, inputs=[audio_input], outputs=[lpmc_cap, llama_trans_cap, img_result]) infer_btn.click(fn=infer, inputs=[audio_input, has_lyrics], outputs=[img_result, llama_trans_cap, tryagain_btn, share_group]) share_button.click(None, [], [], _js=share_js) tryagain_btn.click(fn=solo_xd, inputs=[llama_trans_cap], outputs=[img_result]) demo.queue(max_size=20).launch()