fffiloni's picture
Update app.py
57f1b5a verified
raw
history blame
4.41 kB
import gradio as gr
from gradio_client import Client
import json
import re
def get_caption(image_in):
kosmos2_client = Client("https://ydshieh-kosmos-2.hf.space/")
kosmos2_result = kosmos2_client.predict(
image_in, # str (filepath or URL to image) in 'Test Image' Image component
"Detailed", # str in 'Description Type' Radio component
fn_index=4
)
print(f"KOSMOS2 RETURNS: {kosmos2_result}")
with open(kosmos2_result[1], 'r') as f:
data = json.load(f)
reconstructed_sentence = []
for sublist in data:
reconstructed_sentence.append(sublist[0])
full_sentence = ' '.join(reconstructed_sentence)
#print(full_sentence)
# Find the pattern matching the expected format ("Describe this image in detail:" followed by optional space and then the rest)...
pattern = r'^Describe this image in detail:\s*(.*)$'
# Apply the regex pattern to extract the description text.
match = re.search(pattern, full_sentence)
if match:
description = match.group(1)
print(description)
else:
print("Unable to locate valid description.")
# Find the last occurrence of "."
#last_period_index = full_sentence.rfind('.')
# Truncate the string up to the last period
#truncated_caption = full_sentence[:last_period_index + 1]
# print(truncated_caption)
#print(f"\n—\nIMAGE CAPTION: {truncated_caption}")
return description
def get_magnet(prompt):
amended_prompt = f"No Music. {prompt}"
print(amended_prompt)
client = Client("https://fffiloni-magnet.hf.space/--replicas/oo8sb/")
result = client.predict(
"facebook/audio-magnet-small", # Literal['facebook/magnet-small-10secs', 'facebook/magnet-medium-10secs', 'facebook/magnet-small-30secs', 'facebook/magnet-medium-30secs', 'facebook/audio-magnet-small', 'facebook/audio-magnet-medium'] in 'Model' Radio component
"", # str in 'Model Path (custom models)' Textbox component
amended_prompt, # str in 'Input Text' Textbox component
3, # float in 'Temperature' Number component
0.9, # float in 'Top-p' Number component
10, # float in 'Max CFG coefficient' Number component
1, # float in 'Min CFG coefficient' Number component
20, # float in 'Decoding Steps (stage 1)' Number component
10, # float in 'Decoding Steps (stage 2)' Number component
10, # float in 'Decoding Steps (stage 3)' Number component
10, # float in 'Decoding Steps (stage 4)' Number component
"prod-stride1 (new!)", # Literal['max-nonoverlap', 'prod-stride1 (new!)'] in 'Span Scoring' Radio component
api_name="/predict_full"
)
print(result)
return result[0]['video']
def get_audioldm(prompt):
client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
result = client.predict(
prompt, # str in 'Input text' Textbox component
"Low quality. Music.", # str in 'Negative prompt' Textbox component
10, # int | float (numeric value between 5 and 15) in 'Duration (seconds)' Slider component
3.5, # int | float (numeric value between 0 and 7) in 'Guidance scale' Slider component
45, # int | float in 'Seed' Number component
3, # int | float (numeric value between 1 and 5) in 'Number waveforms to generate' Slider component
fn_index=1
)
print(result)
return result
def infer(image_in):
caption = get_caption(image_in)
magnet_result = get_magnet(caption)
audioldm_result = get_audioldm(caption)
return magnet_result, audioldm_result
with gr.Blocks() as demo:
with gr.Column():
gr.HTML("""
<h2 style="text-align: center;">
Image to SFX
</h2>
<p style="text-align: center;">
Compare MAGNet and AudioLDM2 sound effects generation from image caption (Kosmos2)
</p>
""")
with gr.Column():
image_in = gr.Image(sources=["upload"], type="filepath", label="Image input")
submit_btn = gr.Button("Submit")
with gr.Row():
magnet_o = gr.Video(label="MAGNet output")
audioldm2_o = gr.Video(label="AudioLDM2 output")
submit_btn.click(
fn=infer,
inputs=[image_in],
outputs=[magnet_o, audioldm2_o]
)
demo.queue(max_size=10).launch(debug=True)