import os
import sys
sys.path.append("../")
import gradio as gr
import torch
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams["font.family"] = "serif"
import decord
import PIL, PIL.Image
import librosa
from IPython.display import Markdown, display
import pandas as pd
from util import *
css = """
"""
header = css + """
"""
footer = css + """
"""
from download_youtube import download_youtube_video_ytdlp
def download_from_youtube(
video_id,
save_dir="/tmp/",
convert_to_mp4=False,
):
"""
Downloads a YouTube video from start to end times.
Args:
video_id (str): YouTube video ID.
save_dir (str): Directory to save the video.
convert_to_mp4 (bool): Whether to convert the video to mp4 format.
The saved video is in the format: {save_dir}/{video_id}.mp4
"""
import datetime
from subprocess import call
print("Downloading video from YouTube...")
print("Video ID:", video_id)
command = [
"yt-dlp",
"-o", "'{}%(id)s.%(ext)s'".format(save_dir),
"--cookies ./youtube_cookies.txt",
"--verbose",
"--force-overwrites",
f"https://www.youtube.com/watch?v={video_id}",
]
try:
call(" ".join(command), shell=True)
except Exception as e:
print(e)
raise IOError("Failed to download to download YouTube video.")
# If not mp4, convert to mp4
from glob import glob
saved_filepath = glob(os.path.join(save_dir, f"{video_id}.*"))[0]
print("Saved file:", saved_filepath)
if convert_to_mp4:
ext = saved_filepath.split(".")[-1]
to_save = saved_filepath.replace(ext, "mp4")
if ext != "mp4":
# convert to mp4 using ffmpeg
command = "ffmpeg -y -i {} {}".format(saved_filepath, to_save)
call(command, shell=True)
return to_save
else:
return saved_filepath
def configure_input():
gr.Markdown(
"#### Either upload a video file or provide a YouTube link to a video. Note that the entire video shall be used.",
)
video_input = gr.Video(label="Upload Video", height=520)
youtube_link = gr.Textbox(label="YouTube Link", value=None)
gr.Markdown(
"Note: Often, YouTube download can fail because the video may not be public or YouTube asks for Sign in."\
"We recommend downloading the video in other ways on your machine and uploading it here."\
" Alternatively, you can clone the repository and run the demo locally which can allow for Sign-in.",
)
return [video_input, youtube_link]
# video_backend = "decord"
video_backend = "torchvision"
def get_predictions(video_path):
model = load_model()
frame = load_frame(video_path, video_backend=video_backend)
S = load_spectrogram(video_path)
audio = load_audio_tensor(video_path)
z_audio, y_audio = get_model_output(audio, model)
image, df_show, tsne_image = show_output(frame, S, y_audio, z_audio)
return image, df_show, tsne_image
def get_video_id_from_url(url):
import re
if "v=" in url:
video_id = re.findall(r"v=([a-zA-Z0-9_-]+)", url)
elif "youtu.be" in url:
video_id = re.findall(r"youtu.be/([a-zA-Z0-9_-]+)", url)
elif "shorts" in url:
video_id = re.findall(r"shorts/([a-zA-Z0-9_-]+)", url)
else:
raise ValueError("Invalid YouTube URL")
print("Video URL:", url)
print("Video ID:", video_id)
if len(video_id) > 0:
return video_id[0]
else:
raise ValueError("Invalid YouTube URL")
note = """
**Note**: Radius (as well as height) estimation depends on accurate wavelength estimation towards the end.
Thus, it may not be accurate if the wavelength is not estimated correctly at the end.
$$
H = l(0) = \\frac{\lambda(0) - \lambda(T)}{4} \ \ \\text{and} \ \ R = \\frac{\lambda(T)}{4\\beta}
$$
"""
# Example usage in a Gradio interface
def process_input(video, youtube_link):
provided_video = video is not None
if youtube_link is None:
provided_link = False
elif isinstance(youtube_link, str):
provided_link = len(youtube_link) > 0
else:
raise ValueError(f"Invalid type of link {youtube_link}.")
if provided_video and provided_link:
raise ValueError("Please provide either a video file or a YouTube link, not both.")
if provided_video:
print(video)
# # Load model globally
# model = load_model()
# The input is a video file path
video_path = video
# Get predictions
image, df_show, tsne_image = get_predictions(video_path)
return image, df_show, gr.Markdown(note), tsne_image
else:
print(provided_link)
assert provided_link, \
"YouTube Link cannot be empty if no video is provided."
video_id = get_video_id_from_url(youtube_link)
print("Video ID:", video_id)
video_path = download_youtube_video_ytdlp(
video_id, save_dir="/tmp/",
)
# Get predictions
image, df_show, tsne_image = get_predictions(video_path)
# Add youtube link to the note
local_note = f"{note}\n\nYou can watch the original video here: "\
f"[YouTube Link](https://www.youtube.com/watch?v={video_id})"
return image, df_show, gr.Markdown(local_note), tsne_image
def configure_outputs():
image_wide = gr.Image(label="Estimated pitch")
dataframe = gr.DataFrame(label="Estimated physical properties")
image_tsne = gr.Image(label="TSNE of features", width=300)
markdown = gr.Markdown(label="Note")
return [image_wide, dataframe, markdown, image_tsne]
# Configure pre-defined examples
examples = [
["./media_assets/example_video.mp4", None],
["./media_assets/ayNzH0uygFw_9.0_21.0.mp4", None],
["./media_assets/biDn0Gi6V8U_7.0_15.0.mp4", None],
["./media_assets/goWgiQQMugA_2.5_9.0.mp4", None],
["./media_assets/K87g4RvO-9k_254.0_259.0.mp4", None],
# Shows that it works with background noise
["./media_assets/l74zJHCZ9uA.webm", None],
# Shows that it works with a slightly differently shaped container
["./media_assets/LpRPV0hIymU.webm", None],
["./media_assets/k-HnMsS36J8.webm", None],
# [None, "https://www.youtube.com/shorts/6eUQTdkTooo"],
# [None, "https://www.youtube.com/shorts/VxZT15cG6tw"],
# [None, "https://www.youtube.com/shorts/GSXQnNhliDY"],
]
# Define Gradio interface
with gr.Blocks(
css=custom_css,
theme=gr.themes.Default(),
) as demo:
# Add the header
gr.HTML(header)
gr.Interface(
fn=process_input,
inputs=configure_input(),
outputs=configure_outputs(),
examples=examples,
)
# Add the footer
gr.HTML(footer)
# Launch the interface
demo.launch(allowed_paths=["."], share=True)
Code