Upload 7 files

app_canny_db.py

@@ -0,0 +1,103 @@
+import gradio as gr
+from model import Model
+import gradio_utils
+import os
+on_huggingspace = os.environ.get("SPACE_AUTHOR_NAME") == "PAIR"
+
+
+examples = [
+    ['Anime DB', "woman1", "Portrait of detailed 1girl, feminine, soldier cinematic shot on canon 5d ultra realistic skin intricate clothes accurate hands Rory Lewis Artgerm WLOP Jeremy Lipking Jane Ansell studio lighting"],
+    ['Arcane DB', "woman1", "Oil painting of a beautiful girl arcane style, masterpiece, a high-quality, detailed, and professional photo"],
+    ['GTA-5 DB', "man1", "gtav style"],
+    ['GTA-5 DB', "woman3", "gtav style"],
+    ['Avatar DB', "woman2", "oil painting of a beautiful girl avatar style"],
+]
+
+
+def load_db_model(evt: gr.SelectData):
+    db_name = gradio_utils.get_db_name_from_id(evt.index)
+    return db_name
+
+
+def canny_select(evt: gr.SelectData):
+    canny_name = gradio_utils.get_canny_name_from_id(evt.index)
+    return canny_name
+
+
+def create_demo(model: Model):
+
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown(
+                '## Text, Canny-Edge and DreamBooth Conditional Video Generation')
+        with gr.Row():
+            gr.HTML(
+                """
+                <div style="text-align: left; auto;">
+                <h2 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+                    Description: Our current release supports only four predefined DreamBooth models and four "motion edges". So you must choose one DreamBooth model and one "motion edges" shown below, or use the examples. The keywords <b>1girl</b>, <b>arcane style</b>, <b>gtav</b>, and <b>avatar style</b> correspond to the models from left to right.
+                </h3>
+                </div>
+                """)
+        with gr.Row():
+            with gr.Column():
+                # input_video_path = gr.Video(source='upload', format="mp4", visible=False)
+                gr.Markdown("## Selection")
+                db_text_field = gr.Markdown('DB Model: **Anime DB** ')
+                canny_text_field = gr.Markdown('Motion: **woman1**')
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+                with gr.Accordion('Advanced options', open=False):
+                    watermark = gr.Radio(["Picsart AI Research", "Text2Video-Zero",
+                                         "None"], label="Watermark", value='Picsart AI Research')
+                    chunk_size = gr.Slider(
+                        label="Chunk size", minimum=2, maximum=16, value=12 if on_huggingspace else 8, step=1, visible=not on_huggingspace)
+            with gr.Column():
+                result = gr.Image(label="Generated Video").style(height=400)
+
+        with gr.Row():
+            gallery_db = gr.Gallery(label="Db models", value=[('__assets__/db_files/anime.jpg', "anime"), ('__assets__/db_files/arcane.jpg', "Arcane"), (
+                '__assets__/db_files/gta.jpg', "GTA-5 (Man)"), ('__assets__/db_files/avatar.jpg', "Avatar DB")]).style(grid=[4], height=50)
+        with gr.Row():
+            gallery_canny = gr.Gallery(label="Motions", value=[('__assets__/db_files/woman1.gif', "woman1"), ('__assets__/db_files/woman2.gif', "woman2"), (
+                '__assets__/db_files/man1.gif', "man1"), ('__assets__/db_files/woman3.gif', "woman3")]).style(grid=[4], height=50)
+
+        db_selection = gr.Textbox(label="DB Model", visible=False)
+        canny_selection = gr.Textbox(
+            label="One of the above defined motions", visible=False)
+
+        gallery_db.select(load_db_model, None, db_selection)
+        gallery_canny.select(canny_select, None, canny_selection)
+
+        db_selection.change(on_db_selection_update, None, db_text_field)
+        canny_selection.change(on_canny_selection_update,
+                               None, canny_text_field)
+
+        inputs = [
+            db_selection,
+            canny_selection,
+            prompt,
+            chunk_size,
+            watermark,
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    fn=model.process_controlnet_canny_db,
+                    cache_examples=on_huggingspace,
+                    )
+
+        run_button.click(fn=model.process_controlnet_canny_db,
+                         inputs=inputs,
+                         outputs=result,)
+    return demo
+
+
+def on_db_selection_update(evt: gr.EventData):
+
+    return f"DB model: **{evt._data}**"
+
+
+def on_canny_selection_update(evt: gr.EventData):
+    return f"Motion: **{evt._data}**"

app_text_to_video.py

@@ -0,0 +1,97 @@
+import gradio as gr
+from model import Model
+import os
+from hf_utils import get_model_list
+
+on_huggingspace = os.environ.get("SPACE_AUTHOR_NAME") == "PAIR"
+
+examples = [
+    ["an astronaut waving the arm on the moon"],
+    ["a sloth surfing on a wakeboard"],
+    ["an astronaut walking on a street"],
+    ["a cute cat walking on grass"],
+    ["a horse is galloping on a street"],
+    ["an astronaut is skiing down the hill"],
+    ["a gorilla walking alone down the street"],
+    ["a gorilla dancing on times square"],
+    ["A panda dancing dancing like crazy on Times Square"],
+]
+
+
+def create_demo(model: Model):
+
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown('## Text2Video-Zero: Video Generation')
+        with gr.Row():
+            gr.HTML(
+                """
+                <div style="text-align: left; auto;">
+                <h2 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+                    Description: Simply input <b>any textual prompt</b> to generate videos right away and unleash your creativity and imagination! You can also select from the examples below. For performance purposes, our current preview release allows to generate up to 16 frames, which can be configured in the Advanced Options.
+                </h3>
+                </div>
+                """)
+
+        with gr.Row():
+            with gr.Column():
+                model_name = gr.Dropdown(
+                    label="Model",
+                    choices=get_model_list(),
+                    value="dreamlike-art/dreamlike-photoreal-2.0",
+                )
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+                with gr.Accordion('Advanced options', open=False):
+                    watermark = gr.Radio(["Picsart AI Research", "Text2Video-Zero",
+                                         "None"], label="Watermark", value='Picsart AI Research')
+
+                    if on_huggingspace:
+                        video_length = gr.Slider(
+                            label="Video length", minimum=8, maximum=16, step=1)
+                    else:
+                        video_length = gr.Number(
+                            label="Video length", value=8, precision=0)
+                    chunk_size = gr.Slider(
+                        label="Chunk size", minimum=2, maximum=16, value=12 if on_huggingspace else 8, step=1, visible=not on_huggingspace)
+
+                    motion_field_strength_x = gr.Slider(
+                        label='Global Translation $\delta_{x}$', minimum=-20, maximum=20, value=12, step=1)
+                    motion_field_strength_y = gr.Slider(
+                        label='Global Translation $\delta_{y}$', minimum=-20, maximum=20, value=12, step=1)
+
+                    t0 = gr.Slider(label="Timestep t0", minimum=0,
+                                   maximum=49, value=44, step=1)
+                    t1 = gr.Slider(label="Timestep t1", minimum=0,
+                                   maximum=49, value=47, step=1)
+
+                    n_prompt = gr.Textbox(
+                        label="Optional Negative Prompt", value='')
+            with gr.Column():
+                result = gr.Video(label="Generated Video")
+
+        inputs = [
+            prompt,
+            model_name,
+            motion_field_strength_x,
+            motion_field_strength_y,
+            t0,
+            t1,
+            n_prompt,
+            chunk_size,
+            video_length,
+            watermark,
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    fn=model.process_text2video,
+                    run_on_click=False,
+                    cache_examples=on_huggingspace,
+                    )
+
+        run_button.click(fn=model.process_text2video,
+                         inputs=inputs,
+                         outputs=result,)
+    return demo

config.py

@@ -0,0 +1 @@
+save_memory = False

gradio_utils.py

@@ -0,0 +1,98 @@
+import os
+
+# App Canny utils
+
+
+def edge_path_to_video_path(edge_path):
+    video_path = edge_path
+
+    vid_name = edge_path.split("/")[-1]
+    if vid_name == "butterfly.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/butterfly.mp4"
+    elif vid_name == "deer.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/deer.mp4"
+    elif vid_name == "fox.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/fox.mp4"
+    elif vid_name == "girl_dancing.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/girl_dancing.mp4"
+    elif vid_name == "girl_turning.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/girl_turning.mp4"
+    elif vid_name == "halloween.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/halloween.mp4"
+    elif vid_name == "santa.mp4":
+        video_path = "__assets__/canny_videos_mp4_2fps/santa.mp4"
+
+    assert os.path.isfile(video_path)
+    return video_path
+
+
+# App Pose utils
+def motion_to_video_path(motion):
+    videos = [
+        "__assets__/poses_skeleton_gifs/dance1_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance2_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance3_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance4_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance5_corr.mp4"
+    ]
+    if len(motion.split(" ")) > 1 and motion.split(" ")[1].isnumeric():
+        id = int(motion.split(" ")[1]) - 1
+        return videos[id]
+    else:
+        return motion
+
+
+# App Canny Dreambooth utils
+def get_video_from_canny_selection(canny_selection):
+    if canny_selection == "woman1":
+        input_video_path = "__assets__/db_files_2fps/woman1.mp4"
+
+    elif canny_selection == "woman2":
+        input_video_path = "__assets__/db_files_2fps/woman2.mp4"
+
+    elif canny_selection == "man1":
+        input_video_path = "__assets__/db_files_2fps/man1.mp4"
+
+    elif canny_selection == "woman3":
+        input_video_path = "__assets__/db_files_2fps/woman3.mp4"
+    else:
+        input_video_path = canny_selection
+
+    assert os.path.isfile(input_video_path)
+    return input_video_path
+
+
+def get_model_from_db_selection(db_selection):
+    if db_selection == "Anime DB":
+        input_video_path = 'PAIR/text2video-zero-controlnet-canny-anime'
+    elif db_selection == "Avatar DB":
+        input_video_path = 'PAIR/text2video-zero-controlnet-canny-avatar'
+    elif db_selection == "GTA-5 DB":
+        input_video_path = 'PAIR/text2video-zero-controlnet-canny-gta5'
+    elif db_selection == "Arcane DB":
+        input_video_path = 'PAIR/text2video-zero-controlnet-canny-arcane'
+    else:
+        input_video_path = db_selection
+
+    return input_video_path
+
+
+def get_db_name_from_id(id):
+    db_names = ["Anime DB", "Arcane DB", "GTA-5 DB", "Avatar DB"]
+    return db_names[id]
+
+
+def get_canny_name_from_id(id):
+    canny_names = ["woman1", "woman2", "man1", "woman3"]
+    return canny_names[id]
+
+
+def logo_name_to_path(name):
+    logo_paths = {
+        'Picsart AI Research': '__assets__/pair_watermark.png',
+        'Text2Video-Zero': '__assets__/t2v-z_watermark.png',
+        'None': None
+    }
+    if name in logo_paths:
+        return logo_paths[name]
+    return name

hf_utils.py

@@ -0,0 +1,39 @@
+from bs4 import BeautifulSoup
+import requests
+
+
+def model_url_list():
+    url_list = []
+    for i in range(0, 5):
+        url_list.append(
+            f"https://huggingface.co/models?p={i}&sort=downloads&search=dreambooth")
+    return url_list
+
+
+def data_scraping(url_list):
+    model_list = []
+    for url in url_list:
+        response = requests.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+        div_class = 'grid grid-cols-1 gap-5 2xl:grid-cols-2'
+        div = soup.find('div', {'class': div_class})
+        for a in div.find_all('a', href=True):
+            model_list.append(a['href'])
+    return model_list
+
+
+def get_model_list():
+    model_list = data_scraping(model_url_list())
+    for i in range(len(model_list)):
+        model_list[i] = model_list[i][1:]
+
+    best_model_list = [
+        "dreamlike-art/dreamlike-photoreal-2.0",
+        "dreamlike-art/dreamlike-diffusion-1.0",
+        "runwayml/stable-diffusion-v1-5",
+        "CompVis/stable-diffusion-v1-4",
+        "prompthero/openjourney",
+    ]
+
+    model_list = best_model_list + model_list
+    return model_list

style (1).css

@@ -0,0 +1,3 @@
+h1 {
+  text-align: center;
+}

utils (1).py

@@ -0,0 +1,207 @@
+import os
+
+import PIL.Image
+import numpy as np
+import torch
+import torchvision
+from torchvision.transforms import Resize, InterpolationMode
+import imageio
+from einops import rearrange
+import cv2
+from PIL import Image
+from annotator.util import resize_image, HWC3
+from annotator.canny import CannyDetector
+from annotator.openpose import OpenposeDetector
+import decord
+# decord.bridge.set_bridge('torch')
+
+apply_canny = CannyDetector()
+apply_openpose = OpenposeDetector()
+
+
+def add_watermark(image, watermark_path, wm_rel_size=1/16, boundary=5):
+    '''
+    Creates a watermark on the saved inference image.
+    We request that you do not remove this to properly assign credit to
+    Shi-Lab's work.
+    '''
+    watermark = Image.open(watermark_path)
+    w_0, h_0 = watermark.size
+    H, W, _ = image.shape
+    wmsize = int(max(H, W) * wm_rel_size)
+    aspect = h_0 / w_0
+    if aspect > 1.0:
+        watermark = watermark.resize((wmsize, int(aspect * wmsize)), Image.LANCZOS)
+    else:
+        watermark = watermark.resize((int(wmsize / aspect), wmsize), Image.LANCZOS)
+    w, h = watermark.size
+    loc_h = H - h - boundary
+    loc_w = W - w - boundary
+    image = Image.fromarray(image)
+    mask = watermark if watermark.mode in ('RGBA', 'LA') else None
+    image.paste(watermark, (loc_w, loc_h), mask)
+    return image
+
+
+def pre_process_canny(input_video, low_threshold=100, high_threshold=200):
+    detected_maps = []
+    for frame in input_video:
+        img = rearrange(frame, 'c h w -> h w c').cpu().numpy().astype(np.uint8)
+        detected_map = apply_canny(img, low_threshold, high_threshold)
+        detected_map = HWC3(detected_map)
+        detected_maps.append(detected_map[None])
+    detected_maps = np.concatenate(detected_maps)
+    control = torch.from_numpy(detected_maps.copy()).float() / 255.0
+    return rearrange(control, 'f h w c -> f c h w')
+
+
+def pre_process_pose(input_video, apply_pose_detect: bool = True):
+    detected_maps = []
+    for frame in input_video:
+        img = rearrange(frame, 'c h w -> h w c').cpu().numpy().astype(np.uint8)
+        img = HWC3(img)
+        if apply_pose_detect:
+            detected_map, _ = apply_openpose(img)
+        else:
+            detected_map = img
+        detected_map = HWC3(detected_map)
+        H, W, C = img.shape
+        detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_NEAREST)
+        detected_maps.append(detected_map[None])
+    detected_maps = np.concatenate(detected_maps)
+    control = torch.from_numpy(detected_maps.copy()).float() / 255.0
+    return rearrange(control, 'f h w c -> f c h w')
+
+
+def create_video(frames, fps, rescale=False, path=None, watermark=None):
+    if path is None:
+        dir = "temporal"
+        os.makedirs(dir, exist_ok=True)
+        path = os.path.join(dir, 'movie.mp4')
+
+    outputs = []
+    for i, x in enumerate(frames):
+        x = torchvision.utils.make_grid(torch.Tensor(x), nrow=4)
+        if rescale:
+            x = (x + 1.0) / 2.0  # -1,1 -> 0,1
+        x = (x * 255).numpy().astype(np.uint8)
+
+        if watermark is not None:
+            x = add_watermark(x, watermark)
+        outputs.append(x)
+        # imageio.imsave(os.path.join(dir, os.path.splitext(name)[0] + f'_{i}.jpg'), x)
+
+    imageio.mimsave(path, outputs, fps=fps)
+    return path
+
+def create_gif(frames, fps, rescale=False, path=None, watermark=None):
+    if path is None:
+        dir = "temporal"
+        os.makedirs(dir, exist_ok=True)
+        path = os.path.join(dir, 'canny_db.gif')
+
+    outputs = []
+    for i, x in enumerate(frames):
+        x = torchvision.utils.make_grid(torch.Tensor(x), nrow=4)
+        if rescale:
+            x = (x + 1.0) / 2.0  # -1,1 -> 0,1
+        x = (x * 255).numpy().astype(np.uint8)
+        if watermark is not None:
+            x = add_watermark(x, watermark)
+        outputs.append(x)
+        # imageio.imsave(os.path.join(dir, os.path.splitext(name)[0] + f'_{i}.jpg'), x)
+
+    imageio.mimsave(path, outputs, fps=fps)
+    return path
+
+def prepare_video(video_path:str, resolution:int, device, dtype, normalize=True, start_t:float=0, end_t:float=-1, output_fps:int=-1):
+    vr = decord.VideoReader(video_path)
+    initial_fps = vr.get_avg_fps()
+    if output_fps == -1:
+        output_fps = int(initial_fps)
+    if end_t == -1:
+        end_t = len(vr) / initial_fps
+    else:
+        end_t = min(len(vr) / initial_fps, end_t)
+    assert 0 <= start_t < end_t
+    assert output_fps > 0
+    start_f_ind = int(start_t * initial_fps)
+    end_f_ind = int(end_t * initial_fps)
+    num_f = int((end_t - start_t) * output_fps)
+    sample_idx = np.linspace(start_f_ind, end_f_ind, num_f, endpoint=False).astype(int)
+    video = vr.get_batch(sample_idx)
+    if torch.is_tensor(video):
+        video = video.detach().cpu().numpy()
+    else:
+        video = video.asnumpy()
+    _, h, w, _ = video.shape
+    video = rearrange(video, "f h w c -> f c h w")
+    video = torch.Tensor(video).to(device).to(dtype)
+    if h > w:
+        w = int(w * resolution / h)
+        w = w - w % 8
+        h = resolution - resolution % 8
+    else:
+        h = int(h * resolution / w)
+        h = h - h % 8
+        w = resolution - resolution % 8
+    video = Resize((h, w), interpolation=InterpolationMode.BILINEAR, antialias=True)(video)
+    if normalize:
+        video = video / 127.5 - 1.0
+    return video, output_fps
+
+
+def post_process_gif(list_of_results, image_resolution):
+    output_file = "/tmp/ddxk.gif"
+    imageio.mimsave(output_file, list_of_results, fps=4)
+    return output_file
+
+
+class CrossFrameAttnProcessor:
+    def __init__(self, unet_chunk_size=2):
+        self.unet_chunk_size = unet_chunk_size
+
+    def __call__(
+            self,
+            attn,
+            hidden_states,
+            encoder_hidden_states=None,
+            attention_mask=None):
+        batch_size, sequence_length, _ = hidden_states.shape
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+        query = attn.to_q(hidden_states)
+
+        is_cross_attention = encoder_hidden_states is not None
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.cross_attention_norm:
+            encoder_hidden_states = attn.norm_cross(encoder_hidden_states)
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+        # Sparse Attention
+        if not is_cross_attention:
+            video_length = key.size()[0] // self.unet_chunk_size
+            # former_frame_index = torch.arange(video_length) - 1
+            # former_frame_index[0] = 0
+            former_frame_index = [0] * video_length
+            key = rearrange(key, "(b f) d c -> b f d c", f=video_length)
+            key = key[:, former_frame_index]
+            key = rearrange(key, "b f d c -> (b f) d c")
+            value = rearrange(value, "(b f) d c -> b f d c", f=video_length)
+            value = value[:, former_frame_index]
+            value = rearrange(value, "b f d c -> (b f) d c")
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        return hidden_states