aded files without images

.gitignore

@@ -0,0 +1 @@
+*/.ipynb_checkpoints

README.md

@@ -1,13 +1,15 @@
 ---
 title: StreamMultiDiffusion
-emoji: 🌍
-colorFrom: blue
-colorTo: purple
+emoji: 🦦🦦🦦🦦
+colorFrom: #feecd6
+colorTo: #732a14
 sdk: gradio
-sdk_version: 4.21.0
+sdk_version: 4.26.0
 app_file: app.py
-pinned: false
+pinned: true
 license: mit
+models:
+- KBlueLeaf/kohaku-v2.1
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,1179 @@
+# Copyright (c) 2024 Jaerin Lee
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import sys
+
+sys.path.append('../../src')
+
+import argparse
+import random
+import time
+import json
+import os
+import glob
+import pathlib
+from functools import partial
+from pprint import pprint
+
+import numpy as np
+from PIL import Image
+import torch
+
+import gradio as gr
+from huggingface_hub import snapshot_download
+
+# from model import StreamMultiDiffusionSDXL
+from model import StreamMultiDiffusion
+from util import seed_everything
+from prompt_util import preprocess_prompts, _quality_dict, _style_dict
+
+
+### Utils
+
+
+
+
+def log_state(state):
+    pprint(vars(opt))
+    if isinstance(state, gr.State):
+        state = state.value
+    pprint(vars(state))
+
+
+def is_empty_image(im: Image.Image) -> bool:
+    if im is None:
+        return True
+    im = np.array(im)
+    has_alpha = (im.shape[2] == 4)
+    if not has_alpha:
+        return False
+    elif im.sum() == 0:
+        return True
+    else:
+        return False
+
+
+### Argument passing
+
+# parser = argparse.ArgumentParser(description='Semantic Palette demo powered by StreamMultiDiffusion with SDXL support.')
+# parser.add_argument('-H', '--height', type=int, default=1024)
+# parser.add_argument('-W', '--width', type=int, default=1024)
+parser = argparse.ArgumentParser(description='Semantic Palette demo powered by StreamMultiDiffusion.')
+parser.add_argument('-H', '--height', type=int, default=768)
+parser.add_argument('-W', '--width', type=int, default=768)
+parser.add_argument('--model', type=str, default=None, help='Hugging face model repository or local path for a SD1.5 model checkpoint to run.')
+parser.add_argument('--bootstrap_steps', type=int, default=1)
+parser.add_argument('--guidance_scale', type=float, default=0) # 1.2
+parser.add_argument('--run_time', type=float, default=60)
+parser.add_argument('--seed', type=int, default=-1)
+parser.add_argument('--device', type=int, default=0)
+parser.add_argument('--port', type=int, default=8000)
+opt = parser.parse_args()
+
+
+### Global variables and data structures
+
+device = f'cuda:{opt.device}' if opt.device >= 0 else 'cpu'
+
+
+if opt.model is None:
+    # opt.model = 'cagliostrolab/animagine-xl-3.1'
+    # opt.model = 'ironjr/BlazingDriveV11m'
+    opt.model = 'KBlueLeaf/kohaku-v2.1'
+else:
+    if opt.model.endswith('.safetensors'):
+        opt.model = os.path.abspath(os.path.join('checkpoints', opt.model))
+
+# model = StreamMultiDiffusionSDXL(
+model = StreamMultiDiffusion(
+    device,
+    hf_key=opt.model,
+    height=opt.height,
+    width=opt.width,
+    cfg_type="full",
+    autoflush=True,
+    use_tiny_vae=True,
+    mask_type='continuous',
+    bootstrap_steps=opt.bootstrap_steps,
+    bootstrap_mix_steps=opt.bootstrap_steps,
+    guidance_scale=opt.guidance_scale,
+    seed=opt.seed,
+)
+
+
+prompt_suggestions = [
+    '1girl, souryuu asuka langley, neon genesis evangelion, solo, upper body, v, smile, looking at viewer',
+    '1boy, solo, portrait, looking at viewer, white t-shirt, brown hair',
+    '1girl, arima kana, oshi no ko, solo, upper body, from behind',
+]
+
+opt.max_palettes = 3
+opt.default_prompt_strength = 1.0
+opt.default_mask_strength = 1.0
+opt.default_mask_std = 0.0
+opt.default_negative_prompt = (
+    'nsfw, worst quality, bad quality, normal quality, cropped, framed'
+)
+opt.verbose = True
+opt.colors = [
+    '#000000',
+    '#2692F3',
+    '#F89E12',
+    '#16C232',
+    # '#F92F6C',
+    # '#AC6AEB',
+    # '#92C62C',
+    # '#92C6EC',
+    # '#FECAC0',
+]
+
+
+### Event handlers
+
+def add_palette(state):
+    old_actives = state.active_palettes
+    state.active_palettes = min(state.active_palettes + 1, opt.max_palettes)
+
+    if opt.verbose:
+        log_state(state)
+
+    if state.active_palettes != old_actives:
+        return [state] + [
+            gr.update() if state.active_palettes != opt.max_palettes else gr.update(visible=False)
+        ] + [
+            gr.update() if i != state.active_palettes - 1 else gr.update(value=state.prompt_names[i + 1], visible=True)
+            for i in range(opt.max_palettes)
+        ]
+    else:
+        return [state] + [gr.update() for i in range(opt.max_palettes + 1)]
+
+
+def select_palette(state, button, idx):
+    if idx < 0 or idx > opt.max_palettes:
+        idx = 0
+    old_idx = state.current_palette
+    if old_idx == idx:
+        return [state] + [gr.update() for _ in range(opt.max_palettes + 7)]
+
+    state.current_palette = idx
+
+    if opt.verbose:
+        log_state(state)
+
+    updates = [state] + [
+        gr.update() if i not in (idx, old_idx) else
+        gr.update(variant='secondary') if i == old_idx else gr.update(variant='primary')
+        for i in range(opt.max_palettes + 1)
+    ]
+    label = 'Background' if idx == 0 else f'Palette {idx}'
+    updates.extend([
+        gr.update(value=button, interactive=(idx > 0)),
+        gr.update(value=state.prompts[idx], label=f'Edit Prompt for {label}'),
+        gr.update(value=state.neg_prompts[idx], label=f'Edit Negative Prompt for {label}'),
+        (
+            gr.update(value=state.mask_strengths[idx - 1], interactive=True) if idx > 0 else
+            gr.update(value=opt.default_mask_strength, interactive=False)
+        ),
+        (
+            gr.update(value=state.prompt_strengths[idx - 1], interactive=True) if idx > 0 else
+            gr.update(value=opt.default_prompt_strength, interactive=False)
+        ),
+        (
+            gr.update(value=state.mask_stds[idx - 1], interactive=True) if idx > 0 else
+            gr.update(value=opt.default_mask_std, interactive=False)
+        ),
+    ])
+    return updates
+
+
+def change_prompt_strength(state, strength):
+    if state.current_palette == 0:
+        return state
+
+    state.prompt_strengths[state.current_palette - 1] = strength
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+def change_std(state, std):
+    if state.current_palette == 0:
+        return state
+
+    state.mask_stds[state.current_palette - 1] = std
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+def change_mask_strength(state, strength):
+    if state.current_palette == 0:
+        return state
+
+    state.mask_strengths[state.current_palette - 1] = strength
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+def reset_seed(state, seed):
+    state.seed = seed
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+def rename_prompt(state, name):
+    state.prompt_names[state.current_palette] = name
+    if opt.verbose:
+        log_state(state)
+
+    return [state] + [
+        gr.update() if i != state.current_palette else gr.update(value=name)
+        for i in range(opt.max_palettes + 1)
+    ]
+
+
+def change_prompt(state, prompt):
+    state.prompts[state.current_palette] = prompt
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+def change_neg_prompt(state, neg_prompt):
+    state.neg_prompts[state.current_palette] = neg_prompt
+    if opt.verbose:
+        log_state(state)
+
+    return state
+
+
+# def select_style(state, style_name):
+#     state.style_name = style_name
+#     if opt.verbose:
+#         log_state(state)
+
+#     return state
+
+
+# def select_quality(state, quality_name):
+#     state.quality_name = quality_name
+#     if opt.verbose:
+#         log_state(state)
+
+#     return state
+
+
+def import_state(state, json_text):
+    current_palette = state.current_palette
+    # active_palettes = state.active_palettes
+    state_dict = json.loads(json_text)
+    for k in ('inpainting_mode', 'is_runing', 'active_palettes', 'current_palette'):
+        if k in state_dict:
+            del state_dict[k]
+    state = argparse.Namespace(**state_dict)
+    state.active_palettes = opt.max_palettes
+    return [state] + [
+        gr.update(value=v, visible=True) for v in state.prompt_names
+    ] + [
+        # state.style_name,
+        # state.quality_name,
+        state.prompts[current_palette],
+        state.prompt_names[current_palette],
+        state.neg_prompts[current_palette],
+        state.prompt_strengths[current_palette - 1],
+        state.mask_strengths[current_palette - 1],
+        state.mask_stds[current_palette - 1],
+        state.seed,
+    ]
+
+
+### Main worker
+
+def generate():
+    return model()
+
+
+def register(state, drawpad):
+    seed_everything(state.seed if state.seed >=0 else np.random.randint(2147483647))
+    print('Generate!')
+
+    background = drawpad['background'].convert('RGBA')
+    inpainting_mode = np.asarray(background).sum() != 0
+    if not inpainting_mode:
+        background = Image.new(size=(opt.width, opt.height), mode='RGB', color=(255, 255, 255))
+    print('Inpainting mode: ', inpainting_mode)
+
+    user_input = np.asarray(drawpad['layers'][0]) # (H, W, 4)
+    foreground_mask = torch.tensor(user_input[..., -1])[None, None] # (1, 1, H, W)
+    user_input = torch.tensor(user_input[..., :-1]) # (H, W, 3)
+
+    palette = torch.tensor([
+        tuple(int(s[i+1:i+3], 16) for i in (0, 2, 4))
+        for s in opt.colors[1:]
+    ]) # (N, 3)
+    masks = (palette[:, None, None, :] == user_input[None]).all(dim=-1)[:, None, ...] # (N, 1, H, W)
+    # has_masks = [i for i, m in enumerate(masks.sum(dim=(1, 2, 3)) == 0) if not m]
+    has_masks = list(range(opt.max_palettes))
+    print('Has mask: ', has_masks)
+    masks = masks * foreground_mask
+    masks = masks[has_masks]
+
+    # if inpainting_mode:
+    prompts = [state.prompts[v + 1] for v in has_masks]
+    negative_prompts = [state.neg_prompts[v + 1] for v in has_masks]
+    mask_strengths = [state.mask_strengths[v] for v in has_masks]
+    mask_stds = [state.mask_stds[v] for v in has_masks]
+    prompt_strengths = [state.prompt_strengths[v] for v in has_masks]
+    # else:
+        # masks = torch.cat([torch.ones_like(foreground_mask), masks], dim=0)
+        # prompts = [state.prompts[0]] + [state.prompts[v + 1] for v in has_masks]
+        # negative_prompts = [state.neg_prompts[0]] + [state.neg_prompts[v + 1] for v in has_masks]
+        # mask_strengths = [1] + [state.mask_strengths[v] for v in has_masks]
+        # mask_stds = [0] + [state.mask_stds[v] for v in has_masks]
+        # prompt_strengths = [1] + [state.prompt_strengths[v] for v in has_masks]
+
+    # prompts, negative_prompts = preprocess_prompts(
+    #     prompts, negative_prompts, style_name=state.style_name, quality_name=state.quality_name)
+
+    model.update_background(
+        background.convert('RGB'),
+        prompt=None,
+        negative_prompt=None,
+    )
+    state.prompts[0] = model.background.prompt
+    state.neg_prompts[0] = model.background.negative_prompt
+
+    model.update_layers(
+        prompts=prompts,
+        negative_prompts=negative_prompts,
+        masks=masks.to(device),
+        mask_strengths=mask_strengths,
+        mask_stds=mask_stds,
+        prompt_strengths=prompt_strengths,
+    )
+
+    state.inpainting_mode = inpainting_mode
+    return state
+
+
+def run(state, drawpad):
+    state = register(state, drawpad)
+    state.is_running = True
+
+    tic = time.time()
+    while True:
+        yield [state, generate()]
+        toc = time.time()
+        tdelta = toc - tic
+        if tdelta > opt.run_time:
+            state.is_running = False
+            return [state, generate()]
+
+
+def hide_element():
+    return gr.update(visible=False)
+
+
+def show_element():
+    return gr.update(visible=True)
+
+
+def draw(state, drawpad):
+    if not state.is_running:
+        return
+
+    user_input = np.asarray(drawpad['layers'][0]) # (H, W, 4)
+    foreground_mask = torch.tensor(user_input[..., -1])[None, None] # (1, 1, H, W)
+    user_input = torch.tensor(user_input[..., :-1]) # (H, W, 3)
+
+    palette = torch.tensor([
+        tuple(int(s[i+1:i+3], 16) for i in (0, 2, 4))
+        for s in opt.colors[1:]
+    ]) # (N, 3)
+    masks = (palette[:, None, None, :] == user_input[None]).all(dim=-1)[:, None, ...] # (N, 1, H, W)
+    # has_masks = [i for i, m in enumerate(masks.sum(dim=(1, 2, 3)) == 0) if not m]
+    has_masks = list(range(opt.max_palettes))
+    print('Has mask: ', has_masks)
+    masks = masks * foreground_mask
+    masks = masks[has_masks]
+
+    # if state.inpainting_mode:
+    mask_strengths = [state.mask_strengths[v] for v in has_masks]
+    mask_stds = [state.mask_stds[v] for v in has_masks]
+    # else:
+    #     masks = torch.cat([torch.ones_like(foreground_mask), masks], dim=0)
+    #     mask_strengths = [1] + [state.mask_strengths[v] for v in has_masks]
+    #     mask_stds = [0] + [state.mask_stds[v] for v in has_masks]
+
+    for i in range(len(has_masks)):
+        model.update_single_layer(
+            idx=i,
+            mask=masks[i],
+            mask_strength=mask_strengths[i],
+            mask_std=mask_stds[i],
+        )
+
+### Load examples
+
+
+root = pathlib.Path(__file__).parent
+print(root)
+example_root = os.path.join(root, 'examples')
+example_images = glob.glob(os.path.join(example_root, '*.png'))
+example_images = [Image.open(i) for i in example_images]
+
+# with open(os.path.join(example_root, 'prompt_background_advanced.txt')) as f:
+#     prompts_background = [l.strip() for l in f.readlines() if l.strip() != '']
+
+# with open(os.path.join(example_root, 'prompt_girl.txt')) as f:
+#     prompts_girl = [l.strip() for l in f.readlines() if l.strip() != '']
+
+# with open(os.path.join(example_root, 'prompt_boy.txt')) as f:
+#     prompts_boy = [l.strip() for l in f.readlines() if l.strip() != '']
+
+# with open(os.path.join(example_root, 'prompt_props.txt')) as f:
+#     prompts_props = [l.strip() for l in f.readlines() if l.strip() != '']
+#     prompts_props = {l.split(',')[0].strip(): ','.join(l.split(',')[1:]).strip() for l in prompts_props}
+
+# prompt_background = lambda: random.choice(prompts_background)
+# prompt_girl = lambda: random.choice(prompts_girl)
+# prompt_boy = lambda: random.choice(prompts_boy)
+# prompt_props = lambda: np.random.choice(list(prompts_props.keys()), size=(opt.max_palettes - 2), replace=False).tolist()
+
+
+### Main application
+
+css = f"""
+#run-button {{
+    font-size: 18pt;
+    background-image: linear-gradient(to right, #4338ca 0%, #26a0da 51%, #4338ca 100%);
+    margin: 0;
+    padding: 15px 45px;
+    text-align: center;
+//    text-transform: uppercase;
+    transition: 0.5s;
+    background-size: 200% auto;
+    color: white;
+    box-shadow: 0 0 20px #eee;
+    border-radius: 10px;
+//    display: block;
+    background-position: right center;
+}}
+
+#run-button:hover {{
+    background-position: left center;
+    color: #fff;
+    text-decoration: none;
+}}
+
+#run-anim {{
+    padding: 40px 45px;
+}}
+
+#semantic-palette {{
+    border-style: solid;
+    border-width: 0.2em;
+    border-color: #eee;
+}}
+
+#semantic-palette:hover {{
+    box-shadow: 0 0 20px #eee;
+}}
+
+#output-screen {{
+    width: 100%;
+    aspect-ratio: {opt.width} / {opt.height};
+}}
+
+.layer-wrap {{
+    display: none;
+}}
+"""
+
+for i in range(opt.max_palettes + 1):
+    css = css + f"""
+.secondary#semantic-palette-{i} {{
+    background-image: linear-gradient(to right, #374151 0%, #374151 71%, {opt.colors[i]} 100%);
+    color: white;
+}}
+
+.primary#semantic-palette-{i} {{
+    background-image: linear-gradient(to right, #4338ca 0%, #4338ca 71%, {opt.colors[i]} 100%);
+    color: white;
+}}
+"""
+
+css = css + f"""
+
+.mask-red {{
+    left: 0;
+    width: 0;
+    color: #BE002A;
+    -webkit-animation: text-red {opt.run_time:.1f}s ease infinite;
+            animation: text-red {opt.run_time:.1f}s ease infinite;
+    z-index: 2;
+    background: transparent;
+}}
+.mask-white {{
+    right: 0;
+}}
+
+/* Flames */
+
+#red-flame {{
+    opacity: 0;
+    -webkit-animation: show-flames {opt.run_time:.1f}s ease infinite, red-flame 120ms ease infinite;
+            animation: show-flames {opt.run_time:.1f}s ease infinite, red-flame 120ms ease infinite;
+    transform-origin: center bottom;
+}}
+
+#yellow-flame {{
+    opacity: 0;
+    -webkit-animation: show-flames {opt.run_time:.1f}s ease infinite, yellow-flame 120ms ease infinite;
+            animation: show-flames {opt.run_time:.1f}s ease infinite, yellow-flame 120ms ease infinite;
+    transform-origin: center bottom;
+}}
+
+#white-flame {{
+    opacity: 0;
+    -webkit-animation: show-flames {opt.run_time:.1f}s ease infinite, red-flame 100ms ease infinite;
+            animation: show-flames {opt.run_time:.1f}s ease infinite, red-flame 100ms ease infinite;
+    transform-origin: center bottom;
+}}
+"""
+
+with open(os.path.join(root, 'timer', 'style.css')) as f:
+    added_css = ''.join(f.readlines())
+css = css + added_css
+
+# js = ''
+
+# with open(os.path.join(root, 'timer', 'script.js')) as f:
+#     added_js = ''.join(f.readlines())
+# js = js + added_js
+
+head = f"""
+<link href='https://fonts.googleapis.com/css?family=Oswald' rel='stylesheet' type='text/css'>
+<script src='https://code.jquery.com/jquery-2.2.4.min.js'></script>
+"""
+
+
+with gr.Blocks(theme=gr.themes.Soft(), css=css, head=head) as demo:
+
+    iface = argparse.Namespace()
+
+    def _define_state():
+        state = argparse.Namespace()
+
+        # Cursor.
+        state.is_running = False
+        state.inpainting_mode = False
+        state.current_palette = 0 # 0: Background; 1,2,3,...: Layers
+        state.model_id = opt.model
+        state.style_name = '(None)'
+        state.quality_name = 'Standard v3.1'
+
+        # State variables (one-hot).
+        state.active_palettes = 5
+
+        # Front-end initialized to the default values.
+        # prompt_props_ = prompt_props()
+        state.prompt_names = [
+            '🌄 Background',
+            '👧 Girl',
+            '🐶 Dog',
+            '💐 Garden',
+        ] + ['🎨 New Palette' for _ in range(opt.max_palettes - 3)]
+        state.prompts = [
+            '',
+           'A girl smiling at viewer',
+            'Doggy body part',
+            'Flower garden',
+        ] + ['' for _ in range(opt.max_palettes - 3)]
+        state.neg_prompts = [
+            opt.default_negative_prompt
+            + (', humans, humans, humans' if i == 0 else '')
+            for i in range(opt.max_palettes + 1)
+        ]
+        state.prompt_strengths = [opt.default_prompt_strength for _ in range(opt.max_palettes)]
+        state.mask_strengths = [opt.default_mask_strength for _ in range(opt.max_palettes)]
+        state.mask_stds = [opt.default_mask_std for _ in range(opt.max_palettes)]
+        state.seed = opt.seed
+        return state
+
+    state = gr.State(value=_define_state)
+
+
+    ### Demo user interface
+
+    gr.HTML(
+        """
+<div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+    <div>
+        <h1>🦦🦦 StreamMultiDiffusion: Real-Time Interactive Generation with Region-Based Semantic Control 🦦🦦</h1>
+        <h5 style="margin: 0;">If you ❤️ our project, please visit our Github and give us a 🌟!</h5>
+        </br>
+        <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+            <a href='https://arxiv.org/abs/2403.09055'>
+                <img src="https://img.shields.io/badge/arXiv-2403.09055-red">
+            </a>
+            &nbsp;
+            <a href='https://jaerinlee.com/research/StreamMultiDiffusion'>
+                <img src='https://img.shields.io/badge/Project-Page-green' alt='Project Page'>
+            </a>
+            &nbsp;
+            <a href='https://github.com/ironjr/StreamMultiDiffusion'>
+                <img src='https://img.shields.io/github/stars/ironjr/StreamMultiDiffusion?label=Github&color=blue'>
+            </a>
+            &nbsp;
+            <a href='https://twitter.com/_ironjr_'>
+                <img src='https://img.shields.io/twitter/url?label=_ironjr_&url=https%3A%2F%2Ftwitter.com%2F_ironjr_'>
+            </a>
+            &nbsp;
+            <a href='https://github.com/ironjr/StreamMultiDiffusion/blob/main/LICENSE'>
+                <img src='https://img.shields.io/badge/license-MIT-lightgrey'>
+            </a>
+            &nbsp;
+            <a href='https://huggingface.co/papers/2403.09055'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Paper-StreamMultiDiffusion-yellow'>
+            </a>
+            &nbsp;
+            <a href='https://huggingface.co/spaces/ironjr/StreamMultiDiffusion'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-StreamMultiDiffusion-yellow'>
+            </a>
+            &nbsp;
+            <a href='https://huggingface.co/spaces/ironjr/SemanticPalette'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-SemanticPaletteSD1.5-yellow'>
+            </a>
+            &nbsp;
+            <a href='https://huggingface.co/spaces/ironjr/SemanticPaletteXL'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-SemanticPaletteSDXL-yellow'>
+            </a>
+            &nbsp;
+            <a href='https://colab.research.google.com/github/camenduru/SemanticPalette-jupyter/blob/main/SemanticPalette_jupyter.ipynb'>
+                <img src='https://colab.research.google.com/assets/colab-badge.svg'>
+            </a>
+        </div>
+    </div>
+</div>
+<div>
+    </br>
+</div>
+        """
+    )
+
+    with gr.Row():
+
+        with gr.Column(scale=1):
+
+            with gr.Group(elem_id='semantic-palette'):
+
+                gr.HTML(
+                    """
+<div style="justify-content: center; align-items: center;">
+    <br/>
+    <h3 style="margin: 0; text-align: center;"><b>🧠 Semantic Palette 🎨</b></h3>
+    <br/>
+</div>
+                    """
+                )
+
+                iface.btn_semantics = [gr.Button(
+                    value=state.value.prompt_names[0],
+                    variant='primary',
+                    elem_id='semantic-palette-0',
+                )]
+                for i in range(opt.max_palettes):
+                    iface.btn_semantics.append(gr.Button(
+                        value=state.value.prompt_names[i + 1],
+                        variant='secondary',
+                        visible=(i < state.value.active_palettes),
+                        elem_id=f'semantic-palette-{i + 1}'
+                    ))
+
+                iface.btn_add_palette = gr.Button(
+                    value='Create New Semantic Brush',
+                    variant='primary',
+                    visible=(state.value.active_palettes < opt.max_palettes),
+                )
+
+            with gr.Accordion(label='Import/Export Semantic Palette', open=True):
+                iface.tbox_state_import = gr.Textbox(label='Put Palette JSON Here To Import')
+                iface.json_state_export = gr.JSON(label='Exported Palette')
+                iface.btn_export_state = gr.Button("Export Palette ➡️ JSON", variant='primary')
+                iface.btn_import_state = gr.Button("Import JSON ➡️ Palette", variant='secondary')
+
+            gr.HTML(
+                """
+<div>
+</br>
+</div>
+<div style="justify-content: center; align-items: center;">
+<h3 style="margin: 0; text-align: center;"><b>❓Usage❓</b></h3>
+</br>
+<div style="justify-content: center; align-items: left; text-align: left;">
+    <p>1-1. Type in the background prompt. Background is not required if you paint the whole drawpad.</p>
+    <p>1-2. (Optional: <em><b>Inpainting mode</b></em>) Uploading a background image will make the app into inpainting mode. Removing the image returns to the creation mode. In the inpainting mode, increasing the <em>Mask Blur STD</em> > 8 for every colored palette is recommended for smooth boundaries.</p>
+    <p>2. Select a semantic brush by clicking onto one in the <b>Semantic Palette</b> above. Edit prompt for the semantic brush.</p>
+    <p>2-1. If you are willing to draw more diverse images, try <b>Create New Semantic Brush</b>.</p>
+    <p>3. Start drawing in the <b>Semantic Drawpad</b> tab. The brush color is directly linked to the semantic brushes.</p>
+    <p>4. Click [<b>GENERATE!</b>] button to create your (large-scale) artwork!</p>
+</div>
+</div>
+                """
+            )
+
+            gr.HTML(
+                """
+<div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+<h5 style="margin: 0;"><b>... or run in your own 🤗 space!</b></h5>
+</div>
+                """
+            )
+
+            gr.DuplicateButton()
+
+        with gr.Column(scale=4):
+
+            with gr.Row():
+
+                with gr.Column(scale=2):
+
+                    iface.ctrl_semantic = gr.ImageEditor(
+                        image_mode='RGBA',
+                        sources=['upload', 'clipboard', 'webcam'],
+                        transforms=['crop'],
+                        crop_size=(opt.width, opt.height),
+                        brush=gr.Brush(
+                            colors=opt.colors[1:],
+                            color_mode="fixed",
+                        ),
+                        type='pil',
+                        label='Semantic Drawpad',
+                        elem_id='drawpad',
+                    )
+
+#                     with gr.Accordion(label='Prompt Engineering', open=False):
+#                         iface.quality_select = gr.Dropdown(
+#                             label='Quality Presets',
+#                             interactive=True,
+#                             choices=list(_quality_dict.keys()),
+#                             value='Standard v3.1',
+#                         )
+
+#                         iface.style_select = gr.Radio(
+#                             label='Style Preset',
+#                             container=True,
+#                             interactive=True,
+#                             choices=list(_style_dict.keys()),
+#                             value='(None)',
+#                         )
+
+                with gr.Column(scale=2):
+
+                    iface.image_slot = gr.Image(
+                        interactive=False,
+                        show_label=False,
+                        show_download_button=True,
+                        type='pil',
+                        label='Generated Result',
+                        elem_id='output-screen',
+                        value=lambda: random.choice(example_images),
+                    )
+
+                    iface.btn_generate = gr.Button(
+                        value=f'Lemme try! ({int(opt.run_time // 60)} min)',
+                        variant='primary',
+                        # scale=1,
+                        elem_id='run-button'
+                    )
+
+                    iface.run_animation = gr.HTML(
+                        f"""
+<div id="deadline">
+  <svg preserveAspectRatio="none" id="line" viewBox="0 0 581 158" enable-background="new 0 0 581 158">
+    <g id="fire">
+      <rect id="mask-fire-black" x="511" y="41" width="38" height="34"/>
+      <g>
+        <defs>
+          <rect id="mask_fire" x="511" y="41" width="38" height="34"/>
+        </defs>
+        <clipPath id="mask-fire_1_">
+          <use xlink:href="#mask_fire"  overflow="visible"/>
+        </clipPath>
+        <g id="group-fire" clip-path="url(#mask-fire_1_)">
+          <path id="red-flame" fill="#B71342" d="M528.377,100.291c6.207,0,10.947-3.272,10.834-8.576 c-0.112-5.305-2.934-8.803-8.237-10.383c-5.306-1.581-3.838-7.9-0.79-9.707c-7.337,2.032-7.581,5.891-7.11,8.238 c0.789,3.951,7.56,4.402,5.077,9.48c-2.482,5.079-8.012,1.129-6.319-2.257c-2.843,2.233-4.78,6.681-2.259,9.703 C521.256,98.809,524.175,100.291,528.377,100.291z"/>
+          <path id="yellow-flame" opacity="0.71" fill="#F7B523" d="M528.837,100.291c4.197,0,5.108-1.854,5.974-5.417 c0.902-3.724-1.129-6.207-5.305-9.931c-2.396-2.137-1.581-4.176-0.565-6.32c-4.401,1.918-3.384,5.304-2.482,6.658 c1.511,2.267,2.099,2.364,0.42,5.8c-1.679,3.435-5.42,0.764-4.275-1.527c-1.921,1.512-2.373,4.04-1.528,6.563 C522.057,99.051,525.994,100.291,528.837,100.291z"/>
+          <path id="white-flame" opacity="0.81" fill="#FFFFFF" d="M529.461,100.291c-2.364,0-4.174-1.322-4.129-3.469 c0.04-2.145,1.117-3.56,3.141-4.198c2.022-0.638,1.463-3.195,0.302-3.925c2.798,0.821,2.89,2.382,2.711,3.332 c-0.301,1.597-2.883,1.779-1.938,3.834c0.912,1.975,3.286,0.938,2.409-0.913c1.086,0.903,1.826,2.701,0.864,3.924 C532.18,99.691,531.064,100.291,529.461,100.291z"/>
+        </g>
+      </g>
+    </g>
+    <g id="progress-trail">
+      <path fill="#FFFFFF" d="M491.979,83.878c1.215-0.73-0.62-5.404-3.229-11.044c-2.583-5.584-5.034-10.066-7.229-8.878
+                              c-2.854,1.544-0.192,6.286,2.979,11.628C487.667,80.917,490.667,84.667,491.979,83.878z"/>
+      <path fill="#FFFFFF" d="M571,76v-5h-23.608c0.476-9.951-4.642-13.25-4.642-13.25l-3.125,4c0,0,3.726,2.7,3.625,5.125
+                              c-0.071,1.714-2.711,3.18-4.962,4.125H517v5h10v24h-25v-5.666c0,0,0.839,0,2.839-0.667s6.172-3.667,4.005-6.333
+                              s-7.49,0.333-9.656,0.166s-6.479-1.5-8.146,1.917c-1.551,3.178,0.791,5.25,5.541,6.083l-0.065,4.5H16c-2.761,0-5,2.238-5,5v17
+                              c0,2.762,2.239,5,5,5h549c2.762,0,5-2.238,5-5v-17c0-2.762-2.238-5-5-5h-3V76H571z"/>
+      <path fill="#FFFFFF" d="M535,65.625c1.125,0.625,2.25-1.125,2.25-1.125l11.625-22.375c0,0,0.75-0.875-1.75-2.125
+                              s-3.375,0.25-3.375,0.25s-8.75,21.625-9.875,23.5S533.875,65,535,65.625z"/>
+    </g>
+    <g>
+      <defs>
+        <path id="SVGID_1_" d="M484.5,75.584c-3.172-5.342-5.833-10.084-2.979-11.628c2.195-1.188,4.646,3.294,7.229,8.878
+                               c2.609,5.64,4.444,10.313,3.229,11.044C490.667,84.667,487.667,80.917,484.5,75.584z M571,76v-5h-23.608
+                               c0.476-9.951-4.642-13.25-4.642-13.25l-3.125,4c0,0,3.726,2.7,3.625,5.125c-0.071,1.714-2.711,3.18-4.962,4.125H517v5h10v24h-25
+                               v-5.666c0,0,0.839,0,2.839-0.667s6.172-3.667,4.005-6.333s-7.49,0.333-9.656,0.166s-6.479-1.5-8.146,1.917
+                               c-1.551,3.178,0.791,5.25,5.541,6.083l-0.065,4.5H16c-2.761,0-5,2.238-5,5v17c0,2.762,2.239,5,5,5h549c2.762,0,5-2.238,5-5v-17
+                               c0-2.762-2.238-5-5-5h-3V76H571z M535,65.625c1.125,0.625,2.25-1.125,2.25-1.125l11.625-22.375c0,0,0.75-0.875-1.75-2.125
+                               s-3.375,0.25-3.375,0.25s-8.75,21.625-9.875,23.5S533.875,65,535,65.625z"/>
+      </defs>
+      <clipPath id="SVGID_2_">
+        <use xlink:href="#SVGID_1_"  overflow="visible"/>
+      </clipPath>
+      <rect id="progress-time-fill" x="-100%" y="34" clip-path="url(#SVGID_2_)" fill="#BE002A" width="586" height="103"/>
+    </g>
+
+    <g id="death-group">
+      <path id="death" fill="#BE002A" d="M-46.25,40.416c-5.42-0.281-8.349,3.17-13.25,3.918c-5.716,0.871-10.583-0.918-10.583-0.918
+                                         C-67.5,49-65.175,50.6-62.083,52c5.333,2.416,4.083,3.5,2.084,4.5c-16.5,4.833-15.417,27.917-15.417,27.917L-75.5,84.75
+                                         c-1,12.25-20.25,18.75-20.25,18.75s39.447,13.471,46.25-4.25c3.583-9.333-1.553-16.869-1.667-22.75
+                                         c-0.076-3.871,2.842-8.529,6.084-12.334c3.596-4.22,6.958-10.374,6.958-15.416C-38.125,43.186-39.833,40.75-46.25,40.416z
+                                         M-40,51.959c-0.882,3.004-2.779,6.906-4.154,6.537s-0.939-4.32,0.112-7.704c0.82-2.64,2.672-5.96,3.959-5.583
+                                         C-39.005,45.523-39.073,48.8-40,51.959z"/>
+      <path id="death-arm" fill="#BE002A" d="M-53.375,75.25c0,0,9.375,2.25,11.25,0.25s2.313-2.342,3.375-2.791
+                                             c1.083-0.459,4.375-1.75,4.292-4.75c-0.101-3.627,0.271-4.594,1.333-5.043c1.083-0.457,2.75-1.666,2.75-1.666
+                                             s0.708-0.291,0.5-0.875s-0.791-2.125-1.583-2.959c-0.792-0.832-2.375-1.874-2.917-1.332c-0.542,0.541-7.875,7.166-7.875,7.166
+                                             s-2.667,2.791-3.417,0.125S-49.833,61-49.833,61s-3.417,1.416-3.417,1.541s-1.25,5.834-1.25,5.834l-0.583,5.833L-53.375,75.25z"/>
+      <path id="death-tool" fill="#BE002A" d="M-20.996,26.839l-42.819,91.475l1.812,0.848l38.342-81.909c0,0,8.833,2.643,12.412,7.414
+                                              c5,6.668,4.75,14.084,4.75,14.084s4.354-7.732,0.083-17.666C-10,32.75-19.647,28.676-19.647,28.676l0.463-0.988L-20.996,26.839z"/>
+    </g>
+    <path id="designer-body" fill="#FEFFFE" d="M514.75,100.334c0,0,1.25-16.834-6.75-16.5c-5.501,0.229-5.583,3-10.833,1.666
+                                               c-3.251-0.826-5.084-15.75-0.834-22c4.948-7.277,12.086-9.266,13.334-7.833c2.25,2.583-2,10.833-4.5,14.167
+                                               c-2.5,3.333-1.833,10.416,0.5,9.916s8.026-0.141,10,2.25c3.166,3.834,4.916,17.667,4.916,17.667l0.917,2.5l-4,0.167L514.75,100.334z
+                                               "/>
+
+    <circle id="designer-head" fill="#FEFFFE" cx="516.083" cy="53.25" r="6.083"/>
+
+    <g id="designer-arm-grop">
+      <path id="designer-arm" fill="#FEFFFE" d="M505.875,64.875c0,0,5.875,7.5,13.042,6.791c6.419-0.635,11.833-2.791,13.458-4.041s2-3.5,0.25-3.875
+                                                s-11.375,5.125-16,3.25c-5.963-2.418-8.25-7.625-8.25-7.625l-2,1.125L505.875,64.875z"/>
+      <path id="designer-pen" fill="#FEFFFE" d="M525.75,59.084c0,0-0.423-0.262-0.969,0.088c-0.586,0.375-0.547,0.891-0.547,0.891l7.172,8.984l1.261,0.453
+                                                l-0.104-1.328L525.75,59.084z"/>
+    </g>
+  </svg>
+
+  <div class="deadline-timer">
+    Remaining <span class="day">{opt.run_time}</span> <span class="days">s</span>
+  </div>
+
+</div>
+                        """,
+                        elem_id='run-anim',
+                        visible=False,
+                    )
+
+            with gr.Group(elem_id='control-panel'):
+
+                with gr.Row():
+                    iface.tbox_prompt = gr.Textbox(
+                        label='Edit Prompt for Background',
+                        info='What do you want to draw?',
+                        value=state.value.prompts[0],
+                        placeholder=lambda: random.choice(prompt_suggestions),
+                        scale=2,
+                    )
+
+                    iface.slider_strength = gr.Slider(
+                        label='Prompt Strength',
+                        info='Blends fg & bg in the prompt level, >0.8 Preferred.',
+                        minimum=0.5,
+                        maximum=1.0,
+                        value=opt.default_prompt_strength,
+                        scale=1,
+                    )
+
+                with gr.Row():
+                    iface.tbox_neg_prompt = gr.Textbox(
+                        label='Edit Negative Prompt for Background',
+                        info='Add unwanted objects for this semantic brush.',
+                        value=opt.default_negative_prompt,
+                        scale=2,
+                    )
+
+                    iface.tbox_name = gr.Textbox(
+                        label='Edit Brush Name',
+                        info='Just for your convenience.',
+                        value=state.value.prompt_names[0],
+                        placeholder='🌄 Background',
+                        scale=1,
+                    )
+
+                with gr.Row():
+                    iface.slider_alpha = gr.Slider(
+                        label='Mask Alpha',
+                        info='Factor multiplied to the mask before quantization. Extremely sensitive, >0.98 Preferred.',
+                        minimum=0.5,
+                        maximum=1.0,
+                        value=opt.default_mask_strength,
+                    )
+
+                    iface.slider_std = gr.Slider(
+                        label='Mask Blur STD',
+                        info='Blends fg & bg in the latent level, 0 for generation, 8-32 for inpainting.',
+                        minimum=0.0001,
+                        maximum=100.0,
+                        value=opt.default_mask_std,
+                    )
+
+                    iface.slider_seed = gr.Slider(
+                        label='Seed',
+                        info='The global seed.',
+                        minimum=-1,
+                        maximum=2147483647,
+                        step=1,
+                        value=opt.seed,
+                    )
+
+    ### Attach event handlers
+
+    for idx, btn in enumerate(iface.btn_semantics):
+        btn.click(
+            fn=partial(select_palette, idx=idx),
+            inputs=[state, btn],
+            outputs=[state] + iface.btn_semantics + [
+                iface.tbox_name,
+                iface.tbox_prompt,
+                iface.tbox_neg_prompt,
+                iface.slider_alpha,
+                iface.slider_strength,
+                iface.slider_std,
+            ],
+            api_name=f'select_palette_{idx}',
+        )
+
+    iface.btn_add_palette.click(
+        fn=add_palette,
+        inputs=state,
+        outputs=[state, iface.btn_add_palette] + iface.btn_semantics[1:],
+        api_name='create_new',
+    )
+
+    run_event = iface.btn_generate.click(
+        fn=hide_element,
+        inputs=None,
+        outputs=iface.btn_generate,
+        api_name='hide_run_button',
+    ).then(
+        fn=show_element,
+        inputs=None,
+        outputs=iface.run_animation,
+        api_name='show_run_animation',
+    )
+
+    run_event.then(
+        fn=run,
+        inputs=[state, iface.ctrl_semantic],
+        outputs=[state, iface.image_slot],
+        api_name='run',
+    ).then(
+        fn=hide_element,
+        inputs=None,
+        outputs=iface.run_animation,
+        api_name='hide_run_animation',
+    ).then(
+        fn=show_element,
+        inputs=None,
+        outputs=iface.btn_generate,
+        api_name='show_run_button',
+    )
+
+    run_event.then(
+        fn=None,
+        inputs=None,
+        outputs=None,
+        api_name='run_animation',
+        js=f"""
+async () => {{
+    // timer arguments: 
+    //   #1 - time of animation in mileseconds, 
+    //   #2 - days to deadline
+    const animationTime = {opt.run_time};
+    const days = {opt.run_time};
+
+    jQuery('#progress-time-fill, #death-group').css({{'animation-duration': animationTime+'s'}});
+
+    var deadlineAnimation = function () {{
+        setTimeout(function() {{
+            jQuery('#designer-arm-grop').css({{'animation-duration': '1.5s'}});
+        }}, 0);
+
+        setTimeout(function() {{
+            jQuery('#designer-arm-grop').css({{'animation-duration': '1.0s'}});
+        }}, {int(opt.run_time * 1000 * 0.2)});
+
+        setTimeout(function() {{
+            jQuery('#designer-arm-grop').css({{'animation-duration': '0.7s'}});
+        }}, {int(opt.run_time * 1000 * 0.4)});
+
+        setTimeout(function() {{
+            jQuery('#designer-arm-grop').css({{'animation-duration': '0.3s'}});
+        }}, {int(opt.run_time * 1000 * 0.6)});
+
+        setTimeout(function() {{
+            jQuery('#designer-arm-grop').css({{'animation-duration': '0.2s'}});
+        }}, {int(opt.run_time * 1000 * 0.75)});
+    }};
+
+    var deadlineTextFinished = function () {{
+        var el = jQuery('.deadline-timer');
+        var html = 'Done! Retry?';
+        el.html(html);
+    }};
+
+    function timer(totalTime, deadline) {{
+        var time = totalTime * 1000;
+        var dayDuration = time / deadline;
+        var actualDay = deadline;
+
+        var timer = setInterval(countTime, dayDuration);
+
+        function countTime() {{
+            --actualDay;
+            jQuery('.deadline-timer .day').text(actualDay);
+
+            if (actualDay == 0) {{
+                clearInterval(timer);
+                // jQuery('.deadline-timer .day').text(deadline);
+                deadlineTextFinished();
+            }}
+        }}
+    }}
+
+    var deadlineText = function () {{
+        var el = jQuery('.deadline-timer');
+        var htmlBase = 'Remaining <span class="day">{opt.run_time}</span> <span class="days">s</span>';
+        el.html(html);
+        var html = '<div class="mask-red"><div class="inner">' + htmlBase + '</div></div><div class="mask-white"><div class="inner">' + htmlBase + '</div></div>';
+        el.html(html);
+    }};
+
+    var runAnimation = function() {{
+        timer(animationTime, days);
+        deadlineAnimation();
+        deadlineText();
+
+        console.log('begin interval', animationTime * 1000);
+    }};
+    runAnimation();
+}}
+        """
+    )
+
+    iface.slider_alpha.input(
+        fn=change_mask_strength,
+        inputs=[state, iface.slider_alpha],
+        outputs=state,
+        api_name='change_alpha',
+    )
+    iface.slider_std.input(
+        fn=change_std,
+        inputs=[state, iface.slider_std],
+        outputs=state,
+        api_name='change_std',
+    )
+    iface.slider_strength.input(
+        fn=change_prompt_strength,
+        inputs=[state, iface.slider_strength],
+        outputs=state,
+        api_name='change_strength',
+    )
+    iface.slider_seed.input(
+        fn=reset_seed,
+        inputs=[state, iface.slider_seed],
+        outputs=state,
+        api_name='reset_seed',
+    )
+
+    iface.tbox_name.input(
+        fn=rename_prompt,
+        inputs=[state, iface.tbox_name],
+        outputs=[state] + iface.btn_semantics,
+        api_name='prompt_rename',
+    )
+    iface.tbox_prompt.input(
+        fn=change_prompt,
+        inputs=[state, iface.tbox_prompt],
+        outputs=state,
+        api_name='prompt_edit',
+    )
+    iface.tbox_neg_prompt.input(
+        fn=change_neg_prompt,
+        inputs=[state, iface.tbox_neg_prompt],
+        outputs=state,
+        api_name='neg_prompt_edit',
+    )
+
+    # iface.style_select.change(
+    #     fn=select_style,
+    #     inputs=[state, iface.style_select],
+    #     outputs=state,
+    #     api_name='style_select',
+    # )
+    # iface.quality_select.change(
+    #     fn=select_quality,
+    #     inputs=[state, iface.quality_select],
+    #     outputs=state,
+    #     api_name='quality_select',
+    # )
+
+    iface.btn_export_state.click(lambda x: vars(x), state, iface.json_state_export)
+    iface.btn_import_state.click(import_state, [state, iface.tbox_state_import], [
+        state,
+        *iface.btn_semantics,
+        # iface.style_select,
+        # iface.quality_select,
+        iface.tbox_prompt,
+        iface.tbox_name,
+        iface.tbox_neg_prompt,
+        iface.slider_strength,
+        iface.slider_alpha,
+        iface.slider_std,
+        iface.slider_seed,
+    ])
+
+    # Realtime user input.
+    iface.ctrl_semantic.change(
+        fn=draw,
+        inputs=[state, iface.ctrl_semantic],
+        outputs=None,
+        api_name='draw',
+    )
+
+
+if __name__ == '__main__':
+    demo.launch(server_port=opt.port)

data.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2024 Jaerin Lee
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import copy
+from typing import Optional, Union
+from PIL import Image
+import torch
+
+
+class BackgroundObject:
+    def __init__(
+        self,
+        image: Optional[Image.Image] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+    ) -> None:
+        self.image = image
+        self.prompt = prompt
+        self.negative_prompt = negative_prompt
+
+    @property
+    def is_empty(self) -> bool:
+        return (
+            self.image is None and
+            self.prompt is None and
+            self.negative_prompt is None
+        )
+
+    def extra_repr(self) -> str:
+        return ''
+
+    def __repr__(self) -> str:
+        strings = []
+        if self.image is not None:
+            if isinstance(self.image, Image.Image):
+                image_str = f'Image(size={str(self.image.size)})'
+            else:
+                image_str = f'Tensor(shape={str(self.image.shape)})'
+            strings.append(f'image={image_str}')
+        if self.prompt is not None:
+            strings.append(f'prompt="{self.prompt}"')
+        if self.negative_prompt is not None:
+            strings.append(f'negative_prompt="{self.negative_prompt}"')
+        extra_repr = self.extra_repr()
+        if extra_repr != '':
+            strings.append(extra_repr)
+        return f'{type(self).__name__}({", ".join(strings)})'
+
+
+class LayerObject:
+    def __init__(
+        self,
+        idx: Optional[int] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+        suffix: Optional[str] = None,
+        prompt_strength: Optional[float] = None,
+        mask: Optional[Union[torch.Tensor, Image.Image]] = None,
+        mask_std: Optional[float] = None,
+        mask_strength: Optional[float] = None,
+    ) -> None:
+        self.idx = idx
+        self.prompt = prompt
+        self.negative_prompt = negative_prompt
+        self.suffix = suffix
+        self.prompt_strength = prompt_strength
+        self.mask = mask
+        self.mask_std = mask_std
+        self.mask_strength = mask_strength
+
+    @property
+    def is_empty(self) -> bool:
+        return (
+            self.prompt is None and
+            self.negative_prompt is None and
+            self.prompt_strength is None and
+            self.mask is None and
+            self.mask_strength is None and
+            self.mask_std is None
+        )
+
+    def merge(self, other: 'LayerObject') -> bool: # Overriden or not.
+        if self.idx != other.idx:
+            # Merge only the modification requests for the same layer.
+            return False
+
+        if self.prompt is None and other.prompt is not None:
+            self.prompt = copy.deepcopy(other.prompt)
+        if self.negative_prompt is None and other.negative_prompt is not None:
+            self.negative_prompt = copy.deepcopy(other.negative_prompt)
+        if self.suffix is None and other.suffix is not None:
+            self.suffix = copy.deepcopy(other.suffix)
+        if self.prompt_strength is None and other.prompt_strength is not None:
+            self.prompt_strength = copy.deepcopy(other.prompt_strength)
+        if self.mask is None and other.mask is not None:
+            self.mask = copy.deepcopy(other.mask)
+        if self.mask_strength is None and other.mask_strength is not None:
+            self.mask_strength = copy.deepcopy(other.mask_strength)
+        if self.mask_std is None and other.mask_std is not None:
+            self.mask_std = copy.deepcopy(other.mask_std)
+        return True
+
+    def extra_repr(self) -> str:
+        return ''
+
+    def __repr__(self) -> str:
+        strings = []
+        if self.idx is not None:
+            strings.append(f'idx={self.idx}')
+        if self.prompt is not None:
+            strings.append(f'prompt="{self.prompt}"')
+        if self.negative_prompt is not None:
+            strings.append(f'negative_prompt="{self.negative_prompt}"')
+        if self.suffix is not None:
+            strings.append(f'suffix="{self.suffix}"')
+        if self.prompt_strength is not None:
+            strings.append(f'prompt_strength={self.prompt_strength}')
+        if self.mask is not None:
+            if isinstance(self.mask, Image.Image):
+                mask_str = f'Image(size={str(self.mask.size)})'
+            else:
+                mask_str = f'Tensor(shape={str(self.mask.shape)})'
+            strings.append(f'mask={mask_str}')
+        if self.mask_std is not None:
+            strings.append(f'mask_std={self.mask_std}')
+        if self.mask_strength is not None:
+            strings.append(f'mask_strength={self.mask_strength}')
+        extra_repr = self.extra_repr()
+        if extra_repr != '':
+            strings.append(extra_repr)
+        return f'{type(self).__name__}({", ".join(strings)})'
+
+
+class BackgroundState(BackgroundObject):
+    def __init__(
+        self,
+        image: Optional[Image.Image] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+        latent: Optional[torch.Tensor] = None,
+        embed: Optional[torch.Tensor] = None,
+    ) -> None:
+        super().__init__(image, prompt, negative_prompt)
+        self.latent = latent
+        self.embed = embed
+
+    @property
+    def is_incomplete(self) -> bool:
+        return (
+            self.image is None or
+            self.prompt is None or
+            self.negative_prompt is None or
+            self.latent is None or
+            self.embed is None
+        )
+
+    def extra_repr(self) -> str:
+        strings = []
+        if self.latent is not None:
+            strings.append(f'latent=Tensor(shape={str(self.latent.shape)})')
+        if self.embed is not None:
+            strings.append(f'embed=Tuple[Tensor(shape={str(self.embed[0].shape)})]')
+        return ', '.join(strings)
+
+
+# TODO
+# class LayerState:
+#     def __init__(
+#         self,
+#         prompst: List[str] = [],
+#         negative_prompts: List[str] = [],
+#         suffix: List[str] = [],
+#         masks: Optional[torch.Tensor] = None,
+#         mask_std: Optional[torch.Tensor] = None,
+#         mask_strength: Optional[torch.Tensor] = None,
+#         original_masks: Optional[Union[torch.Tensor, List[Image.Image]]] = None,
+#     ) -> None:
+#         self.prompts = prompts
+#         self.negative_prompts = negative_prompts
+#         self.suffix = suffix
+#         self.masks = masks
+#         self.mask_std = mask_std
+#         self.mask_strength = mask_strength
+#         self.original_masks = original_masks
+
+#     def __len__(self) -> int:
+#         self.check_integrity(True)
+#         return len(self.prompts)
+
+#     @property
+#     def is_empty(self) -> bool:
+#         self.check_integrity(True)
+#         return len(self.prompt) == 0
+
+#     def check_integrity(self, throw_error: bool = True) -> bool:
+#         p = len(self.prompts)
+#         flag = (
+#             p != len(self.negative_prompts) or
+#             p != len(self.suffix) or
+#             p != len(self.masks) or
+#             p != len(self.mask_std) or
+#             p != len(self.mask_strength) or
+#             p != len(self.original_masks)
+#         )
+#         if flag and throw_error:
+#             print(
+#                 f'LayerState(\n\tlen(prompts): {p},\n\tlen(negative_prompts): {len(self.negative_prompts)},\n\t'
+#                 f'len(suffix): {len(self.suffix)},\n\tlen(masks): {len(self.masks)},\n\t'
+#                 f'len(mask_std): {len(self.mask_std)},\n\tlen(mask_strength): {len(self.mask_strength)},\n\t'
+#                 f'len(original_masks): {len(self.original_masks)}\n)'
+#             )
+#             raise ValueError('LayerState is corrupted!')
+#         return not flag
+
+#     def extra_repr(self) -> str:
+#         strings = []
+#         if self.idx is not None:
+#             strings.append(f'idx={self.idx}')
+#         if self.prompt is not None:
+#             strings.append(f'prompt="{self.prompt}"')
+#         if self.negative_prompt is not None:
+#             strings.append(f'negative_prompt="{self.negative_prompt}"')
+#         if self.suffix is not None:
+#             strings.append(f'suffix="{self.suffix}"')
+#         if self.mask is not None:
+#             if isinstance(self.mask, Image.Image):
+#                 mask_str = f'PIL.Image.Image(size={str(self.mask.size)})'
+#             else:
+#                 mask_str = f'torch.Tensor(shape={str(self.mask.shape)})'
+#             strings.append(f'mask={mask_str}')
+#         if self.mask_std is not None:
+#             strings.append(f'mask_std={self.mask_std}')
+#         if self.mask_strength is not None:
+#             strings.append(f'mask_strength={self.mask_strength}')
+#         return f'{type(self).__name__}({", ".join(strings)})'

examples/prompt_background.txt

@@ -0,0 +1,8 @@
+Maximalism, best quality, high quality, no humans, background, clear sky, ㅠblack sky, starry universe, planets
+Maximalism, best quality, high quality, no humans, background, clear sky, blue sky
+Maximalism, best quality, high quality, no humans, background, universe, void, black, galaxy, galaxy, stars, stars, stars
+Maximalism, best quality, high quality, no humans, background, galaxy
+Maximalism, best quality, high quality, no humans, background, sky, daylight
+Maximalism, best quality, high quality, no humans, background, skyscrappers, rooftop, city of light, helicopters, bright night, sky
+Maximalism, best quality, high quality, flowers, flowers, flowers, flower garden, no humans, background
+Maximalism, best quality, high quality, flowers, flowers, flowers, flower garden

examples/prompt_boy.txt

@@ -0,0 +1,15 @@
+1boy, looking at viewer, brown hair, blue shirt
+1boy, looking at viewer, brown hair, red shirt
+1boy, looking at viewer, brown hair, purple shirt
+1boy, looking at viewer, brown hair, orange shirt
+1boy, looking at viewer, brown hair, yellow shirt
+1boy, looking at viewer, brown hair, green shirt
+1boy, looking back, side shaved hair, cyberpunk cloths, robotic suit, large body
+1boy, looking back, short hair, renaissance cloths, noble boy
+1boy, looking back, long hair, ponytail, leather jacket, heavy metal boy
+1boy, looking at viewer, a king, kingly grace, majestic cloths, crown
+1boy, looking at viewer, an astronaut, brown hair, faint smile, engineer
+1boy, looking at viewer, a medieval knight, helmet, swordman, plate armour
+1boy, looking at viewer, black haired, old eastern cloth
+1boy, looking back, messy hair, suit, short beard, noir
+1boy, looking at viewer, cute face, light smile, starry eyes, jeans

examples/prompt_girl.txt

@@ -0,0 +1,16 @@
+1girl, looking at viewer, pretty face, light smile, haughty smile, proud, long wavy hair, charcoal dark eyes, chinese cloths
+1girl, looking at viewer, princess, pretty face, light smile, haughty smile, proud, long wavy hair, charcoal dark eyes, majestic gown
+1girl, looking at viewer, astronaut girl, long red hair, space suit, black starry eyes, happy face, pretty face
+1girl, looking at viewer, fantasy adventurer, backpack
+1girl, looking at viewer, astronaut girl, spacesuit, eva, happy face
+1girl, looking at viewer, soldier, rusty cloths, backpack, pretty face, sad smile, tears
+1girl, looking at viewer, majestic cloths, long hair, glittering eye, pretty face
+1girl, looking at viewer, from behind, majestic cloths, long hair, glittering eye
+1girl, looking at viewer, evil smile, very short hair, suit, evil genius
+1girl, looking at viewer, elven queen, green hair, haughty face, eyes wide open, crazy smile, brown jacket, leaves
+1girl, looking at viewer, purple hair, happy face, black leather jacket
+1girl, looking at viewer, pink hair, happy face, blue jeans, black leather jacket
+1girl, looking at viewer, knight, medium length hair, red hair, plate armour, blue eyes, sad, pretty face, determined face
+1girl, looking at viewer, pretty face, light smile, orange hair, casual cloths
+1girl, looking at viewer, pretty face, large smile, open mouth, uniform, mcdonald employee, short wavy hair
+1girl, looking at viewer, brown hair, ponytail, happy face, bright smile, blue jeans and white shirt

examples/prompt_props.txt

@@ -0,0 +1,43 @@
+🏯 Palace, Gyeongbokgung palace
+🌳 Garden, Chinese garden
+🏛️ Rome, Ancient city of Rome
+🧱 Wall, Castle wall
+🔴 Mars, Martian desert, Red rocky desert
+🌻 Grassland, Grasslands
+🏡 Village, A fantasy village
+🐉 Dragon, a flying chinese dragon
+🌏 Earth, Earth seen from ISS
+🚀 Space Station, the international space station
+🪻 Grassland, Rusty grassland with flowers
+🖼️ Tapestry, majestic tapestry, glittering effect, glowing in light, mural painting with mountain
+🏙️ City Ruin, city, ruins, ruins, ruins, deserted
+🏙️ Renaissance City, renaissance city, renaissance city, renaissance city
+🌷 Flowers, Flower garden
+🌼 Flowers, Flower garden, spring garden
+🌹 Flowers, Flowers flowers, flowers
+⛰️ Dolomites Mountains, Dolomites
+⛰️ Himalayas Mountains, Himalayas
+⛰️ Alps Mountains, Alps
+⛰️ Mountains, Mountains
+❄️⛰️ Mountains, Winter mountains
+🌷⛰️ Mountains, Spring mountains
+🌞⛰️ Mountains, Summer mountains
+🌵 Desert, A sandy desert, dunes
+🪨🌵 Desert, A rocky desert
+💦 Waterfall, A giant waterfall
+🌊 Ocean, Ocean
+⛱️ Seashore, Seashore
+🌅 Sea Horizon, Sea horizon
+🌊 Lake, Clear blue lake
+💻 Computer, A giant supecomputer
+🌳 Tree, A giant tree
+🌳 Forest, A forest
+🌳🌳 Forest, A dense forest
+🌲 Forest, Winter forest
+🌴 Forest, Summer forest, tropical forest
+👒 Hat, A hat
+🐶 Dog, Doggy body parts
+😻 Cat, A cat
+🦉 Owl, A small sitting owl
+🦅 Eagle, A small sitting eagle
+🚀 Rocket, A flying rocket

model.py

@@ -0,0 +1,1212 @@
+# Copyright (c) 2024 Jaerin Lee
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+from diffusers import DiffusionPipeline, LCMScheduler, EulerDiscreteScheduler, AutoencoderTiny
+from huggingface_hub import hf_hub_download
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as T
+from einops import rearrange
+
+from collections import deque
+from typing import Tuple, List, Literal, Optional, Union
+from PIL import Image
+
+from util import load_model, gaussian_lowpass, shift_to_mask_bbox_center
+from data import BackgroundObject, LayerObject, BackgroundState #, LayerState
+
+
+class StreamMultiDiffusion(nn.Module):
+    def __init__(
+        self,
+        device: torch.device,
+        dtype: torch.dtype = torch.float16,
+        sd_version: Literal['1.5'] = '1.5',
+        hf_key: Optional[str] = None,
+        lora_key: Optional[str] = None,
+        use_tiny_vae: bool = True,
+        t_index_list: List[int] = [0, 4, 12, 25, 37], # [0, 5, 16, 18, 20, 37], Magic number.
+        width: int = 512,
+        height: int = 512,
+        frame_buffer_size: int = 1,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 1.2,
+        delta: float = 1.0,
+        cfg_type: Literal['none', 'full', 'self', 'initialize'] = 'none',
+        seed: int = 2024,
+        autoflush: bool = True,
+        default_mask_std: float = 8.0,
+        default_mask_strength: float = 1.0,
+        default_prompt_strength: float = 0.95,
+        bootstrap_steps: int = 1,
+        bootstrap_mix_steps: float = 1.0,
+        # bootstrap_leak_sensitivity: float = 0.2,
+        preprocess_mask_cover_alpha: float = 0.3, # TODO
+        prompt_queue_capacity: int = 256,
+        mask_type: Literal['discrete', 'semi-continuous', 'continuous'] = 'continuous',
+        use_xformers: bool = True,
+    ) -> None:
+        super().__init__()
+
+        self.device = device
+        self.dtype = dtype
+        self.seed = seed
+        self.sd_version = sd_version
+
+        self.autoflush = autoflush
+        self.default_mask_std = default_mask_std
+        self.default_mask_strength = default_mask_strength
+        self.default_prompt_strength = default_prompt_strength
+        self.bootstrap_steps = (
+            bootstrap_steps > torch.arange(len(t_index_list))).to(dtype=self.dtype, device=self.device)
+        self.bootstrap_mix_steps = bootstrap_mix_steps
+        self.bootstrap_mix_ratios = (
+            bootstrap_mix_steps - torch.arange(len(t_index_list), dtype=self.dtype, device=self.device)).clip_(0, 1)
+        # self.bootstrap_leak_sensitivity = bootstrap_leak_sensitivity
+        self.preprocess_mask_cover_alpha = preprocess_mask_cover_alpha
+        self.mask_type = mask_type
+
+        ### State definition
+
+        # [0. Start]                 -(prepare)->           [1. Initialized]
+        # [1. Initialized]           -(update_background)-> [2. Background Registered] (len(self.prompts)==0)
+        # [2. Background Registered] -(update_layers)->     [3. Unflushed] (len(self.prompts)>0)
+
+        # [3. Unflushed]             -(flush)->             [4. Ready]
+        # [4. Ready]                 -(any updates)->       [3. Unflushed]
+        # [4. Ready]                 -(__call__)->          [4. Ready], continuously returns generated image.
+
+        self.ready_checklist = {
+            'initialized': False,
+            'background_registered': False,
+            'layers_ready': False,
+            'flushed': False,
+        }
+
+        ### Session state update queue: for lazy update policy for streaming applications.
+
+        self.update_buffer = {
+            'background': None,                            # Maintains a single instance of BackgroundObject
+            'layers': deque(maxlen=prompt_queue_capacity), # Maintains a queue of LayerObjects
+        }
+
+        print(f'[INFO]     Loading Stable Diffusion...')
+        get_scheduler = lambda pipe: LCMScheduler.from_config(pipe.scheduler.config)
+        lora_weight_name = None
+        if self.sd_version == '1.5':
+            if hf_key is not None:
+                print(f'[INFO]     Using custom model key: {hf_key}')
+                model_key = hf_key
+            else:
+                model_key = 'runwayml/stable-diffusion-v1-5'
+            lora_key = 'latent-consistency/lcm-lora-sdv1-5'
+            lora_weight_name = 'pytorch_lora_weights.safetensors'
+        # elif self.sd_version == 'xl':
+        #     model_key = 'stabilityai/stable-diffusion-xl-base-1.0'
+        #     lora_key = 'latent-consistency/lcm-lora-sdxl'
+        #     lora_weight_name = 'pytorch_lora_weights.safetensors'
+        else:
+            raise ValueError(f'Stable Diffusion version {self.sd_version} not supported.')
+
+        ### Internally stored "Session" states
+
+        self.state = {
+            'background': BackgroundState(), # Maintains a single instance of BackgroundState
+            # 'layers': LayerState(),          # Maintains a single instance of LayerState
+            'model_key': model_key,          # The Hugging Face model ID.
+        }
+
+        # Create model
+        self.i2t_processor = Blip2Processor.from_pretrained('Salesforce/blip2-opt-2.7b')
+        self.i2t_model = Blip2ForConditionalGeneration.from_pretrained('Salesforce/blip2-opt-2.7b')
+
+        self.pipe = load_model(model_key, self.sd_version, self.device, self.dtype)
+
+        self.pipe.load_lora_weights(lora_key, weight_name=lora_weight_name, adapter_name='lcm')
+        self.pipe.fuse_lora(
+            fuse_unet=True,
+            fuse_text_encoder=True,
+            lora_scale=1.0,
+            safe_fusing=False,
+        )
+        if use_xformers:
+            self.pipe.enable_xformers_memory_efficient_attention()
+
+        self.vae = (
+            AutoencoderTiny.from_pretrained('madebyollin/taesd').to(device=self.device, dtype=self.dtype)
+            if use_tiny_vae else self.pipe.vae
+        )
+        # self.tokenizer = self.pipe.tokenizer
+        self.text_encoder = self.pipe.text_encoder
+        self.unet = self.pipe.unet
+        self.vae_scale_factor = self.pipe.vae_scale_factor
+
+        self.scheduler = get_scheduler(self.pipe)
+        self.scheduler.set_timesteps(num_inference_steps)
+
+        self.generator = None
+
+        # Lock the canvas size--changing the canvas size can be implemented by reloading the module.
+        self.height = height
+        self.width = width
+        self.latent_height = int(height // self.pipe.vae_scale_factor)
+        self.latent_width = int(width // self.pipe.vae_scale_factor)
+
+        # For bootstrapping.
+        self.white = self.encode_imgs(torch.ones(1, 3, height, width, dtype=self.dtype, device=self.device))
+
+        # StreamDiffusion setting.
+        self.t_list = t_index_list
+        assert len(self.t_list) > 1, 'Current version only supports diffusion models with multiple steps.'
+        self.frame_bff_size = frame_buffer_size  # f
+        self.denoising_steps_num = len(self.t_list)  # t=2
+        self.cfg_type = cfg_type
+        self.num_inference_steps = num_inference_steps
+        self.guidance_scale = 1.0 if self.cfg_type == 'none' else guidance_scale
+        self.delta = delta
+
+        self.batch_size = self.denoising_steps_num * frame_buffer_size  # T = t*f
+        if self.cfg_type == 'initialize':
+            self.trt_unet_batch_size = (self.denoising_steps_num + 1) * self.frame_bff_size
+        elif self.cfg_type == 'full':
+            self.trt_unet_batch_size = 2 * self.denoising_steps_num * self.frame_bff_size
+        else:
+            self.trt_unet_batch_size = self.denoising_steps_num * frame_buffer_size
+
+        print(f'[INFO]     Model is loaded!')
+
+        self.reset_seed(self.generator, seed)
+        self.reset_latent()
+        self.prepare()
+
+        print(f'[INFO]     Parameters prepared!')
+
+        self.ready_checklist['initialized'] = True
+
+    @property
+    def background(self) -> BackgroundState:
+        return self.state['background']
+
+    # @property
+    # def layers(self) -> LayerState:
+    #     return self.state['layers']
+
+    @property
+    def num_layers(self) -> int:
+        return len(self.prompts) if hasattr(self, 'prompts') else 0
+
+    @property
+    def is_ready_except_flush(self) -> bool:
+        return all(v for k, v in self.ready_checklist.items() if k != 'flushed')
+
+    @property
+    def is_flush_needed(self) -> bool:
+        return self.autoflush and not self.ready_checklist['flushed']
+
+    @property
+    def is_ready(self) -> bool:
+        return self.is_ready_except_flush and not self.is_flush_needed
+
+    @property
+    def is_dirty(self) -> bool:
+        return not (self.update_buffer['background'] is None and len(self.update_buffer['layers']) == 0)
+
+    @property
+    def has_background(self) -> bool:
+        return self.background.is_empty
+
+    # @property
+    # def has_layers(self) -> bool:
+    #     return len(self.layers) > 0
+
+    def __repr__(self) -> str:
+        return (
+            f'{type(self).__name__}(\n\tbackground: {str(self.background)},\n\t'
+            f'model_key: {self.state["model_key"]}\n)'
+            # f'layers: {str(self.layers)},\n\tmodel_key: {self.state["model_key"]}\n)'
+        )
+
+    def check_integrity(self, throw_error: bool = True) -> bool:
+        p = len(self.prompts)
+        flag = (
+            p != len(self.negative_prompts) or
+            p != len(self.prompt_strengths) or
+            p != len(self.masks) or
+            p != len(self.mask_strengths) or
+            p != len(self.mask_stds) or
+            p != len(self.original_masks)
+        )
+        if flag and throw_error:
+            print(
+                f'LayerState(\n\tlen(prompts): {p},\n\tlen(negative_prompts): {len(self.negative_prompts)},\n\t'
+                f'len(prompt_strengths): {len(self.prompt_strengths)},\n\tlen(masks): {len(self.masks)},\n\t'
+                f'len(mask_stds): {len(self.mask_stds)},\n\tlen(mask_strengths): {len(self.mask_strengths)},\n\t'
+                f'len(original_masks): {len(self.original_masks)}\n)'
+            )
+            raise ValueError('[ERROR]    LayerState is corrupted!')
+        return not flag
+
+    def check_ready(self) -> bool:
+        all_except_flushed = all(v for k, v in self.ready_checklist.items() if k != 'flushed')
+        if all_except_flushed:
+            if self.is_flush_needed:
+                self.flush()
+            return True
+
+        print('[WARNING]  MagicDraw module is not ready yet! Complete the checklist:')
+        for k, v in self.ready_checklist.items():
+            prefix = '  [ v ] ' if v else '  [ x ] '
+            print(prefix + k.replace('_', ' '))
+        return False
+
+    def reset_seed(self, generator: Optional[torch.Generator] = None, seed: Optional[int] = None) -> None:
+        generator = torch.Generator(self.device) if generator is None else generator
+        seed = self.seed if seed is None else seed
+        self.generator = generator
+        self.generator.manual_seed(seed)
+
+        self.init_noise = torch.randn((self.batch_size, 4, self.latent_height, self.latent_width),
+            generator=generator, device=self.device, dtype=self.dtype)
+        self.stock_noise = torch.zeros_like(self.init_noise)
+
+        self.ready_checklist['flushed'] = False
+
+    def reset_latent(self) -> None:
+        # initialize x_t_latent (it can be any random tensor)
+        b = (self.denoising_steps_num - 1) * self.frame_bff_size
+        self.x_t_latent_buffer = torch.zeros(
+            (b, 4, self.latent_height, self.latent_width), dtype=self.dtype, device=self.device)
+
+    def reset_state(self) -> None:
+        # TODO Reset states for context switch between multiple users.
+        pass
+
+    def prepare(self) -> None:
+        # make sub timesteps list based on the indices in the t_list list and the values in the timesteps list
+        self.timesteps = self.scheduler.timesteps.to(self.device)
+        self.sub_timesteps = []
+        for t in self.t_list:
+            self.sub_timesteps.append(self.timesteps[t])
+        sub_timesteps_tensor = torch.tensor(self.sub_timesteps, dtype=torch.long, device=self.device)
+        self.sub_timesteps_tensor = sub_timesteps_tensor.repeat_interleave(self.frame_bff_size, dim=0)
+
+        c_skip_list = []
+        c_out_list = []
+        for timestep in self.sub_timesteps:
+            c_skip, c_out = self.scheduler.get_scalings_for_boundary_condition_discrete(timestep)
+            c_skip_list.append(c_skip)
+            c_out_list.append(c_out)
+        self.c_skip = torch.stack(c_skip_list).view(len(self.t_list), 1, 1, 1).to(dtype=self.dtype, device=self.device)
+        self.c_out = torch.stack(c_out_list).view(len(self.t_list), 1, 1, 1).to(dtype=self.dtype, device=self.device)
+
+        alpha_prod_t_sqrt_list = []
+        beta_prod_t_sqrt_list = []
+        for timestep in self.sub_timesteps:
+            alpha_prod_t_sqrt = self.scheduler.alphas_cumprod[timestep].sqrt()
+            beta_prod_t_sqrt = (1 - self.scheduler.alphas_cumprod[timestep]).sqrt()
+            alpha_prod_t_sqrt_list.append(alpha_prod_t_sqrt)
+            beta_prod_t_sqrt_list.append(beta_prod_t_sqrt)
+        alpha_prod_t_sqrt = (torch.stack(alpha_prod_t_sqrt_list).view(len(self.t_list), 1, 1, 1)
+            .to(dtype=self.dtype, device=self.device))
+        beta_prod_t_sqrt = (torch.stack(beta_prod_t_sqrt_list).view(len(self.t_list), 1, 1, 1)
+            .to(dtype=self.dtype, device=self.device))
+        self.alpha_prod_t_sqrt = alpha_prod_t_sqrt.repeat_interleave(self.frame_bff_size, dim=0)
+        self.beta_prod_t_sqrt = beta_prod_t_sqrt.repeat_interleave(self.frame_bff_size, dim=0)
+
+        noise_lvs = ((1 - self.scheduler.alphas_cumprod.to(self.device)[self.sub_timesteps_tensor]) ** 0.5)
+        self.noise_lvs = noise_lvs[None, :, None, None, None]
+        self.next_noise_lvs = torch.cat([noise_lvs[1:], noise_lvs.new_zeros(1)])[None, :, None, None, None]
+
+    @torch.no_grad()
+    def get_text_prompts(self, image: Image.Image) -> str:
+        r"""A convenient method to extract text prompt from an image.
+
+        This is called if the user does not provide background prompt but only
+        the background image. We use BLIP-2 to automatically generate prompts.
+
+        Args:
+            image (Image.Image): A PIL image.
+
+        Returns:
+            A single string of text prompt.
+        """
+        question = 'Question: What are in the image? Answer:'
+        inputs = self.i2t_processor(image, question, return_tensors='pt')
+        out = self.i2t_model.generate(**inputs, max_new_tokens=77)
+        prompt = self.i2t_processor.decode(out[0], skip_special_tokens=True).strip()
+        return prompt
+
+    @torch.no_grad()
+    def encode_imgs(
+        self,
+        imgs: torch.Tensor,
+        generator: Optional[torch.Generator] = None,
+        add_noise: bool = False,
+    ) -> torch.Tensor:
+        r"""A wrapper function for VAE encoder of the latent diffusion model.
+
+        Args:
+            imgs (torch.Tensor): An image to get StableDiffusion latents.
+                Expected shape: (B, 3, H, W). Expected pixel scale: [0, 1].
+            generator (Optional[torch.Generator]): Seed for KL-Autoencoder.
+            add_noise (bool): Turn this on for a noisy latent.
+
+        Returns:
+            An image latent embedding with 1/8 size (depending on the auto-
+            encoder. Shape: (B, 4, H//8, W//8).
+        """
+        def _retrieve_latents(
+            encoder_output: torch.Tensor,
+            generator: Optional[torch.Generator] = None,
+            sample_mode: str = 'sample',
+        ):
+            if hasattr(encoder_output, 'latent_dist') and sample_mode == 'sample':
+                return encoder_output.latent_dist.sample(generator)
+            elif hasattr(encoder_output, 'latent_dist') and sample_mode == 'argmax':
+                return encoder_output.latent_dist.mode()
+            elif hasattr(encoder_output, 'latents'):
+                return encoder_output.latents
+            else:
+                raise AttributeError('[ERROR]    Could not access latents of provided encoder_output')
+
+        imgs = 2 * imgs - 1
+        latents = self.vae.config.scaling_factor * _retrieve_latents(self.vae.encode(imgs), generator=generator)
+        if add_noise:
+            latents = self.alpha_prod_t_sqrt[0] * latents + self.beta_prod_t_sqrt[0] * self.init_noise[0]
+        return latents
+
+    @torch.no_grad()
+    def decode_latents(self, latents: torch.Tensor) -> torch.Tensor:
+        r"""A wrapper function for VAE decoder of the latent diffusion model.
+
+        Args:
+            latents (torch.Tensor): An image latent to get associated images.
+                Expected shape: (B, 4, H//8, W//8).
+
+        Returns:
+            An image latent embedding with 1/8 size (depending on the auto-
+            encoder. Shape: (B, 3, H, W).
+        """
+        latents = 1 / self.vae.config.scaling_factor * latents
+        imgs = self.vae.decode(latents).sample
+        imgs = (imgs / 2 + 0.5).clip_(0, 1)
+        return imgs
+
+    @torch.no_grad()
+    def update_background(
+        self,
+        image: Optional[Image.Image] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+    ) -> bool:
+        flag_changed = False
+        if image is not None:
+            image_ = image.resize((self.width, self.height))
+            prompt = self.get_text_prompts(image_) if prompt is None else prompt
+            negative_prompt = '' if negative_prompt is None else negative_prompt
+            embed = self.pipe.encode_prompt(
+                prompt=[prompt],
+                device=self.device,
+                num_images_per_prompt=1,
+                do_classifier_free_guidance=(self.guidance_scale > 1.0),
+                negative_prompt=[negative_prompt],
+            )  # ((1, 77, 768): cond, (1, 77, 768): uncond)
+
+            self.state['background'].image = image
+            self.state['background'].latent = (
+                self.encode_imgs(T.ToTensor()(image_)[None].to(self.device, self.dtype))
+            )  # (1, 3, H, W)
+            self.state['background'].prompt = prompt
+            self.state['background'].negative_prompt = negative_prompt
+            self.state['background'].embed = embed
+
+            if self.bootstrap_steps[0] > 0:
+                mix_ratio = self.bootstrap_mix_ratios[:, None, None, None]
+                self.bootstrap_latent = mix_ratio * self.white + (1.0 - mix_ratio) * self.state['background'].latent
+
+            self.ready_checklist['background_registered'] = True
+            flag_changed = True
+        else:
+            if not self.ready_checklist['background_registered']:
+                print('[WARNING]  Register background image first! Request ignored.')
+                return False
+
+            if prompt is not None:
+                self.background.prompt = prompt
+                flag_changed = True
+            if negative_prompt is not None:
+                self.background.negative_prompt = negative_prompt
+                flag_changed = True
+            if flag_changed:
+                self.background.embed = self.pipe.encode_prompt(
+                    prompt=[self.background.prompt],
+                    device=self.device,
+                    num_images_per_prompt=1,
+                    do_classifier_free_guidance=(self.guidance_scale > 1.0),
+                    negative_prompt=[self.background.negative_prompt],
+                )  # ((1, 77, 768): cond, (1, 77, 768): uncond)
+    
+        self.ready_checklist['flushed'] = not flag_changed
+        return flag_changed
+
+    @torch.no_grad()
+    def process_mask(
+        self,
+        masks: Optional[Union[torch.Tensor, Image.Image, List[Image.Image]]] = None,
+        strength: Optional[Union[torch.Tensor, float]] = None,
+        std: Optional[Union[torch.Tensor, float]] = None,
+    ) -> Tuple[torch.Tensor]:
+        r"""Fast preprocess of masks for region-based generation with fine-
+        grained controls.
+
+        Mask preprocessing is done in four steps:
+         1. Resizing: Resize the masks into the specified width and height by
+            nearest neighbor interpolation.
+         2. (Optional) Ordering: Masks with higher indices are considered to
+            cover the masks with smaller indices. Covered masks are decayed
+            in its alpha value by the specified factor of
+            `preprocess_mask_cover_alpha`.
+         3. Blurring: Gaussian blur is applied to the mask with the specified
+            standard deviation (isotropic). This results in gradual increase of
+            masked region as the timesteps evolve, naturally blending fore-
+            ground and the predesignated background. Not strictly required if
+            you want to produce images from scratch withoout background.
+         4. Quantization: Split the real-numbered masks of value between [0, 1]
+            into predefined noise levels for each quantized scheduling step of
+            the diffusion sampler. For example, if the diffusion model sampler
+            has noise level of [0.9977, 0.9912, 0.9735, 0.8499, 0.5840], which
+            is the default noise level of this module with schedule [0, 4, 12,
+            25, 37], the masks are split into binary masks whose values are
+            greater than these levels. This results in tradual increase of mask
+            region as the timesteps increase. Details are described in our
+            paper at https://arxiv.org/pdf/2403.09055.pdf.
+
+        On the Three Modes of `mask_type`:
+            `self.mask_type` is predefined at the initialization stage of this
+            pipeline. Three possible modes are available: 'discrete', 'semi-
+            continuous', and 'continuous'. These define the mask quantization
+            modes we use. Basically, this (subtly) controls the smoothness of
+            foreground-background blending. Continuous modes produces nonbinary
+            masks to further blend foreground and background latents by linear-
+            ly interpolating between them. Semi-continuous masks only applies
+            continuous mask at the last step of the LCM sampler. Due to the
+            large step size of the LCM scheduler, we find that our continuous
+            blending helps generating seamless inpainting and editing results.
+
+        Args:
+            masks (Union[torch.Tensor, Image.Image, List[Image.Image]]): Masks.
+            strength (Optional[Union[torch.Tensor, float]]): Mask strength that
+                overrides the default value. A globally multiplied factor to
+                the mask at the initial stage of processing. Can be applied
+                seperately for each mask.
+            std (Optional[Union[torch.Tensor, float]]): Mask blurring Gaussian
+                kernel's standard deviation. Overrides the default value. Can
+                be applied seperately for each mask.
+
+        Returns: A tuple of tensors.
+          - masks: Preprocessed (ordered, blurred, and quantized) binary/non-
+                binary masks (see the explanation on `mask_type` above) for
+                region-based image synthesis.
+          - strengths: Return mask strengths for caching.
+          - std: Return mask blur standard deviations for caching.
+          - original_masks: Return original masks for caching.
+        """
+        if masks is None:
+            kwargs = {'dtype': self.dtype, 'device': self.device}
+            original_masks = torch.zeros((0, 1, self.latent_height, self.latent_width), dtype=self.dtype)
+            masks = torch.zeros((0, self.batch_size, 1, self.latent_height, self.latent_width), **kwargs)
+            strength = torch.zeros((0,), **kwargs)
+            std = torch.zeros((0,), **kwargs)
+            return masks, strength, std, original_masks
+
+        if isinstance(masks, Image.Image):
+            masks = [masks]
+        if isinstance(masks, (tuple, list)):
+            # Assumes white background for Image.Image;
+            # inverted boolean masks with shape (1, 1, H, W) for torch.Tensor.
+            masks = torch.cat([
+                # (T.ToTensor()(mask.resize((self.width, self.height), Image.NEAREST)) < 0.5)[None, :1]
+                (1.0 - T.ToTensor()(mask.resize((self.width, self.height), Image.BILINEAR)))[None, :1]
+                for mask in masks
+            ], dim=0).float().clip_(0, 1)
+        original_masks = masks            
+        masks = masks.float().to(self.device)
+
+        # Background mask alpha is decayed by the specified factor where foreground masks covers it.
+        if self.preprocess_mask_cover_alpha > 0:
+            masks = torch.stack([
+                torch.where(
+                    masks[i + 1:].sum(dim=0) > 0,
+                    mask * self.preprocess_mask_cover_alpha,
+                    mask,
+                ) if i < len(masks) - 1 else mask
+                for i, mask in enumerate(masks)
+            ], dim=0)
+
+        if std is None:
+            std = self.default_mask_std
+        if isinstance(std, (int, float)):
+            std = [std] * len(masks)
+        if isinstance(std, (list, tuple)):
+            std = torch.as_tensor(std, dtype=torch.float, device=self.device)
+
+        # Mask preprocessing parameters are fetched from the default settings.
+        if strength is None:
+            strength = self.default_mask_strength
+        if isinstance(strength, (int, float)):
+            strength = [strength] * len(masks)
+        if isinstance(strength, (list, tuple)):
+            strength = torch.as_tensor(strength, dtype=torch.float, device=self.device)
+
+        if (std > 0).any():
+            std = torch.where(std > 0, std, 1e-5)
+            masks = gaussian_lowpass(masks, std)
+        # NOTE: This `strength` aligns with `denoising strength`. However, with LCM, using strength < 0.96
+        #       gives unpleasant results.
+        masks = masks * strength[:, None, None, None]
+        masks = masks.unsqueeze(1).repeat(1, self.noise_lvs.shape[1], 1, 1, 1)
+
+        if self.mask_type == 'discrete':
+            # Discrete mode.
+            masks = masks > self.noise_lvs
+        elif self.mask_type == 'semi-continuous':
+            # Semi-continuous mode (continuous at the last step only).
+            masks = torch.cat((
+                masks[:, :-1] > self.noise_lvs[:, :-1],
+                (
+                    (masks[:, -1:] - self.next_noise_lvs[:, -1:])
+                    / (self.noise_lvs[:, -1:] - self.next_noise_lvs[:, -1:])
+                ).clip_(0, 1),
+            ), dim=1)
+        elif self.mask_type == 'continuous':
+            # Continuous mode: Have the exact same `1` coverage with discrete mode, but the mask gradually
+            #                  decreases continuously after the discrete mode boundary to become `0` at the
+            #                  next lower threshold.
+            masks = ((masks - self.next_noise_lvs) / (self.noise_lvs - self.next_noise_lvs)).clip_(0, 1)
+
+        # NOTE: Post processing mask strength does not align with conventional 'denoising_strength'. However,
+        #       fine-grained mask alpha channel tuning is available with this form.
+        # masks = masks * strength[None, :, None, None, None]
+
+        masks = rearrange(masks.float(), 'p t () h w -> (p t) () h w')
+        masks = F.interpolate(masks, size=(self.latent_height, self.latent_width), mode='nearest')
+        masks = rearrange(masks.to(self.dtype), '(p t) () h w -> p t () h w', p=len(std))
+        return masks, strength, std, original_masks
+
+    @torch.no_grad()
+    def update_layers(
+        self,
+        prompts: Union[str, List[str]],
+        negative_prompts: Optional[Union[str, List[str]]] = None,
+        suffix: Optional[str] = None, #', background is ',
+        prompt_strengths: Optional[Union[torch.Tensor, float, List[float]]] = None,
+        masks: Optional[Union[torch.Tensor, Image.Image, List[Image.Image]]] = None,
+        mask_strengths: Optional[Union[torch.Tensor, float, List[float]]] = None,
+        mask_stds: Optional[Union[torch.Tensor, float, List[float]]] = None,
+    ) -> None:
+        if not self.ready_checklist['background_registered']:
+            print('[WARNING]  Register background image first! Request ignored.')
+            return
+
+        ### Register prompts
+
+        if isinstance(prompts, str):
+            prompts = [prompts]
+        if negative_prompts is None:
+            negative_prompts = ''
+        if isinstance(negative_prompts, str):
+            negative_prompts = [negative_prompts]
+        fg_prompt = [p + suffix + self.background.prompt if suffix is not None else p for p in prompts]
+        self.prompts = fg_prompt
+        self.negative_prompts = negative_prompts
+        p = self.num_layers
+
+        e = self.pipe.encode_prompt(
+            prompt=fg_prompt,
+            device=self.device,
+            num_images_per_prompt=1,
+            do_classifier_free_guidance=(self.guidance_scale > 1.0),
+            negative_prompt=negative_prompts,
+        )  # (p, 77, 768)
+
+        if prompt_strengths is None:
+            prompt_strengths = self.default_prompt_strength
+        if isinstance(prompt_strengths, (int, float)):
+            prompt_strengths = [prompt_strengths] * p
+        if isinstance(prompt_strengths, (list, tuple)):
+            prompt_strengths = torch.as_tensor(prompt_strengths, dtype=self.dtype, device=self.device)
+        self.prompt_strengths = prompt_strengths
+
+        s = prompt_strengths[:, None, None]
+        self.prompt_embeds = torch.lerp(self.background.embed[0], e[0], s).repeat(self.batch_size, 1, 1)  # (T * p, 77, 768)
+        if self.guidance_scale > 1.0 and self.cfg_type in ('initialize', 'full'):
+            b = self.batch_size if self.cfg_type == 'full' else self.frame_bff_size
+            uncond_prompt_embeds = torch.lerp(self.background.embed[1], e[1], s).repeat(b, 1, 1)  # (T * p, 77, 768)
+            self.prompt_embeds = torch.cat([uncond_prompt_embeds, self.prompt_embeds], dim=0)  # (2 * T * p, 77, 768)
+
+        self.sub_timesteps_tensor_ = self.sub_timesteps_tensor.repeat_interleave(p)  # (T * p,)
+        self.init_noise_ = self.init_noise.repeat_interleave(p, dim=0)  # (T * p, 77, 768)
+        self.stock_noise_ = self.stock_noise.repeat_interleave(p, dim=0)  # (T * p, 77, 768)
+        self.c_out_ = self.c_out.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+        self.c_skip_ = self.c_skip.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+        self.beta_prod_t_sqrt_ = self.beta_prod_t_sqrt.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+        self.alpha_prod_t_sqrt_ = self.alpha_prod_t_sqrt.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+
+        ### Register new masks
+
+        if isinstance(masks, Image.Image):
+            masks = [masks]
+        n = len(masks) if masks is not None else 0
+
+        # Modificiation.
+        masks, mask_strengths, mask_stds, original_masks = self.process_mask(masks, mask_strengths, mask_stds)
+
+        self.counts = masks.sum(dim=0)  # (T, 1, h, w)
+        self.bg_mask = (1 - self.counts).clip_(0, 1)  # (T, 1, h, w)
+        self.masks = masks  # (p, T, 1, h, w)
+        self.mask_strengths = mask_strengths  # (p,)
+        self.mask_stds = mask_stds  # (p,)
+        self.original_masks = original_masks  # (p, 1, h, w)
+
+        if p > n:
+            # Add more masks: counts and bg_masks are not changed, but only masks are changed.
+            self.masks = torch.cat((
+                self.masks,
+                torch.zeros(
+                    (p - n, self.batch_size, 1, self.latent_height, self.latent_width),
+                    dtype=self.dtype,
+                    device=self.device,
+                ),
+            ), dim=0)
+            print(f'[WARNING]  Detected more prompts ({p}) than masks ({n}). '
+                  'Automatically adds blank masks for the additional prompts.')
+        elif p < n:
+            # Warns user to add more prompts.
+            print(f'[WARNING]  Detected more masks ({n}) than prompts ({p}). '
+                  'Additional masks are ignored until more prompts are provided.')
+
+        self.ready_checklist['layers_ready'] = True
+        self.ready_checklist['flushed'] = False
+
+    @torch.no_grad()
+    def update_single_layer(
+        self,
+        idx: Optional[int] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+        suffix: Optional[str] = None, #', background is ',
+        prompt_strength: Optional[float] = None,
+        mask: Optional[Union[torch.Tensor, Image.Image]] = None,
+        mask_strength: Optional[float] = None,
+        mask_std: Optional[float] = None,
+    ) -> None:
+
+        ### Possible input combinations and expected behaviors
+
+        # The module will consider a layer, a pair of (prompt, mask), to be 'active' only if a prompt
+        # is registered. A blank mask will be assigned if no mask is provided for the 'active' layer.
+        # The layers should be in either of ('active', 'inactive') states. 'inactive' layers will not
+        # receive any input unless equipped with prompt. 'active' layers receive any input and modify
+        # their states accordingly. In the actual implementation, only the 'active' layers are stored
+        # and can be accessed by the fields. Values len(self.prompts) = self.num_layers is the number
+        # of 'active' layers.
+
+        # If no background is registered. The layers should be all 'inactive'.
+        if not self.ready_checklist['background_registered']:
+            print('[WARNING]  Register background image first! Request ignored.')
+            return
+
+        # The first layer create request should be carrying a prompt. If only mask is drawn without a
+        # prompt, it just ignores the request--the user will update her request soon.
+        if self.num_layers == 0:
+            if prompt is not None:
+                self.update_layers(
+                    prompts=prompt,
+                    negative_prompts=negative_prompt,
+                    suffix=suffix,
+                    prompt_strengths=prompt_strength,
+                    masks=mask,
+                    mask_strengths=mask_strength,
+                    mask_stds=mask_std,
+                )
+            return
+
+        # Invalid request indices -> considered as a layer add request.
+        if idx is None or idx > self.num_layers or idx < 0:
+            idx = self.num_layers
+
+        # Two modes for the layer edits: 'append mode' and 'edit mode'. 'append mode' appends a new
+        # layer at the end of the layers list. 'edit mode' modifies internal variables for the given
+        # index. 'append mode' is defined by the request index and strictly requires a prompt input.
+        is_appending = idx == self.num_layers
+        if is_appending and prompt is None:
+            print(f'[WARNING]  Creating a new prompt at index ({idx}) but found no prompt. Request ignored.')
+            return
+
+        ### Register prompts
+
+        # | prompt    | neg_prompt | append mode (idx==len)  | edit mode (idx<len)  |
+        # | --------- | ---------- | ----------------------- | -------------------- |
+        # | given     | given      | append new prompt embed | replace prompt embed |
+        # | given     | not given  | append new prompt embed | replace prompt embed |
+        # | not given | given      | NOT ALLOWED             | replace prompt embed |
+        # | not given | not given  | NOT ALLOWED             | do nothing           |
+
+        # | prompt_strength | append mode (idx==len) | edit mode (idx<len)                            |
+        # | --------------- | ---------------------- | ---------------------------------------------- |
+        # | given           | use given strength     | use given strength                             |
+        # | not given       | use default strength   | replace strength / if no existing, use default |
+
+        p = self.num_layers
+
+        flag_prompt_edited = (
+            prompt is not None or
+            negative_prompt is not None or
+            prompt_strength is not None
+        )
+
+        if flag_prompt_edited:
+            is_double_cond = self.guidance_scale > 1.0 and self.cfg_type in ('initialize', 'full')
+
+            # Synchonize the internal state.
+
+            # We have asserted that prompt is not None if the mode is 'appending'.
+            if prompt is not None:
+                if suffix is not None:
+                    prompt = prompt + suffix + self.background.prompt
+                if is_appending:
+                    self.prompts.append(prompt)
+                else:
+                    self.prompts[idx] = prompt
+
+            if negative_prompt is not None:
+                if is_appending:
+                    self.negative_prompts.append(negative_prompt)
+                else:
+                    self.negative_prompts[idx] = negative_prompt
+            elif is_appending:
+                # Make sure that negative prompts are well specified.
+                self.negative_prompts.append('')
+
+            if is_appending:
+                if prompt_strength is None:
+                    prompt_strength = self.default_prompt_strength
+                self.prompt_strengths = torch.cat((
+                    self.prompt_strengths,
+                    torch.as_tensor([prompt_strength], dtype=self.dtype, device=self.device),
+                ), dim=0)
+            elif prompt_strength is not None:
+                self.prompt_strengths[idx] = prompt_strength
+
+            # Edit currently stored prompt embeddings.
+
+            if is_double_cond:
+                uncond_prompt_embed_, prompt_embed_ = torch.chunk(self.prompt_embeds, 2, dim=0)
+                uncond_prompt_embed_ = rearrange(uncond_prompt_embed_, '(t p) c1 c2 -> t p c1 c2', p=p)
+                prompt_embed_ = rearrange(prompt_embed_, '(t p) c1 c2 -> t p c1 c2', p=p)
+            else:
+                uncond_prompt_embed_ = None
+                prompt_embed_ = rearrange(self.prompt_embeds, '(t p) c1 c2 -> t p c1 c2', p=p)
+
+            e = self.pipe.encode_prompt(
+                prompt=self.prompts[idx],
+                device=self.device,
+                num_images_per_prompt=1,
+                do_classifier_free_guidance=(self.guidance_scale > 1.0),
+                negative_prompt=self.negative_prompts[idx],
+            )  # (1, 77, 768), (1, 77, 768)
+
+            s = self.prompt_strengths[idx]
+            t = prompt_embed_.shape[0]
+            prompt_embed = torch.lerp(self.background.embed[0], e[0], s)[None].repeat(t, 1, 1, 1)  # (1, 77, 768)
+            if is_double_cond:
+                uncond_prompt_embed = torch.lerp(self.background.embed[1], e[1], s)[None].repeat(t, 1, 1, 1)  # (1, 77, 768)
+
+            if is_appending:
+                prompt_embed_ = torch.cat((prompt_embed_, prompt_embed), dim=1)
+                if is_double_cond:
+                    uncond_prompt_embed_ = torch.cat((uncond_prompt_embed_, uncond_prompt_embed), dim=1)
+            else:
+                prompt_embed_[:, idx:(idx + 1)] = prompt_embed
+                if is_double_cond:
+                    uncond_prompt_embed_[:, idx:(idx + 1)] = uncond_prompt_embed
+
+            self.prompt_embeds = rearrange(prompt_embed_, 't p c1 c2 -> (t p) c1 c2')
+            if is_double_cond:
+                uncond_prompt_embeds = rearrange(uncond_prompt_embed_, 't p c1 c2 -> (t p) c1 c2')
+                self.prompt_embeds = torch.cat([uncond_prompt_embeds, self.prompt_embeds], dim=0)  # (2 * T * p, 77, 768)
+
+            self.ready_checklist['flushed'] = False
+
+        if is_appending:
+            p = self.num_layers
+            self.sub_timesteps_tensor_ = self.sub_timesteps_tensor.repeat_interleave(p)  # (T * p,)
+            self.init_noise_ = self.init_noise.repeat_interleave(p, dim=0)  # (T * p, 77, 768)
+            self.stock_noise_ = self.stock_noise.repeat_interleave(p, dim=0)  # (T * p, 77, 768)
+            self.c_out_ = self.c_out.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+            self.c_skip_ = self.c_skip.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+            self.beta_prod_t_sqrt_ = self.beta_prod_t_sqrt.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+            self.alpha_prod_t_sqrt_ = self.alpha_prod_t_sqrt.repeat_interleave(p, dim=0)  # (T * p, 1, 1, 1)
+
+        ### Register new masks
+
+        # | mask      | std / str | append mode (idx==len)       | edit mode (idx<len)           |
+        # | --------- | --------- | ---------------------------- | ----------------------------- |
+        # | given     | given     | create mask with given val   | create mask with given val    |
+        # | given     | not given | create mask with default val | create mask with existing val |
+        # | not given | given     | create blank mask            | replace mask with given val   |
+        # | not given | not given | create blank mask            | do nothing                    |
+
+        flag_nonzero_mask = False
+        if mask is not None:
+            # Mask image is given -> create mask.
+            mask, strength, std, original_mask = self.process_mask(mask, mask_strength, mask_std)
+            flag_nonzero_mask = True
+
+        elif is_appending:
+            # No given mask & append mode  -> create white mask.
+            mask = torch.zeros(
+                (1, self.batch_size, 1, self.latent_height, self.latent_width),
+                dtype=self.dtype,
+                device=self.device,
+            )
+            strength = torch.as_tensor([self.default_mask_strength], dtype=self.dtype, device=self.device)
+            std = torch.as_tensor([self.default_mask_std], dtype=self.dtype, device=self.device)
+            original_mask = torch.zeros((1, 1, self.latent_height, self.latent_width), dtype=self.dtype)
+
+        elif mask_std is not None or mask_strength is not None:
+            # No given mask & edit mode & given std / str -> replace existing mask with given std / str.
+            if mask_std is None:
+                mask_std = self.mask_stds[idx:(idx + 1)]
+            if mask_strength is None:
+                mask_strength = self.mask_strengths[idx:(idx + 1)]
+            mask, strength, std, original_mask = self.process_mask(
+                self.original_masks[idx:(idx + 1)], mask_strength, mask_std)
+            flag_nonzero_mask = True
+
+        else:
+            # No given mask & no given std & edit mode -> Do nothing.
+            return
+
+        if is_appending:
+            # Append mode.
+            self.masks = torch.cat((self.masks, mask), dim=0)  # (p, T, 1, h, w)
+            self.mask_strengths = torch.cat((self.mask_strengths, strength), dim=0)  # (p,)
+            self.mask_stds = torch.cat((self.mask_stds, std), dim=0)  # (p,)
+            self.original_masks = torch.cat((self.original_masks, original_mask), dim=0)  # (p, 1, h, w)
+            if flag_nonzero_mask:
+                self.counts = self.counts + mask[0] if hasattr(self, 'counts') else mask[0]  # (T, 1, h, w)
+                self.bg_mask = (1 - self.counts).clip_(0, 1)  # (T, 1, h, w)
+        else:
+            # Edit mode.
+            if flag_nonzero_mask:
+                self.counts = self.counts - self.masks[idx] + mask[0]  # (T, 1, h, w)
+                self.bg_mask = (1 - self.counts).clip_(0, 1)  # (T, 1, h, w) 
+            self.masks[idx:(idx + 1)] = mask  # (p, T, 1, h, w)
+            self.mask_strengths[idx:(idx + 1)] = strength  # (p,)
+            self.mask_stds[idx:(idx + 1)] = std  # (p,)
+            self.original_masks[idx:(idx + 1)] = original_mask  # (p, 1, h, w)
+
+        # if flag_nonzero_mask:
+        #     self.ready_checklist['flushed'] = False
+
+    @torch.no_grad()
+    def register_all(
+        self,
+        prompts: Union[str, List[str]],
+        masks: Union[Image.Image, List[Image.Image]],
+        background: Image.Image,
+        background_prompt: Optional[str] = None,
+        background_negative_prompt: str = '',
+        negative_prompts: Union[str, List[str]] = '',
+        suffix: Optional[str] = None, #', background is ',
+        prompt_strengths: float = 1.0,
+        mask_strengths: float = 1.0,
+        mask_stds: Union[torch.Tensor, float] = 10.0,
+    ) -> None:
+        # The order of this registration should not be changed!
+        self.update_background(background, background_prompt, background_negative_prompt)
+        self.update_layers(prompts, negative_prompts, suffix, prompt_strengths, masks, mask_strengths, mask_stds)
+
+    def update(
+        self,
+        background: Optional[Image.Image] = None,
+        background_prompt: Optional[str] = None,
+        background_negative_prompt: Optional[str] = None,
+        idx: Optional[int] = None,
+        prompt: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+        suffix: Optional[str] = None,
+        prompt_strength: Optional[float] = None,
+        mask: Optional[Union[torch.Tensor, Image.Image]] = None,
+        mask_strength: Optional[float] = None,
+        mask_std: Optional[float] = None,
+    ) -> None:
+        # For lazy update (to solve minor synchonization problem with gradio).
+        bq = BackgroundObject(
+            image=background,
+            prompt=background_prompt,
+            negative_prompt=background_negative_prompt,
+        )
+        if not bq.is_empty:
+            self.update_buffer['background'] = bq
+
+        lq = LayerObject(
+            idx=idx,
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            suffix=suffix,
+            prompt_strength=prompt_strength,
+            mask=mask,
+            mask_strength=mask_strength,
+            mask_std=mask_std,
+        )
+        if not lq.is_empty:
+            limit = self.update_buffer['layers'].maxlen
+
+            # Optimize the prompt queue: Overrride uncommitted layers with the same idx.
+            new_q = deque(maxlen=limit)
+            for _ in range(len(self.update_buffer['layers'])):
+                # Check from the newest to the oldest.
+                # Copy old requests only if the current query does not carry those requests.
+                query = self.update_buffer['layers'].pop()
+                overriden = lq.merge(query)
+                if not overriden:
+                    new_q.appendleft(query)
+            self.update_buffer['layers'] = new_q
+
+            if len(self.update_buffer['layers']) == limit:
+                print(f'[WARNING]  Maximum prompt change query limit ({limit}) is reached. '
+                      f'Current query {lq} will be ignored.')
+            else:
+                self.update_buffer['layers'].append(lq)
+
+    @torch.no_grad()
+    def commit(self) -> None:
+        flag_changed = self.is_dirty
+        bq = self.update_buffer['background']
+        lq = self.update_buffer['layers']
+        count_bq_req = int(bq is not None and not bq.is_empty)
+        count_lq_req = len(lq)
+
+        if flag_changed:
+            print(f'[INFO]     Requests found: {count_bq_req} background requests '
+                  f'& {count_lq_req} layer requests:\n{str(bq)}, {", ".join([str(l) for l in lq])}')
+
+        bq = self.update_buffer['background']
+        if bq is not None:
+            self.update_background(**vars(bq))
+            self.update_buffer['background'] = None
+
+        while len(lq) > 0:
+            l = lq.popleft()
+            self.update_single_layer(**vars(l))
+
+        if flag_changed:
+            print(f'[INFO]     Requests resolved: {count_bq_req} background requests '
+                  f'& {count_lq_req} layer requests.')
+
+    def scheduler_step_batch(
+        self,
+        model_pred_batch: torch.Tensor,
+        x_t_latent_batch: torch.Tensor,
+        idx: Optional[int] = None,
+    ) -> torch.Tensor:
+        r"""Denoise-only step for reverse diffusion scheduler.
+
+        Args:
+            model_pred_batch (torch.Tensor): Noise prediction results.
+            x_t_latent_batch (torch.Tensor): Noisy latent.
+            idx (Optional[int]): Instead of timesteps (in [0, 1000]-scale) use
+                indices for the timesteps tensor (ranged in
+                [0, len(timesteps)-1]). Specify only if a single-index, not
+                stream-batched inference is what you want.
+
+        Returns:
+            A denoised tensor with the same size as latent.
+        """
+        if idx is None:
+            F_theta = (x_t_latent_batch - self.beta_prod_t_sqrt_ * model_pred_batch) / self.alpha_prod_t_sqrt_
+            denoised_batch = self.c_out_ * F_theta + self.c_skip_ * x_t_latent_batch
+        else:
+            F_theta = (x_t_latent_batch - self.beta_prod_t_sqrt[idx] * model_pred_batch) / self.alpha_prod_t_sqrt[idx]
+            denoised_batch = self.c_out[idx] * F_theta + self.c_skip[idx] * x_t_latent_batch
+        return denoised_batch
+
+    def unet_step(
+        self,
+        x_t_latent: torch.Tensor,  # (T, 4, h, w)
+        idx: Optional[int] = None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        p = self.num_layers
+        x_t_latent = x_t_latent.repeat_interleave(p, dim=0)  # (T * p, 4, h, w)
+
+        if self.bootstrap_steps[0] > 0:
+            # Background bootstrapping.
+            bootstrap_latent = self.scheduler.add_noise(
+                self.bootstrap_latent,
+                self.stock_noise,
+                torch.tensor(self.sub_timesteps_tensor, device=self.device),
+            )
+            x_t_latent = rearrange(x_t_latent, '(t p) c h w -> p t c h w', p=p)
+            bootstrap_mask = (
+                self.masks * self.bootstrap_steps[None, :, None, None, None]
+                + (1.0 - self.bootstrap_steps[None, :, None, None, None])
+            ) # (p, t, c, h, w)
+            x_t_latent = (1.0 - bootstrap_mask) * bootstrap_latent[None] + bootstrap_mask * x_t_latent
+            x_t_latent = rearrange(x_t_latent, 'p t c h w -> (t p) c h w')
+
+            # Centering.
+            x_t_latent = shift_to_mask_bbox_center(x_t_latent, rearrange(self.masks, 'p t c h w -> (t p) c h w'), reverse=True)
+
+        t_list = self.sub_timesteps_tensor_  # (T * p,)
+        if self.guidance_scale > 1.0 and self.cfg_type == 'initialize':
+            x_t_latent_plus_uc = torch.concat([x_t_latent[:p], x_t_latent], dim=0)  # (T * p + 1, 4, h, w)
+            t_list = torch.concat([t_list[:p], t_list], dim=0)  # (T * p + 1, 4, h, w)
+        elif self.guidance_scale > 1.0 and self.cfg_type == 'full':
+            x_t_latent_plus_uc = torch.concat([x_t_latent, x_t_latent], dim=0)  # (2 * T * p, 4, h, w)
+            t_list = torch.concat([t_list, t_list], dim=0)  # (2 * T * p,)
+        else:
+            x_t_latent_plus_uc = x_t_latent  # (T * p, 4, h, w)
+
+        model_pred = self.unet(
+            x_t_latent_plus_uc,  # (B, 4, h, w)
+            t_list,  # (B,)
+            encoder_hidden_states=self.prompt_embeds,  # (B, 77, 768)
+            return_dict=False,
+            # TODO: Add SDXL Support.
+            # added_cond_kwargs={'text_embeds': add_text_embeds, 'time_ids': add_time_ids},
+        )[0]  # (B, 4, h, w)
+
+        if self.bootstrap_steps[0] > 0:
+            # Uncentering.
+            bootstrap_mask = rearrange(self.masks, 'p t c h w -> (t p) c h w')
+            if self.guidance_scale > 1.0 and self.cfg_type == 'initialize':
+                bootstrap_mask_ = torch.concat([bootstrap_mask[:p], bootstrap_mask], dim=0)
+            elif self.guidance_scale > 1.0 and self.cfg_type == 'full':
+                bootstrap_mask_ = torch.concat([bootstrap_mask, bootstrap_mask], dim=0)
+            else:
+                bootstrap_mask_ = bootstrap_mask
+            model_pred = shift_to_mask_bbox_center(model_pred, bootstrap_mask_)
+            x_t_latent = shift_to_mask_bbox_center(x_t_latent, bootstrap_mask)
+
+            # # Remove leakage (optional).
+            # leak = (latent_ - bg_latent_).pow(2).mean(dim=1, keepdim=True)
+            # leak_sigmoid = torch.sigmoid(leak / self.bootstrap_leak_sensitivity) * 2 - 1
+            # fg_mask_ = fg_mask_ * leak_sigmoid
+
+        ### noise_pred_text, noise_pred_uncond: (T * p, 4, h, w)
+        ### self.stock_noise, init_noise: (T, 4, h, w)
+
+        if self.guidance_scale > 1.0 and self.cfg_type == 'initialize':
+            noise_pred_text = model_pred[p:]
+            self.stock_noise_ = torch.concat([model_pred[:p], self.stock_noise_[p:]], dim=0)
+        elif self.guidance_scale > 1.0 and self.cfg_type == 'full':
+            noise_pred_uncond, noise_pred_text = model_pred.chunk(2)
+        else:
+            noise_pred_text = model_pred
+        if self.guidance_scale > 1.0 and self.cfg_type in ('self', 'initialize'):
+            noise_pred_uncond = self.stock_noise_ * self.delta
+
+        if self.guidance_scale > 1.0 and self.cfg_type != 'none':
+            model_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
+        else:
+            model_pred = noise_pred_text
+
+        # compute the previous noisy sample x_t -> x_t-1
+        denoised_batch = self.scheduler_step_batch(model_pred, x_t_latent, idx)
+
+        if self.cfg_type in ('self' , 'initialize'):
+            scaled_noise = self.beta_prod_t_sqrt_ * self.stock_noise_
+            delta_x = self.scheduler_step_batch(model_pred, scaled_noise, idx)
+
+            # Do mask edit.
+            alpha_next = torch.concat([self.alpha_prod_t_sqrt_[p:], torch.ones_like(self.alpha_prod_t_sqrt_[:p])], dim=0)
+            delta_x = alpha_next * delta_x
+            beta_next = torch.concat([self.beta_prod_t_sqrt_[p:], torch.ones_like(self.beta_prod_t_sqrt_[:p])], dim=0)
+            delta_x = delta_x / beta_next
+            init_noise = torch.concat([self.init_noise_[p:], self.init_noise_[:p]], dim=0)
+            self.stock_noise_ = init_noise + delta_x
+
+        p2 = len(self.t_list) - 1
+        background = torch.concat([
+            self.scheduler.add_noise(
+                self.background.latent.repeat(p2, 1, 1, 1),
+                self.stock_noise[1:],
+                torch.tensor(self.t_list[1:], device=self.device),
+            ),
+            self.background.latent,
+        ], dim=0)
+
+        denoised_batch = rearrange(denoised_batch, '(t p) c h w -> p t c h w', p=p)
+        latent = (self.masks * denoised_batch).sum(dim=0)  # (T, 4, h, w)
+        latent = torch.where(self.counts > 0, latent / self.counts, latent)
+
+        # latent = (
+        #     (1 - self.bg_mask) * self.mask_strengths * latent
+        #     + ((1 - self.bg_mask) * (1.0 - self.mask_strengths) + self.bg_mask) * background
+        # )
+        latent = (1 - self.bg_mask) * latent + self.bg_mask * background
+
+        return latent
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        no_decode: bool = False,
+        ignore_check_ready: bool = False,
+    ) -> Optional[Union[torch.Tensor, Image.Image]]:
+        if not ignore_check_ready and not self.check_ready():
+            return
+        if not ignore_check_ready and self.is_dirty:
+            print("I'm so dirty now!")
+            self.commit()
+            self.flush()
+
+        latent = torch.randn((1, self.unet.config.in_channels, self.latent_height, self.latent_width),
+            dtype=self.dtype, device=self.device)  # (1, 4, h, w)
+        latent = torch.cat((latent, self.x_t_latent_buffer), dim=0)  # (t, 4, h, w)
+        self.stock_noise = torch.cat((self.init_noise[:1], self.stock_noise[:-1]), dim=0)  # (t, 4, h, w)
+        if self.cfg_type in ('self', 'initialize'):
+            self.stock_noise_ = self.stock_noise.repeat_interleave(self.num_layers, dim=0)  # (T * p, 77, 768)
+
+        x_0_pred_batch = self.unet_step(latent)
+
+        latent = x_0_pred_batch[-1:]
+        self.x_t_latent_buffer = (
+            self.alpha_prod_t_sqrt[1:] * x_0_pred_batch[:-1]
+            + self.beta_prod_t_sqrt[1:] * self.init_noise[1:]
+        )
+
+        # For pipeline flushing.
+        if no_decode:
+            return latent
+
+        imgs = self.decode_latents(latent.half())  # (1, 3, H, W)
+        img = T.ToPILImage()(imgs[0].cpu())
+        return img
+
+    def flush(self) -> None:
+        for _ in self.t_list:
+            self(True, True)
+        self.ready_checklist['flushed'] = True

prompt_util.py

@@ -0,0 +1,154 @@
+from typing import Dict, List, Tuple, Union
+
+
+quality_prompt_list = [
+    {
+        "name": "(None)",
+        "prompt": "{prompt}",
+        "negative_prompt": "nsfw, lowres",
+    },
+    {
+        "name": "Standard v3.0",
+        "prompt": "{prompt}, masterpiece, best quality",
+        "negative_prompt": "nsfw, lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name",
+    },
+    {
+        "name": "Standard v3.1",
+        "prompt": "{prompt}, masterpiece, best quality, very aesthetic, absurdres",
+        "negative_prompt": "nsfw, lowres, (bad), text, error, fewer, extra, missing, worst quality, jpeg artifacts, low quality, watermark, unfinished, displeasing, oldest, early, chromatic aberration, signature, extra digits, artistic error, username, scan, [abstract]",
+    },
+    {
+        "name": "Light v3.1",
+        "prompt": "{prompt}, (masterpiece), best quality, very aesthetic, perfect face",
+        "negative_prompt": "nsfw, (low quality, worst quality:1.2), very displeasing, 3d, watermark, signature, ugly, poorly drawn",
+    },
+    {
+        "name": "Heavy v3.1",
+        "prompt": "{prompt}, (masterpiece), (best quality), (ultra-detailed), very aesthetic, illustration, disheveled hair, perfect composition, moist skin, intricate details",
+        "negative_prompt": "nsfw, longbody, lowres, bad anatomy, bad hands, missing fingers, pubic hair, extra digit, fewer digits, cropped, worst quality, low quality, very displeasing",
+    },
+]
+
+style_list = [
+    {
+        "name": "(None)",
+        "prompt": "{prompt}",
+        "negative_prompt": "",
+    },
+    {
+        "name": "Cinematic",
+        "prompt": "{prompt}, cinematic still, emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
+        "negative_prompt": "nsfw, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
+    },
+    {
+        "name": "Photographic",
+        "prompt": "{prompt}, cinematic photo, 35mm photograph, film, bokeh, professional, 4k, highly detailed",
+        "negative_prompt": "nsfw, drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly",
+    },
+    {
+        "name": "Anime",
+        "prompt": "{prompt}, anime artwork, anime style, key visual, vibrant, studio anime, highly detailed",
+        "negative_prompt": "nsfw, photo, deformed, black and white, realism, disfigured, low contrast",
+    },
+    {
+        "name": "Manga",
+        "prompt": "{prompt}, manga style, vibrant, high-energy, detailed, iconic, Japanese comic style",
+        "negative_prompt": "nsfw, ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style",
+    },
+    {
+        "name": "Digital Art",
+        "prompt": "{prompt}, concept art, digital artwork, illustrative, painterly, matte painting, highly detailed",
+        "negative_prompt": "nsfw, photo, photorealistic, realism, ugly",
+    },
+    {
+        "name": "Pixel art",
+        "prompt": "{prompt}, pixel-art, low-res, blocky, pixel art style, 8-bit graphics",
+        "negative_prompt": "nsfw, sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic",
+    },
+    {
+        "name": "Fantasy art",
+        "prompt": "{prompt}, ethereal fantasy concept art, magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy",
+        "negative_prompt": "nsfw, photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white",
+    },
+    {
+        "name": "Neonpunk",
+        "prompt": "{prompt}, neonpunk style, cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional",
+        "negative_prompt": "nsfw, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",
+    },
+    {
+        "name": "3D Model",
+        "prompt": "{prompt}, professional 3d model, octane render, highly detailed, volumetric, dramatic lighting",
+        "negative_prompt": "nsfw, ugly, deformed, noisy, low poly, blurry, painting",
+    },
+]
+
+
+_style_dict = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
+_quality_dict = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in quality_prompt_list}
+
+
+def preprocess_prompt(
+    positive: str,
+    negative: str = "",
+    style_dict: Dict[str, dict] = _quality_dict,
+    style_name: str = "Standard v3.1", # "Heavy v3.1"
+    add_style: bool = True,
+) -> Tuple[str, str]:
+    p, n = style_dict.get(style_name, style_dict["(None)"])
+
+    if add_style and positive.strip():
+        formatted_positive = p.format(prompt=positive)
+    else:
+        formatted_positive = positive
+
+    combined_negative = n
+    if negative.strip():
+        if combined_negative:
+            combined_negative += ", " + negative
+        else:
+            combined_negative = negative
+
+    return formatted_positive, combined_negative
+
+
+def preprocess_prompts(
+    positives: List[str],
+    negatives: List[str] = None,
+    style_dict = _style_dict,
+    style_name: str = "Manga", # "(None)"
+    quality_dict = _quality_dict,
+    quality_name: str = "Standard v3.1", # "Heavy v3.1"
+    add_style: bool = True,
+    add_quality_tags = True,
+) -> Tuple[List[str], List[str]]:
+    if negatives is None:
+        negatives = ['' for _ in positives]
+
+    positives_ = []
+    negatives_ = []
+    for pos, neg in zip(positives, negatives):
+        pos, neg = preprocess_prompt(pos, neg, quality_dict, quality_name, add_quality_tags)
+        pos, neg = preprocess_prompt(pos, neg, style_dict, style_name, add_style)
+        positives_.append(pos)
+        negatives_.append(neg)
+    return positives_, negatives_
+
+
+def print_prompts(
+    positives: Union[str, List[str]],
+    negatives: Union[str, List[str]],
+    has_background: bool = False,
+) -> None:
+    if isinstance(positives, str):
+        positives = [positives]
+    if isinstance(negatives, str):
+        negatives = [negatives]
+
+    for i, prompt in enumerate(positives):
+        prefix = ((f'Prompt{i}' if i > 0 else 'Background Prompt')
+                  if has_background else f'Prompt{i + 1}')
+        print(prefix + ': ' + prompt)
+    for i, prompt in enumerate(negatives):
+        prefix = ((f'Negative Prompt{i}' if i > 0 else 'Background Negative Prompt')
+                  if has_background else f'Negative Prompt{i + 1}')
+        print(prefix + ': ' + prompt)

requirements.txt

@@ -0,0 +1,13 @@
+torch==2.0.1
+torchvision
+xformers==0.0.22
+einops
+diffusers
+transformers
+huggingface_hub[torch]
+Pillow
+emoji
+numpy
+tqdm
+jupyterlab
+gradio @ https://gradio-builds.s3.amazonaws.com/7129aa5719aaa95a75397a83d3e1f3b72adf8050/gradio-4.26.0-py3-none-any.whl

timer/LICENSE_timer.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Jonathan Trancozo (https://codepen.io/jtrancozo/pen/mEoEVw)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

timer/index.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html>
+<html lang="en" >
+<head>
+  <meta charset="UTF-8">
+  <title>CodePen - #2 - Project Deadline - SVG animation with CSS3</title>
+  <link href='https://fonts.googleapis.com/css?family=Oswald' rel='stylesheet' type='text/css'>
+
+<meta property="og:image" content="https://i.imgur.com/9xiPyyv.png" /><link rel="stylesheet" href="./style.css">
+
+</head>
+<body>
+<!-- partial:index.partial.html -->
+<div id="deadline">
+  <svg preserveAspectRatio="none" id="line" viewBox="0 0 581 158" enable-background="new 0 0 581 158">
+    <g id="fire">
+      <rect id="mask-fire-black" x="511" y="41" width="38" height="34"/>
+      <g>
+        <defs>
+          <rect id="mask_fire" x="511" y="41" width="38" height="34"/>
+        </defs>
+        <clipPath id="mask-fire_1_">
+          <use xlink:href="#mask_fire"  overflow="visible"/>
+        </clipPath>
+        <g id="group-fire" clip-path="url(#mask-fire_1_)">
+          <path id="red-flame" fill="#B71342" d="M528.377,100.291c6.207,0,10.947-3.272,10.834-8.576 c-0.112-5.305-2.934-8.803-8.237-10.383c-5.306-1.581-3.838-7.9-0.79-9.707c-7.337,2.032-7.581,5.891-7.11,8.238 c0.789,3.951,7.56,4.402,5.077,9.48c-2.482,5.079-8.012,1.129-6.319-2.257c-2.843,2.233-4.78,6.681-2.259,9.703 C521.256,98.809,524.175,100.291,528.377,100.291z"/>
+          <path id="yellow-flame" opacity="0.71" fill="#F7B523" d="M528.837,100.291c4.197,0,5.108-1.854,5.974-5.417 c0.902-3.724-1.129-6.207-5.305-9.931c-2.396-2.137-1.581-4.176-0.565-6.32c-4.401,1.918-3.384,5.304-2.482,6.658 c1.511,2.267,2.099,2.364,0.42,5.8c-1.679,3.435-5.42,0.764-4.275-1.527c-1.921,1.512-2.373,4.04-1.528,6.563 C522.057,99.051,525.994,100.291,528.837,100.291z"/>
+          <path id="white-flame" opacity="0.81" fill="#FFFFFF" d="M529.461,100.291c-2.364,0-4.174-1.322-4.129-3.469 c0.04-2.145,1.117-3.56,3.141-4.198c2.022-0.638,1.463-3.195,0.302-3.925c2.798,0.821,2.89,2.382,2.711,3.332 c-0.301,1.597-2.883,1.779-1.938,3.834c0.912,1.975,3.286,0.938,2.409-0.913c1.086,0.903,1.826,2.701,0.864,3.924 C532.18,99.691,531.064,100.291,529.461,100.291z"/>
+        </g>
+      </g>
+    </g>
+    <g id="progress-trail">
+      <path fill="#FFFFFF" d="M491.979,83.878c1.215-0.73-0.62-5.404-3.229-11.044c-2.583-5.584-5.034-10.066-7.229-8.878
+                              c-2.854,1.544-0.192,6.286,2.979,11.628C487.667,80.917,490.667,84.667,491.979,83.878z"/>
+      <path fill="#FFFFFF" d="M571,76v-5h-23.608c0.476-9.951-4.642-13.25-4.642-13.25l-3.125,4c0,0,3.726,2.7,3.625,5.125
+                              c-0.071,1.714-2.711,3.18-4.962,4.125H517v5h10v24h-25v-5.666c0,0,0.839,0,2.839-0.667s6.172-3.667,4.005-6.333
+                              s-7.49,0.333-9.656,0.166s-6.479-1.5-8.146,1.917c-1.551,3.178,0.791,5.25,5.541,6.083l-0.065,4.5H16c-2.761,0-5,2.238-5,5v17
+                              c0,2.762,2.239,5,5,5h549c2.762,0,5-2.238,5-5v-17c0-2.762-2.238-5-5-5h-3V76H571z"/>
+      <path fill="#FFFFFF" d="M535,65.625c1.125,0.625,2.25-1.125,2.25-1.125l11.625-22.375c0,0,0.75-0.875-1.75-2.125
+                              s-3.375,0.25-3.375,0.25s-8.75,21.625-9.875,23.5S533.875,65,535,65.625z"/>
+    </g>
+    <g>
+      <defs>
+        <path id="SVGID_1_" d="M484.5,75.584c-3.172-5.342-5.833-10.084-2.979-11.628c2.195-1.188,4.646,3.294,7.229,8.878
+                               c2.609,5.64,4.444,10.313,3.229,11.044C490.667,84.667,487.667,80.917,484.5,75.584z M571,76v-5h-23.608
+                               c0.476-9.951-4.642-13.25-4.642-13.25l-3.125,4c0,0,3.726,2.7,3.625,5.125c-0.071,1.714-2.711,3.18-4.962,4.125H517v5h10v24h-25
+                               v-5.666c0,0,0.839,0,2.839-0.667s6.172-3.667,4.005-6.333s-7.49,0.333-9.656,0.166s-6.479-1.5-8.146,1.917
+                               c-1.551,3.178,0.791,5.25,5.541,6.083l-0.065,4.5H16c-2.761,0-5,2.238-5,5v17c0,2.762,2.239,5,5,5h549c2.762,0,5-2.238,5-5v-17
+                               c0-2.762-2.238-5-5-5h-3V76H571z M535,65.625c1.125,0.625,2.25-1.125,2.25-1.125l11.625-22.375c0,0,0.75-0.875-1.75-2.125
+                               s-3.375,0.25-3.375,0.25s-8.75,21.625-9.875,23.5S533.875,65,535,65.625z"/>
+      </defs>
+      <clipPath id="SVGID_2_">
+        <use xlink:href="#SVGID_1_"  overflow="visible"/>
+      </clipPath>
+      <rect id="progress-time-fill" x="-100%" y="34" clip-path="url(#SVGID_2_)" fill="#BE002A" width="586" height="103"/>
+    </g>
+
+    <g id="death-group">
+      <path id="death" fill="#BE002A" d="M-46.25,40.416c-5.42-0.281-8.349,3.17-13.25,3.918c-5.716,0.871-10.583-0.918-10.583-0.918
+                                         C-67.5,49-65.175,50.6-62.083,52c5.333,2.416,4.083,3.5,2.084,4.5c-16.5,4.833-15.417,27.917-15.417,27.917L-75.5,84.75
+                                         c-1,12.25-20.25,18.75-20.25,18.75s39.447,13.471,46.25-4.25c3.583-9.333-1.553-16.869-1.667-22.75
+                                         c-0.076-3.871,2.842-8.529,6.084-12.334c3.596-4.22,6.958-10.374,6.958-15.416C-38.125,43.186-39.833,40.75-46.25,40.416z
+                                         M-40,51.959c-0.882,3.004-2.779,6.906-4.154,6.537s-0.939-4.32,0.112-7.704c0.82-2.64,2.672-5.96,3.959-5.583
+                                         C-39.005,45.523-39.073,48.8-40,51.959z"/>
+      <path id="death-arm" fill="#BE002A" d="M-53.375,75.25c0,0,9.375,2.25,11.25,0.25s2.313-2.342,3.375-2.791
+                                             c1.083-0.459,4.375-1.75,4.292-4.75c-0.101-3.627,0.271-4.594,1.333-5.043c1.083-0.457,2.75-1.666,2.75-1.666
+                                             s0.708-0.291,0.5-0.875s-0.791-2.125-1.583-2.959c-0.792-0.832-2.375-1.874-2.917-1.332c-0.542,0.541-7.875,7.166-7.875,7.166
+                                             s-2.667,2.791-3.417,0.125S-49.833,61-49.833,61s-3.417,1.416-3.417,1.541s-1.25,5.834-1.25,5.834l-0.583,5.833L-53.375,75.25z"/>
+      <path id="death-tool" fill="#BE002A" d="M-20.996,26.839l-42.819,91.475l1.812,0.848l38.342-81.909c0,0,8.833,2.643,12.412,7.414
+                                              c5,6.668,4.75,14.084,4.75,14.084s4.354-7.732,0.083-17.666C-10,32.75-19.647,28.676-19.647,28.676l0.463-0.988L-20.996,26.839z"/>
+    </g>
+    <path id="designer-body" fill="#FEFFFE" d="M514.75,100.334c0,0,1.25-16.834-6.75-16.5c-5.501,0.229-5.583,3-10.833,1.666
+                                               c-3.251-0.826-5.084-15.75-0.834-22c4.948-7.277,12.086-9.266,13.334-7.833c2.25,2.583-2,10.833-4.5,14.167
+                                               c-2.5,3.333-1.833,10.416,0.5,9.916s8.026-0.141,10,2.25c3.166,3.834,4.916,17.667,4.916,17.667l0.917,2.5l-4,0.167L514.75,100.334z
+                                               "/>
+
+    <circle id="designer-head" fill="#FEFFFE" cx="516.083" cy="53.25" r="6.083"/>
+
+    <g id="designer-arm-grop">
+      <path id="designer-arm" fill="#FEFFFE" d="M505.875,64.875c0,0,5.875,7.5,13.042,6.791c6.419-0.635,11.833-2.791,13.458-4.041s2-3.5,0.25-3.875
+                                                s-11.375,5.125-16,3.25c-5.963-2.418-8.25-7.625-8.25-7.625l-2,1.125L505.875,64.875z"/>
+      <path id="designer-pen" fill="#FEFFFE" d="M525.75,59.084c0,0-0.423-0.262-0.969,0.088c-0.586,0.375-0.547,0.891-0.547,0.891l7.172,8.984l1.261,0.453
+                                                l-0.104-1.328L525.75,59.084z"/>
+    </g>
+  </svg>
+
+  <div class="deadline-days">
+    Deadline <span class="day">7</span> <span class="days">days</span>
+  </div>
+
+</div>
+<!-- partial -->
+  <script src='https://code.jquery.com/jquery-2.2.4.min.js'></script><script  src="./script.js"></script>
+
+</body>
+</html>

timer/script.js

@@ -0,0 +1,73 @@
+// Init
+var $ = jQuery;
+var animationTime = 20,
+    days = 7;
+ 
+$(document).ready(function(){
+
+    // timer arguments: 
+    //   #1 - time of animation in mileseconds, 
+    //   #2 - days to deadline
+
+    $('#progress-time-fill, #death-group').css({'animation-duration': animationTime+'s'});
+
+    var deadlineAnimation = function () {
+        setTimeout(function(){
+            $('#designer-arm-grop').css({'animation-duration': '1.5s'});
+        },0);
+
+        setTimeout(function(){
+            $('#designer-arm-grop').css({'animation-duration': '1s'});
+        },4000);
+
+        setTimeout(function(){
+            $('#designer-arm-grop').css({'animation-duration': '0.7s'});
+        },8000);
+
+        setTimeout(function(){
+            $('#designer-arm-grop').css({'animation-duration': '0.3s'});
+        },12000);
+
+        setTimeout(function(){
+            $('#designer-arm-grop').css({'animation-duration': '0.2s'});
+        },15000);
+    };
+
+    function timer(totalTime, deadline) {
+        var time = totalTime * 1000;
+        var dayDuration = time / deadline;
+        var actualDay = deadline;
+
+        var timer = setInterval(countTime, dayDuration);
+
+        function countTime() {
+            --actualDay;
+            $('.deadline-days .day').text(actualDay);
+
+            if (actualDay == 0) {
+                clearInterval(timer);
+                $('.deadline-days .day').text(deadline);
+            }
+        }
+    }
+
+    var deadlineText = function () {
+        var $el = $('.deadline-days');
+        var html = '<div class="mask-red"><div class="inner">' + $el.html() + '</div></div><div class="mask-white"><div class="inner">' + $el.html() + '</div></div>';
+        $el.html(html);
+    };
+
+    deadlineText();
+
+    deadlineAnimation();
+    timer(animationTime, days);
+
+    setInterval(function(){
+        timer(animationTime, days);
+        deadlineAnimation();
+
+        console.log('begin interval', animationTime * 1000);
+
+    }, animationTime * 1000);
+
+});

timer/style.css

@@ -0,0 +1,504 @@
+/*
+Svg Projects
+Author: Jonathan Trancozo
+Language: HTML, CSS3 and SVG
+Project_version: V1
+Project_description: 
+[pt-br]
+Por anos eu vi essa imagem e pensava “Isso ficaria bem massa animado” e hoje consegui expressar um pouco da minha imaginação.
+O desenho foi produzido no Adobe Illustrator e exportado em SVG. As animações foram feitas com CSS3 usando principalmente [transform].
+
+Até uma próxima.
+      
+
+[en]
+For years I saw this picture and thought "That would be amazing animated " and today I managed to express some of my imagination.
+The design was produced in Adobe Illustrator and export in SVG . The animations were made with CSS3 using mainly [ transform ].
+
+See you.
+
+*/
+
+html {
+    font-size: 1em;
+    line-height: 1.4;
+}
+
+html,
+body {
+    height: 100%;
+}
+
+body {
+    margin: 0;
+    padding: 0;
+    background: transparent;
+}
+
+
+#deadline {
+    width:581px;
+    max-width: 100%;
+    height:158px;
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    z-index: 1;
+    transform: translate(-50%, -50%);
+}
+
+#deadline svg {
+    width: 100%;
+}
+
+#progress-time-fill {
+    -webkit-animation-name: progress-fill;
+            animation-name: progress-fill; 
+    -webkit-animation-timing-function: linear; 
+            animation-timing-function: linear; 
+    -webkit-animation-iteration-count: infinite; 
+            animation-iteration-count: infinite;
+}
+
+/* Death */
+#death-group {
+    -webkit-animation-name: walk;
+            animation-name: walk;
+    -webkit-animation-timing-function: ease;
+            animation-timing-function: ease; 
+    -webkit-animation-iteration-count: infinite; 
+            animation-iteration-count: infinite;
+    transform: translateX(0);
+}
+
+#death-arm {
+    -webkit-animation: move-arm 3s ease infinite;
+            animation: move-arm 3s ease infinite;
+    /* transform-origin: left center; */
+    transform-origin: -60px 74px;
+}
+
+#death-tool {
+    -webkit-animation: move-tool 3s ease infinite;
+            animation: move-tool 3s ease infinite;
+    transform-origin: -48px center;
+}
+
+/* Designer */
+
+#designer-arm-grop {
+    -webkit-animation: write 1.5s ease infinite;
+            animation: write 1.5s ease infinite;
+    transform: translate(0, 0) rotate(0deg) scale(1, 1);
+    transform-origin: 90% top;
+}
+
+.deadline-timer {
+    color: #fff;
+    text-align: center;
+    width: 200px;
+    margin: 0 auto;
+    position: relative;
+    height: 40px;
+    font-family: 'Oswald', sans-serif;
+    font-size: 18pt;
+    margin-top: -90px;
+}
+
+.deadline-timer .inner {
+    width: 200px;
+    position: relative;
+    top: 0;
+    left: 0;
+}
+
+.mask-red,
+.mask-white {
+    position: absolute;
+    top: 0;
+    width: 100%;
+    overflow: hidden;
+    height: 100%;
+}
+
+@-webkit-keyframes progress-fill {
+    0% {
+        x: -100%;
+    }
+
+    100% {
+        x: -3%;
+    }
+}
+
+@keyframes progress-fill {
+    0% {
+        x: -100%;
+    }
+
+    100% {
+        x: -3%;
+    }
+}
+
+@-webkit-keyframes walk {
+    0% {
+        transform: translateX(0);
+    }
+    6% {
+        transform: translateX(0);
+    }
+    10% {
+        transform: translateX(100px);
+    },
+
+    15% {
+        transform: translateX(140px);
+    }
+
+    25% {
+        transform: translateX(170px);
+    }
+
+    35% {
+        transform: translateX(220px);
+    }
+
+    45% {
+        transform: translateX(280px);
+    }
+
+    55% {
+        transform: translateX(340px);
+    }
+
+    65% {
+        transform: translateX(370px);
+    }
+
+    75% {
+        transform: translateX(430px);
+    }
+
+    85% {
+        transform: translateX(460px);
+    }
+
+    100% {
+        transform: translateX(520px);
+    }
+}
+
+@keyframes walk {
+    0% {
+        transform: translateX(0);
+    }
+    6% {
+        transform: translateX(0);
+    }
+    10% {
+        transform: translateX(100px);
+    },
+
+    15% {
+        transform: translateX(140px);
+    }
+
+    25% {
+        transform: translateX(170px);
+    }
+
+    35% {
+        transform: translateX(220px);
+    }
+
+    45% {
+        transform: translateX(280px);
+    }
+
+    55% {
+        transform: translateX(340px);
+    }
+
+    65% {
+        transform: translateX(370px);
+    }
+
+    75% {
+        transform: translateX(430px);
+    }
+
+    85% {
+        transform: translateX(460px);
+    }
+
+    100% {
+        transform: translateX(520px);
+    }
+}
+
+@-webkit-keyframes move-arm {
+    0% {
+        transform: rotate(0);
+    }
+
+    5% {
+        transform: rotate(0);
+    }
+
+    9% {
+        transform: rotate(40deg);
+    }
+
+    80% {
+        transform: rotate(0);
+    }
+}
+
+@keyframes move-arm {
+    0% {
+        transform: rotate(0);
+    }
+
+    5% {
+        transform: rotate(0);
+    }
+
+    9% {
+        transform: rotate(40deg);
+    }
+
+    80% {
+        transform: rotate(0);
+    }
+}
+
+@-webkit-keyframes move-tool {
+    0% {
+        transform: rotate(0);
+    }
+
+    5% {
+        transform: rotate(0);
+    }
+
+    9% {
+        transform: rotate(50deg);
+    }
+
+    80% {
+        transform: rotate(0);
+    }
+}
+
+@keyframes move-tool {
+    0% {
+        transform: rotate(0);
+    }
+
+    5% {
+        transform: rotate(0);
+    }
+
+    9% {
+        transform: rotate(50deg);
+    }
+
+    80% {
+        transform: rotate(0);
+    }
+}
+
+/* Design animations */
+
+@-webkit-keyframes write {
+    0% {
+        transform: translate(0, 0) rotate(0deg) scale(1, 1);
+    }
+
+    16% {
+        transform: translate(0px, 0px) rotate(5deg) scale(0.8, 1);
+    }
+
+    32% {
+        transform: translate(0px, 0px) rotate(0deg) scale(1, 1);
+    }
+
+    48% {
+        transform: translate(0px, 0px) rotate(6deg) scale(0.8, 1);
+    }
+
+    65% {
+        transform: translate(0px, 0px) rotate(0deg) scale(1, 1);
+    }
+
+    83% {
+        transform: translate(0px, 0px) rotate(4deg) scale(0.8, 1);
+    }
+}
+
+@keyframes write {
+    0% {
+        transform: translate(0, 0) rotate(0deg) scale(1, 1);
+    }
+
+    16% {
+        transform: translate(0px, 0px) rotate(5deg) scale(0.8, 1);
+    }
+
+    32% {
+        transform: translate(0px, 0px) rotate(0deg) scale(1, 1);
+    }
+
+    48% {
+        transform: translate(0px, 0px) rotate(6deg) scale(0.8, 1);
+    }
+
+    65% {
+        transform: translate(0px, 0px) rotate(0deg) scale(1, 1);
+    }
+
+    83% {
+        transform: translate(0px, 0px) rotate(4deg) scale(0.8, 1);
+    }
+}
+
+@-webkit-keyframes text-red {
+    0% {
+        width: 0%;
+    }
+
+    100% {
+        width: 98%;
+    }
+}
+
+@keyframes text-red {
+    0% {
+        width: 0%;
+    }
+
+    100% {
+        width: 98%;
+    }
+}
+
+/* Flames */
+
+/* @keyframes show-flames {
+    0% {
+        transform:  translateY(0);
+    }
+    74% {
+        transform:  translateY(0);
+    }
+    80% {
+        transform:  translateY(-30px);
+    }
+    97% {
+        transform:  translateY(-30px);
+    }
+    100% {
+        transform: translateY(0px);
+    }
+} */
+
+@-webkit-keyframes show-flames {
+    0% {
+        opacity: 0;
+    }
+    74% {
+        opacity: 0;
+    }
+    80% {
+        opacity: 1;
+    }
+    99% {
+        opacity: 1;
+    }
+    100% {
+        opacity: 0;
+    }
+}
+
+@keyframes show-flames {
+    0% {
+        opacity: 0;
+    }
+    74% {
+        opacity: 0;
+    }
+    80% {
+        opacity: 1;
+    }
+    99% {
+        opacity: 1;
+    }
+    100% {
+        opacity: 0;
+    }
+}
+
+@-webkit-keyframes red-flame {
+    0% {
+        transform: translateY(-30px) scale(1, 1);
+    }
+
+    25% {
+        transform: translateY(-30px) scale(1.1, 1.1);
+    }
+
+    75% {
+        transform: translateY(-30px) scale(0.8, 0.7); 
+    }
+
+    100% {
+        transform: translateY(-30px) scale(1, 1); 
+    }
+}
+
+@keyframes red-flame {
+    0% {
+        transform: translateY(-30px) scale(1, 1);
+    }
+
+    25% {
+        transform: translateY(-30px) scale(1.1, 1.1);
+    }
+
+    75% {
+        transform: translateY(-30px) scale(0.8, 0.7); 
+    }
+
+    100% {
+        transform: translateY(-30px) scale(1, 1); 
+    }
+}
+
+@-webkit-keyframes yellow-flame {
+    0% {
+        transform: translateY(-30px) scale(0.8, 0.7);
+    }
+
+    50% {
+        transform: translateY(-30px) scale(1.1, 1.2);
+    }
+
+    100% {
+        transform: translateY(-30px) scale(1, 1);
+    }
+}
+
+@keyframes yellow-flame {
+    0% {
+        transform: translateY(-30px) scale(0.8, 0.7);
+    }
+
+    50% {
+        transform: translateY(-30px) scale(1.1, 1.2);
+    }
+
+    100% {
+        transform: translateY(-30px) scale(1, 1);
+    }
+}

util.py

@@ -0,0 +1,315 @@
+# Copyright (c) 2024 Jaerin Lee
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import concurrent.futures
+import time
+from typing import Any, Callable, List, Literal, Tuple, Union
+
+from PIL import Image
+import numpy as np
+
+import torch
+import torch.nn.functional as F
+import torch.cuda.amp as amp
+import torchvision.transforms as T
+import torchvision.transforms.functional as TF
+
+from diffusers import (
+    DiffusionPipeline,
+    StableDiffusionPipeline,
+    StableDiffusionXLPipeline,
+)
+
+
+def seed_everything(seed: int) -> None:
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = True
+
+
+def load_model(
+    model_key: str,
+    sd_version: Literal['1.5', 'xl'],
+    device: torch.device,
+    dtype: torch.dtype,
+) -> torch.nn.Module:
+    if model_key.endswith('.safetensors'):
+        if sd_version == '1.5':
+            pipeline = StableDiffusionPipeline
+        elif sd_version == 'xl':
+            pipeline = StableDiffusionXLPipeline
+        else:
+            raise ValueError(f'Stable Diffusion version {sd_version} not supported.')
+        return pipeline.from_single_file(model_key, torch_dtype=dtype).to(device)
+    try:
+        return DiffusionPipeline.from_pretrained(model_key, variant='fp16', torch_dtype=dtype).to(device)
+    except:
+        return DiffusionPipeline.from_pretrained(model_key, variant=None, torch_dtype=dtype).to(device)
+
+
+def get_cutoff(cutoff: float = None, scale: float = None) -> float:
+    if cutoff is not None:
+        return cutoff
+
+    if scale is not None and cutoff is None:
+        return 0.5 / scale
+
+    raise ValueError('Either one of `cutoff`, or `scale` should be specified.')
+
+
+def get_scale(cutoff: float = None, scale: float = None) -> float:
+    if scale is not None:
+        return scale
+
+    if cutoff is not None and scale is None:
+        return 0.5 / cutoff
+
+    raise ValueError('Either one of `cutoff`, or `scale` should be specified.')
+
+
+def filter_2d_by_kernel_1d(x: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
+    assert len(k.shape) in (1,), 'Kernel size should be one of (1,).'
+    #  assert len(k.shape) in (1, 2), 'Kernel size should be one of (1, 2).'
+
+    b, c, h, w = x.shape
+    ks = k.shape[-1]
+    k = k.view(1, 1, -1).repeat(c, 1, 1)
+
+    x = x.permute(0, 2, 1, 3)
+    x = x.reshape(b * h, c, w)
+    x = F.pad(x, (ks // 2, (ks - 1) // 2), mode='replicate')
+    x = F.conv1d(x, k, groups=c)
+    x = x.reshape(b, h, c, w).permute(0, 3, 2, 1).reshape(b * w, c, h)
+    x = F.pad(x, (ks // 2, (ks - 1) // 2), mode='replicate')
+    x = F.conv1d(x, k, groups=c)
+    x = x.reshape(b, w, c, h).permute(0, 2, 3, 1)
+    return x
+
+
+def filter_2d_by_kernel_2d(x: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
+    assert len(k.shape) in (2, 3), 'Kernel size should be one of (2, 3).'
+
+    x = F.pad(x, (
+        k.shape[-2] // 2, (k.shape[-2] - 1) // 2,
+        k.shape[-1] // 2, (k.shape[-1] - 1) // 2,
+    ), mode='replicate')
+
+    b, c, _, _ = x.shape
+    if len(k.shape) == 2 or (len(k.shape) == 3 and k.shape[0] == 1):
+        k = k.view(1, 1, *k.shape[-2:]).repeat(c, 1, 1, 1)
+        x = F.conv2d(x, k, groups=c)
+    elif len(k.shape) == 3:
+        assert k.shape[0] == b, \
+            'The number of kernels should match the batch size.'
+
+        k = k.unsqueeze(1)
+        x = F.conv2d(x.permute(1, 0, 2, 3), k, groups=b).permute(1, 0, 2, 3)
+    return x
+
+
+@amp.autocast(False)
+def filter_by_kernel(
+    x: torch.Tensor,
+    k: torch.Tensor,
+    is_batch: bool = False,
+) -> torch.Tensor:
+    k_dim = len(k.shape)
+    if k_dim == 1 or k_dim == 2 and is_batch:
+        return filter_2d_by_kernel_1d(x, k)
+    elif k_dim == 2 or k_dim == 3 and is_batch:
+        return filter_2d_by_kernel_2d(x, k)
+    else:
+        raise ValueError('Kernel size should be one of (1, 2, 3).')
+
+
+def gen_gauss_lowpass_filter_2d(
+    std: torch.Tensor,
+    window_size: int = None,
+) -> torch.Tensor:
+    # Gaussian kernel size is odd in order to preserve the center.
+    if window_size is None:
+        window_size = (
+            2 * int(np.ceil(3 * std.max().detach().cpu().numpy())) + 1)
+
+    y = torch.arange(
+        window_size, dtype=std.dtype, device=std.device
+    ).view(-1, 1).repeat(1, window_size)
+    grid = torch.stack((y.t(), y), dim=-1)
+    grid -= 0.5 * (window_size - 1) # (W, W)
+    var = (std * std).unsqueeze(-1).unsqueeze(-1)
+    distsq = (grid * grid).sum(dim=-1).unsqueeze(0).repeat(*std.shape, 1, 1)
+    k = torch.exp(-0.5 * distsq / var)
+    k /= k.sum(dim=(-2, -1), keepdim=True)
+    return k
+
+
+def gaussian_lowpass(
+    x: torch.Tensor,
+    std: Union[float, Tuple[float], torch.Tensor] = None,
+    cutoff: Union[float, torch.Tensor] = None,
+    scale: Union[float, torch.Tensor] = None,
+) -> torch.Tensor:
+    if std is None:
+        cutoff = get_cutoff(cutoff, scale)
+        std = 0.5 / (np.pi * cutoff)
+    if isinstance(std, (float, int)):
+        std = (std, std)
+    if isinstance(std, torch.Tensor):
+        """Using nn.functional.conv2d with Gaussian kernels built in runtime is
+        80% faster than transforms.functional.gaussian_blur for individual
+        items.
+        
+        (in GPU); However, in CPU, the result is exactly opposite. But you
+        won't gonna run this on CPU, right?
+        """
+        if len(list(s for s in std.shape if s != 1)) >= 2:
+            raise NotImplementedError(
+                'Anisotropic Gaussian filter is not currently available.')
+
+        # k.shape == (B, W, W).
+        k = gen_gauss_lowpass_filter_2d(std=std.view(-1))
+        if k.shape[0] == 1:
+            return filter_by_kernel(x, k[0], False)
+        else:
+            return filter_by_kernel(x, k, True)
+    else:
+        # Gaussian kernel size is odd in order to preserve the center.
+        window_size = tuple(2 * int(np.ceil(3 * s)) + 1 for s in std)
+        return TF.gaussian_blur(x, window_size, std)
+
+
+def blend(
+    fg: Union[torch.Tensor, Image.Image],
+    bg: Union[torch.Tensor, Image.Image],
+    mask: Union[torch.Tensor, Image.Image],
+    std: float = 0.0,
+) -> Image.Image:
+    if not isinstance(fg, torch.Tensor):
+        fg = T.ToTensor()(fg)
+    if not isinstance(bg, torch.Tensor):
+        bg = T.ToTensor()(bg)
+    if not isinstance(mask, torch.Tensor):
+        mask = (T.ToTensor()(mask) < 0.5).float()[:1]
+    if std > 0:
+        mask = gaussian_lowpass(mask[None], std)[0].clip_(0, 1)
+    return T.ToPILImage()(fg * mask + bg * (1 - mask))
+
+
+def get_panorama_views(
+    panorama_height: int,
+    panorama_width: int,
+    window_size: int = 64,
+) -> tuple[List[Tuple[int]], torch.Tensor]:
+    stride = window_size // 2
+    is_horizontal = panorama_width > panorama_height
+    num_blocks_height = (panorama_height - window_size + stride - 1) // stride + 1
+    num_blocks_width = (panorama_width - window_size + stride - 1) // stride + 1
+    total_num_blocks = num_blocks_height * num_blocks_width
+
+    half_fwd = torch.linspace(0, 1, (window_size + 1) // 2)
+    half_rev = half_fwd.flip(0)
+    if window_size % 2 == 1:
+        half_rev = half_rev[1:]
+    c = torch.cat((half_fwd, half_rev))
+    one = torch.ones_like(c)
+    f = c.clone()
+    f[:window_size // 2] = 1
+    b = c.clone()
+    b[-(window_size // 2):] = 1
+
+    h = [one] if num_blocks_height == 1 else [f] + [c] * (num_blocks_height - 2) + [b]
+    w = [one] if num_blocks_width == 1 else [f] + [c] * (num_blocks_width - 2) + [b]
+
+    views = []
+    masks = torch.zeros(total_num_blocks, panorama_height, panorama_width) # (n, h, w)
+    for i in range(total_num_blocks):
+        hi, wi = i // num_blocks_width, i % num_blocks_width
+        h_start = hi * stride
+        h_end = min(h_start + window_size, panorama_height)
+        w_start = wi * stride
+        w_end = min(w_start + window_size, panorama_width)
+        views.append((h_start, h_end, w_start, w_end))
+
+        h_width = h_end - h_start
+        w_width = w_end - w_start
+        masks[i, h_start:h_end, w_start:w_end] = h[hi][:h_width, None] * w[wi][None, :w_width]
+
+    # Sum of the mask weights at each pixel `masks.sum(dim=1)` must be unity.
+    return views, masks[None] # (1, n, h, w)
+
+
+def shift_to_mask_bbox_center(im: torch.Tensor, mask: torch.Tensor, reverse: bool = False) -> List[int]:
+    h, w = mask.shape[-2:]
+    device = mask.device
+    mask = mask.reshape(-1, h, w)
+    # assert mask.shape[0] == im.shape[0]
+    h_occupied = mask.sum(dim=-2) > 0
+    w_occupied = mask.sum(dim=-1) > 0
+    l = torch.argmax(h_occupied * torch.arange(w, 0, -1).to(device), 1, keepdim=True).cpu()
+    r = torch.argmax(h_occupied * torch.arange(w).to(device), 1, keepdim=True).cpu()
+    t = torch.argmax(w_occupied * torch.arange(h, 0, -1).to(device), 1, keepdim=True).cpu()
+    b = torch.argmax(w_occupied * torch.arange(h).to(device), 1, keepdim=True).cpu()
+    tb = (t + b + 1) // 2
+    lr = (l + r + 1) // 2
+    shifts = (tb - (h // 2), lr - (w // 2))
+    shifts = torch.cat(shifts, dim=1) # (p, 2)
+    if reverse:
+        shifts = shifts * -1
+    return torch.stack([i.roll(shifts=s.tolist(), dims=(-2, -1)) for i, s in zip(im, shifts)], dim=0)
+
+
+class Streamer:
+    def __init__(self, fn: Callable, ema_alpha: float = 0.9) -> None:
+        self.fn = fn
+        self.ema_alpha = ema_alpha
+
+        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+        self.future = self.executor.submit(fn)
+        self.image = None
+
+        self.prev_exec_time = 0
+        self.ema_exec_time = 0
+
+    @property
+    def throughput(self) -> float:
+        return 1.0 / self.ema_exec_time if self.ema_exec_time else float('inf')
+
+    def timed_fn(self) -> Any:
+        start = time.time()
+        res = self.fn()
+        end = time.time()
+        self.prev_exec_time = end - start
+        self.ema_exec_time = self.ema_exec_time * self.ema_alpha + self.prev_exec_time * (1 - self.ema_alpha)
+        return res
+
+    def __call__(self) -> Any:
+        if self.future.done() or self.image is None:
+            # get the result (the new image) and start a new task
+            image = self.future.result()
+            self.future = self.executor.submit(self.timed_fn)
+            self.image = image
+            return image
+        else:
+            # if self.fn() is not ready yet, use the previous image
+            # NOTE: This assumes that we have access to a previously generated image here.
+            # If there's no previous image (i.e., this is the first invocation), you could fall 
+            # back to some default image or handle it differently based on your requirements.
+            return self.image