# TODO import numpy as np import argparse import torch from torchvision.utils import make_grid import tempfile import gradio as gr from omegaconf import OmegaConf from einops import rearrange from scripts.pub.V3D_512 import ( sample_one, get_batch, get_unique_embedder_keys_from_conditioner, load_model, ) from sgm.util import default, instantiate_from_config from safetensors.torch import load_file as load_safetensors from PIL import Image from kiui.op import recenter from torchvision.transforms import ToTensor from einops import rearrange, repeat import rembg import os from glob import glob from mediapy import write_video from pathlib import Path import spaces @spaces.GPU def do_sample( image, model, clip_model, ae_model, device, num_frames, num_steps, decoding_t, border_ratio, ignore_alpha, rembg_session, output_folder, ): # if image.mode == "RGBA": # image = image.convert("RGB") image = Image.fromarray(image) w, h = image.size if border_ratio > 0: if image.mode != "RGBA" or ignore_alpha: image = image.convert("RGB") image = np.asarray(image) carved_image = rembg.remove(image, session=rembg_session) # [H, W, 4] else: image = np.asarray(image) carved_image = image mask = carved_image[..., -1] > 0 image = recenter(carved_image, mask, border_ratio=border_ratio) image = image.astype(np.float32) / 255.0 if image.shape[-1] == 4: image = image[..., :3] * image[..., 3:4] + (1 - image[..., 3:4]) image = Image.fromarray((image * 255).astype(np.uint8)) else: print("Ignore border ratio") image = image.resize((512, 512)) image = ToTensor()(image) image = image * 2.0 - 1.0 image = image.unsqueeze(0).to(device) H, W = image.shape[2:] assert image.shape[1] == 3 F = 8 C = 4 shape = (num_frames, C, H // F, W // F) value_dict = {} value_dict["motion_bucket_id"] = 0 value_dict["fps_id"] = 0 value_dict["cond_aug"] = 0.05 value_dict["cond_frames_without_noise"] = clip_model(image) value_dict["cond_frames"] = ae_model.encode(image) value_dict["cond_frames"] += 0.05 * torch.randn_like(value_dict["cond_frames"]) value_dict["cond_aug"] = 0.05 with torch.no_grad(): with torch.autocast(device): batch, batch_uc = get_batch( get_unique_embedder_keys_from_conditioner(model.conditioner), value_dict, [1, num_frames], T=num_frames, device=device, ) c, uc = model.conditioner.get_unconditional_conditioning( batch, batch_uc=batch_uc, force_uc_zero_embeddings=[ "cond_frames", "cond_frames_without_noise", ], ) for k in ["crossattn", "concat"]: uc[k] = repeat(uc[k], "b ... -> b t ...", t=num_frames) uc[k] = rearrange(uc[k], "b t ... -> (b t) ...", t=num_frames) c[k] = repeat(c[k], "b ... -> b t ...", t=num_frames) c[k] = rearrange(c[k], "b t ... -> (b t) ...", t=num_frames) randn = torch.randn(shape, device=device) randn = randn.to(device) additional_model_inputs = {} additional_model_inputs["image_only_indicator"] = torch.zeros( 2, num_frames ).to(device) additional_model_inputs["num_video_frames"] = batch["num_video_frames"] def denoiser(input, sigma, c): return model.denoiser( model.model, input, sigma, c, **additional_model_inputs ) samples_z = model.sampler(denoiser, randn, cond=c, uc=uc) model.en_and_decode_n_samples_a_time = decoding_t samples_x = model.decode_first_stage(samples_z) samples = torch.clamp((samples_x + 1.0) / 2.0, min=0.0, max=1.0) os.makedirs(output_folder, exist_ok=True) base_count = len(glob(os.path.join(output_folder, "*.mp4"))) video_path = os.path.join(output_folder, f"{base_count:06d}.mp4") frames = ( (rearrange(samples, "t c h w -> t h w c") * 255) .cpu() .numpy() .astype(np.uint8) ) write_video(video_path, frames, fps=6) return video_path def change_model_params(model, min_cfg, max_cfg): model.sampler.guider.max_scale = max_cfg model.sampler.guider.min_scale = min_cfg @spaces.GPU def launch(device="cuda", share=False): model_config = "./scripts/pub/configs/V3D_512.yaml" num_frames = OmegaConf.load( model_config ).model.params.sampler_config.params.guider_config.params.num_frames print("Detected num_frames:", num_frames) # num_steps = default(num_steps, 25) num_steps = 25 output_folder = "outputs/V3D_512" sd = load_safetensors("./ckpts/svd_xt.safetensors") clip_model_config = OmegaConf.load("./configs/embedder/clip_image.yaml") clip_model = instantiate_from_config(clip_model_config).eval() clip_sd = dict() for k, v in sd.items(): if "conditioner.embedders.0" in k: clip_sd[k.replace("conditioner.embedders.0.", "")] = v clip_model.load_state_dict(clip_sd) clip_model = clip_model.to(device) ae_model_config = OmegaConf.load("./configs/ae/video.yaml") ae_model = instantiate_from_config(ae_model_config).eval() encoder_sd = dict() for k, v in sd.items(): if "first_stage_model" in k: encoder_sd[k.replace("first_stage_model.", "")] = v ae_model.load_state_dict(encoder_sd) ae_model = ae_model.to(device) rembg_session = rembg.new_session() model, _ = load_model( model_config, device, num_frames, num_steps, min_cfg=3.5, max_cfg=3.5 ) with gr.Blocks(title="V3D", theme=gr.themes.Monochrome()) as demo: with gr.Row(equal_height=True): with gr.Column(): input_image = gr.Image(value=None, label="Input Image") border_ratio_slider = gr.Slider( value=0.3, label="Border Ratio", minimum=0.05, maximum=0.5, step=0.05, ) decoding_t_slider = gr.Slider( value=1, label="Number of Decoding frames", minimum=1, maximum=num_frames, step=1, ) min_guidance_slider = gr.Slider( value=3.5, label="Min CFG Value", minimum=0.05, maximum=0.5, step=0.05, ) max_guidance_slider = gr.Slider( value=3.5, label="Max CFG Value", minimum=0.05, maximum=0.5, step=0.05, ) run_button = gr.Button(value="Run V3D") with gr.Column(): output_video = gr.Video(value=None, label="Output Orbit Video") @run_button.click( inputs=[ input_image, border_ratio_slider, min_guidance_slider, max_guidance_slider, decoding_t_slider, ], outputs=[output_video], ) def _(image, border_ratio, min_guidance, max_guidance, decoding_t): change_model_params(model, min_guidance, max_guidance) return do_sample( image, model, clip_model, ae_model, device, num_frames, num_steps, int(decoding_t), border_ratio, False, rembg_session, output_folder, ) # do_sample( # np.asarray(Image.open("assets/baby_yoda.png")), # model, # clip_model, # ae_model, # device, # num_frames, # num_steps, # 1, # 0.3, # False, # rembg_session, # output_folder, # ) demo.launch(inbrowser=True, inline=False, share=share, show_error=True) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--device", type=str, default="cuda") parser.add_argument("--share", action="store_true") opt = parser.parse_args() test = OmegaConf.load("./scripts/pub/configs/V3D_512.yaml") print(test) exit(0) def download_if_need(path, url): if Path(path).exists(): return import wget path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) wget.download(url, out=str(path)) # download_if_need( # "ckpts/svd_xt.safetensors", # "https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt/resolve/main/svd_xt.safetensors", # ) # download_if_need( # "ckpts/V3D_512.ckpt", "https://huggingface.co/heheyas/V3D/resolve/main/V3D.ckpt" # ) launch(opt.device, opt.share)