#!/usr/bin/env python3 import torch import os from huggingface_hub import HfApi from pathlib import Path from diffusers.utils import load_image import cv2 from PIL import Image import numpy as np from diffusers import ( ControlNetModel, EulerDiscreteScheduler, StableDiffusionControlNetPipeline, StableDiffusionXLControlNetPipeline, UniPCMultistepScheduler, ) import sys checkpoint = sys.argv[1] prompts = [ "beautiful room", "a photo-realistic image of two paradise birds", "a snowy house behind a forest", "a couple watching a romantic sunset", "boats in the Amazonas", "a photo of a beautiful face of a woman", "a skater in Brooklyn", "a tornado in Iowa" ] sd_xl = "control_v11p" not in checkpoint if sd_xl: base_ckpt = "stabilityai/stable-diffusion-xl-base-0.9" controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetPipeline.from_pretrained( base_ckpt, controlnet=controlnet, torch_dtype=torch.float16 ) size = 1024 else: base_ckpt = "runwayml/stable-diffusion-v1-5" controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16) pipe = StableDiffusionControlNetPipeline.from_pretrained( base_ckpt, controlnet=controlnet, torch_dtype=torch.float16 ) size = 512 import ipdb; ipdb.set_trace() # pipe.enable_model_cpu_offload() pipe.to("cuda") for i in range(8): for seed in range(4): image = load_image( f"https://huggingface.co/datasets/patrickvonplaten/webdatasets_images/resolve/main/image_{i}.png" ) image = image.resize((size, size)) prompt = prompts[i] image = np.array(image) low_threshold = 100 high_threshold = 200 image = cv2.Canny(image, low_threshold, high_threshold) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) canny_image = Image.fromarray(image) generator = torch.manual_seed(seed) out_image = pipe(prompt, generator=generator, num_inference_steps=20, image=canny_image, controlnet_conditioning_scale=1.0).images[0] path = os.path.join(Path.home(), "images", "control_sdxl", f"{i}_{seed}.png") path_in_repo = "/".join(path.split("/")[-2:]) out_image.save(path) api = HfApi() api.upload_file( path_or_fileobj=path, path_in_repo=path_in_repo, repo_id="patrickvonplaten/images", repo_type="dataset", ) print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/control_sdxl/{i}_{seed}.png")