all

README.md

@@ -1,98 +1,58 @@
-# Diffusers Tools
+---
+language:
+- en
+license: creativeml-openrail-m
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- inpainting
+- art
+- artistic
+- diffusers
+- anime
+- absolute-realism
+---
 
-This is a collection of scripts that can be useful for various tasks related to the [diffusers library](https://github.com/huggingface/diffusers)
+# Absolute realism 1.6525 inpainting
 
-## 1. Test against original checkpoints
+`lykon-absolute-realism/absolute-realism-1.6525-inpainting` is a Stable Diffusion Inpainting model that has been fine-tuned on [runwayml/stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting).
 
-**It's very important to have visually the exact same results as the original code bases.!**
+Please consider supporting me: 
+- on [Patreon](https://www.patreon.com/Lykon275)
+- or [buy me a coffee](https://snipfeed.co/lykon)
 
-E.g. to make use `diffusers` is identical to the original [CompVis codebase](https://github.com/CompVis/stable-diffusion), you can run the following script in the original CompVis codebase:
+## Diffusers
 
-1. Download the original [SD-1-4 checkpoint](https://huggingface.co/CompVis/stable-diffusion-v1-4) and put it in the correct folder following the instructions on: https://github.com/CompVis/stable-diffusion
+For more general information on how to run inpainting models with 🧨 Diffusers, see [the docs](https://huggingface.co/docs/diffusers/using-diffusers/inpaint).
+
+1. Installation
 
-2. Run the following command
 ```
-python scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --seed 0 --n_samples 1 --n_rows 1 --n_iter 1
+pip install diffusers transformers accelerate
 ```
 
-and compare this to the same command in diffusers:
-
-```python
-from diffusers import DiffusionPipeline, StableDiffusionPipeline, DDIMScheduler
+2. Run
+```py
+from diffusers import AutoPipelineForInpainting, DEISMultistepScheduler
 import torch
+from diffusers.utils import load_image
 
-#  python scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --seed 0 --n_samples 1 --n_rows 1 --n_iter 1
-seed = 0
-
-prompt = "a photograph of an astronaut riding a horse"
-pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16)
-
+pipe = AutoPipelineForInpainting.from_pretrained('lykon-absolute-realism/absolute-realism-1.6525-inpainting', torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
 pipe = pipe.to("cuda")
-pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
-torch.manual_seed(0)
-image = pipe(prompt, num_inference_steps=50).images[0]
 
-image.save("/home/patrick_huggingface_co/images/aa_comp.png")
-```
+img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
+mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
 
-Both commands should give the following image on a V100:
+image = load_image(img_url)
+mask_image = load_image(mask_url)
 
 
-## 2. Test against [k-diffusion](https://github.com/crowsonkb/k-diffusion):
-
-You can run the following script to compare against k-diffusion.
-
-See results [here](https://huggingface.co/datasets/patrickvonplaten/images)
-
-```python
-from diffusers import StableDiffusionKDiffusionPipeline, HeunDiscreteScheduler, StableDiffusionPipeline, DPMSolverMultistepScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler
-import torch
-import os
+prompt = "a majestic tiger sitting on a park bench"
 
-seed = 13
-inference_steps = 25
-#checkpoint = "CompVis/stable-diffusion-v1-4"
-checkpoint = "stabilityai/stable-diffusion-2-1"
-prompts = ["astronaut riding horse", "whale falling from sky", "magical forest", "highly photorealistic picture of johnny depp"]
-prompts = 8 * ["highly photorealistic picture of johnny depp"]
-#prompts = prompts[:1]
-samplers = ["sample_dpmpp_2m", "sample_euler", "sample_heun", "sample_dpm_2", "sample_lms"]
-#samplers = samplers[:1]
-
-pipe = StableDiffusionKDiffusionPipeline.from_pretrained(checkpoint, torch_dtype=torch.float16, safety_checker=None)
-pipe = pipe.to("cuda")
-
-for i, prompt in enumerate(prompts):
-    prompt_f = f"{'_'.join(prompt.split())}_{i}"
-    for sampler in samplers:
-        pipe.set_scheduler(sampler)
-        torch.manual_seed(seed + i)
-        image = pipe(prompt, num_inference_steps=inference_steps).images[0]
-        checkpoint_f = f"{'--'.join(checkpoint.split('/'))}"
-        os.makedirs(f"/home/patrick_huggingface_co/images/{checkpoint_f}", exist_ok=True)
-        os.makedirs(f"/home/patrick_huggingface_co/images/{checkpoint_f}/{sampler}", exist_ok=True)
-        image.save(f"/home/patrick_huggingface_co/images/{checkpoint_f}/{sampler}/{prompt_f}.png")
-
-
-pipe = StableDiffusionPipeline(**pipe.components)
-pipe = pipe.to("cuda")
-
-for i, prompt in enumerate(prompts):
-    prompt_f = f"{'_'.join(prompt.split())}_{i}"
-    for sampler in samplers:
-        if sampler == "sample_euler":
-            pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
-        elif sampler == "sample_heun":
-            pipe.scheduler = HeunDiscreteScheduler.from_config(pipe.scheduler.config)
-        elif sampler == "sample_dpmpp_2m":
-            pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
-        elif sampler == "sample_lms":
-            pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config)
- 
-        torch.manual_seed(seed + i)
-        image = pipe(prompt, num_inference_steps=inference_steps).images[0]
-        checkpoint_f = f"{'--'.join(checkpoint.split('/'))}"
-        os.makedirs("/home/patrick_huggingface_co/images/{checkpoint_f}", exist_ok=True)
-        os.makedirs(f"/home/patrick_huggingface_co/images/{checkpoint_f}/{sampler}", exist_ok=True)
-        image.save(f"/home/patrick_huggingface_co/images/{checkpoint_f}/{sampler}/{prompt_f}_hf.png")
+generator = torch.manual_seed(33)
+image = pipe(prompt, image=image, mask_image=mask_image, generator=generator, num_inference_steps=25).images[0]  
+image.save("./image.png")
 ```
+
+![](./image.png)

README_t2i.py

@@ -0,0 +1,100 @@
+import sys
+
+model_name = sys.argv[1]
+
+model_card = f"""---
+language:
+- en
+license: creativeml-openrail-m
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- text-to-image
+- art
+- artistic
+- diffusers
+- absolute_realism
+---
+
+# {model_name.split("/")[-1].replace("-", " ").capitalize()}
+
+`{model_name}` is a Stable Diffusion model that has been fine-tuned on [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5).
+
+Please consider supporting me: 
+- on [Patreon](https://www.patreon.com/Lykon275)
+- or [buy me a coffee](https://snipfeed.co/lykon)
+
+## Diffusers
+
+For more general information on how to run text-to-image models with 🧨 Diffusers, see [the docs](https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation).
+
+1. Installation
+
+```
+pip install diffusers transformers accelerate
+```
+
+2. Run
+```py
+from diffusers import AutoPipelineForText2Image, DEISMultistepScheduler
+import torch
+
+pipe = AutoPipelineForText2Image.from_pretrained('{model_name}', torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to("cuda")
+
+prompt = "portrait photo of muscular bearded guy in a worn mech suit, light bokeh, intricate, steel metal, elegant, sharp focus, soft lighting, vibrant colors"
+
+generator = torch.manual_seed(33)
+image = pipe(prompt, generator=generator, num_inference_steps=25).images[0]  
+image.save("./image.png")
+```
+"""
+
+"""
+## Notes
+
+- **Version 8** focuses on improving what V7 started. Might be harder to do photorealism compared to realism focused models, as it might be hard to do anime compared to anime focused models, but it can do both pretty well if you're skilled enough. Check the examples!
+- **Version 7** improves lora support, NSFW and realism. If you're interested in "absolute" realism, try AbsoluteReality.
+- **Version 6** adds more lora support and more style in general. It should also be better at generating directly at 1024 height (but be careful with it). 6.x are all improvements.
+- **Version 5** is the best at photorealism and has noise offset.
+- **Version 4** is much better with anime (can do them with no LoRA) and booru tags. It might be harder to control if you're used to caption style, so you might still want to use version 3.31. V4 is also better with eyes at lower resolutions. Overall is like a "fix" of V3 and shouldn't be too much different.
+"""
+from huggingface_hub import HfApi
+api = HfApi()
+
+read_me_path = "./README.md"
+with open(read_me_path, "w") as f:
+    f.write(model_card)
+
+api.upload_file(
+    path_or_fileobj=read_me_path,
+    path_in_repo=read_me_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+from diffusers import AutoPipelineForText2Image, DEISMultistepScheduler
+import torch
+
+pipe = AutoPipelineForText2Image.from_pretrained(model_name, torch_dtype=torch.float16)
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+
+pipe = pipe.to("cuda")
+
+prompt = "portrait photo of muscular bearded guy in a worn mech suit, light bokeh, intricate, steel metal, elegant, sharp focus, soft lighting, vibrant colors"
+
+generator = torch.manual_seed(33)
+image = pipe(prompt, generator=generator, num_inference_steps=25).images[0]  
+image_path = "./image.png"
+
+image.save(image_path)
+
+api.upload_file(
+    path_or_fileobj=image_path,
+    path_in_repo=image_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+pipe.push_to_hub(model_name, variant="fp16")

README_t2i_inpaint.py

@@ -0,0 +1,120 @@
+import sys
+
+model_name = sys.argv[1]
+
+model_card = f"""---
+language:
+- en
+license: creativeml-openrail-m
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- inpainting
+- art
+- artistic
+- diffusers
+- anime
+- absolute-realism
+---
+
+# {model_name.split("/")[-1].replace("-", " ").capitalize()}
+
+`{model_name}` is a Stable Diffusion Inpainting model that has been fine-tuned on [runwayml/stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting).
+
+Please consider supporting me: 
+- on [Patreon](https://www.patreon.com/Lykon275)
+- or [buy me a coffee](https://snipfeed.co/lykon)
+
+## Diffusers
+
+For more general information on how to run inpainting models with 🧨 Diffusers, see [the docs](https://huggingface.co/docs/diffusers/using-diffusers/inpaint).
+
+1. Installation
+
+```
+pip install diffusers transformers accelerate
+```
+
+2. Run
+```py
+from diffusers import AutoPipelineForInpainting, DEISMultistepScheduler
+import torch
+from diffusers.utils import load_image
+
+pipe = AutoPipelineForInpainting.from_pretrained('{model_name}', torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to("cuda")
+
+img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
+mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
+
+image = load_image(img_url)
+mask_image = load_image(mask_url)
+
+
+prompt = "a majestic tiger sitting on a park bench"
+
+generator = torch.manual_seed(33)
+image = pipe(prompt, image=image, mask_image=mask_image, generator=generator, num_inference_steps=25).images[0]  
+image.save("./image.png")
+```
+
+![](./image.png)
+"""
+
+"""
+## Notes
+
+- **Version 8** focuses on improving what V7 started. Might be harder to do photorealism compared to realism focused models, as it might be hard to do anime compared to anime focused models, but it can do both pretty well if you're skilled enough. Check the examples!
+- **Version 7** improves lora support, NSFW and realism. If you're interested in "absolute" realism, try AbsoluteReality.
+- **Version 6** adds more lora support and more style in general. It should also be better at generating directly at 1024 height (but be careful with it). 6.x are all improvements.
+- **Version 5** is the best at photorealism and has noise offset.
+- **Version 4** is much better with anime (can do them with no LoRA) and booru tags. It might be harder to control if you're used to caption style, so you might still want to use version 3.31. V4 is also better with eyes at lower resolutions. Overall is like a "fix" of V3 and shouldn't be too much different.
+"""
+from huggingface_hub import HfApi
+api = HfApi()
+
+read_me_path = "./README.md"
+with open(read_me_path, "w") as f:
+    f.write(model_card)
+
+api.upload_file(
+    path_or_fileobj=read_me_path,
+    path_in_repo=read_me_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+from diffusers import AutoPipelineForInpainting, DEISMultistepScheduler
+import torch
+from diffusers.utils import load_image
+
+pipe = AutoPipelineForInpainting.from_pretrained(model_name, torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to("cuda")
+
+img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
+mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
+
+image = load_image(img_url)
+mask_image = load_image(mask_url)
+
+
+prompt = "a majestic tiger sitting on a park bench"
+
+generator = torch.manual_seed(33)
+image = pipe(prompt, image=image, mask_image=mask_image, generator=generator, num_inference_steps=25).images[0]  
+image.save("./image.png")
+
+image_path = "./image.png"
+
+image.save(image_path)
+
+api.upload_file(
+    path_or_fileobj=image_path,
+    path_in_repo=image_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+pipe.push_to_hub(model_name, variant="fp16")

README_t2i_xl.py

@@ -0,0 +1,93 @@
+import sys
+
+model_name = sys.argv[1]
+
+model_card = f"""---
+language:
+- en
+license: openrail++
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- stable-diffusion-xl
+- text-to-image
+- art
+- artistic
+- diffusers
+- anime
+---
+
+# {model_name.split("/")[-1].replace("-", " ").capitalize()}
+
+`{model_name}` is a Stable Diffusion model that has been fine-tuned on [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0).
+
+Please consider supporting me: 
+- on [Patreon](https://www.patreon.com/Lykon275)
+- or [buy me a coffee](https://snipfeed.co/lykon)
+
+## Diffusers
+
+For more general information on how to run text-to-image models with 🧨 Diffusers, see [the docs](https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation).
+
+1. Installation
+
+```
+pip install diffusers transformers accelerate
+```
+
+2. Run
+```py
+from diffusers import AutoPipelineForText2Image, DEISMultistepScheduler
+import torch
+
+pipe = AutoPipelineForText2Image.from_pretrained('{model_name}', torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to("cuda")
+
+prompt = "portrait photo of muscular bearded guy in a worn mech suit, light bokeh, intricate, steel metal, elegant, sharp focus, soft lighting, vibrant colors"
+
+generator = torch.manual_seed(0)
+image = pipe(prompt, num_inference_steps=25).images[0]  
+image.save("./image.png")
+```
+
+![](./image.png)
+"""
+from huggingface_hub import HfApi
+api = HfApi()
+
+read_me_path = "./README.md"
+with open(read_me_path, "w") as f:
+    f.write(model_card)
+
+api.upload_file(
+    path_or_fileobj=read_me_path,
+    path_in_repo=read_me_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+from diffusers import AutoPipelineForText2Image, DEISMultistepScheduler
+import torch
+
+pipe = AutoPipelineForText2Image.from_pretrained(model_name, torch_dtype=torch.float16)
+pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
+
+pipe = pipe.to("cuda")
+
+prompt = "portrait photo of muscular bearded guy in a worn mech suit, light bokeh, intricate, steel metal, elegant, sharp focus, soft lighting, vibrant colors"
+
+generator = torch.manual_seed(0)
+image = pipe(prompt, num_inference_steps=25).images[0]  
+image_path = "./image.png"
+
+image.save(image_path)
+
+api.upload_file(
+    path_or_fileobj=image_path,
+    path_in_repo=image_path,
+    repo_id=model_name,
+    repo_type="model",
+)
+
+pipe.push_to_hub(model_name, variant="fp16")

fuse_unfuse.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import torch
+from diffusers import StableDiffusionXLPipeline
+  
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+).to("cuda")
+
+#Fuse/unfuse first LoRA  
+pipe.load_lora_weights("joachimsallstrom/aether-cloud-lora-for-sdxl")
+pipe.fuse_lora()
+pipe.unload_lora_weights()
+pipe.unfuse_lora()
+
+#Now it would be expected that the the LoRA has been unfused and `pipe` in the its original state, however...
+
+#Fuse/unfuse second LoRA
+pipe.load_lora_weights("jbilcke-hf/sdxl-zelda64")
+pipe.fuse_lora()
+pipe.unload_lora_weights()
+pipe.unfuse_lora() #errors out

if_m.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline
+from diffusers.utils import pt_to_pil
+import torch
+
+# stage 1
+stage_1 = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-M-v1.0", variant="fp16", torch_dtype=torch.float16)
+stage_1.enable_xformers_memory_efficient_attention()  # remove line if torch.__version__ >= 2.0.0
+stage_1.enable_model_cpu_offload()
+
+# stage 2
+stage_2 = DiffusionPipeline.from_pretrained(
+    "DeepFloyd/IF-II-M-v1.0", text_encoder=None, variant="fp16", torch_dtype=torch.float16
+)
+stage_2.enable_xformers_memory_efficient_attention()  # remove line if torch.__version__ >= 2.0.0
+stage_2.enable_model_cpu_offload()
+
+# stage 3
+safety_modules = {"feature_extractor": stage_1.feature_extractor, "safety_checker": stage_1.safety_checker, "watermarker": stage_1.watermarker}
+stage_3 = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-x4-upscaler", **safety_modules, torch_dtype=torch.float16)
+stage_3.enable_xformers_memory_efficient_attention()  # remove line if torch.__version__ >= 2.0.0
+stage_3.enable_model_cpu_offload()
+
+prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"'
+
+# text embeds
+prompt_embeds, negative_embeds = stage_1.encode_prompt(prompt)
+
+generator = torch.manual_seed(0)
+
+image = stage_1(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type="pt").images
+pt_to_pil(image)[0].save("./if_stage_I.png")
+
+image = stage_2(
+    image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type="pt"
+).images
+pt_to_pil(image)[0].save("./if_stage_II.png")
+
+image = stage_3(prompt=prompt, image=image, generator=generator, noise_level=100).images
+image[0].save("./if_stage_III.png")
+

img2img.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

light_and_shadow.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:15a630ab2255460b2e99b1e5a6765a1cedcd210e2ffca32474bd02df24232624
-size 151110122

load_replicate.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+#!/usr/bin/env python3
+from diffusers import StableDiffusionXLPipeline, AutoencoderKL
+import torch
+
+path = "stabilityai/stable-diffusion-xl-base-1.0"
+vae_path = "madebyollin/sdxl-vae-fp16-fix"
+
+# vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16)
+# pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, vae=vae, variant="fp16", use_safetensors=True, local_files_only=True, add_watermarker=False)
+pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+pipe.to("cuda")
+
+prompt = "An astronaut riding a green horse on Mars"
+steps = 20
+
+for _ in range(5):
+    image = pipe(prompt=prompt, num_inference_steps=steps).images[0]

monthly_downloads.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+from huggingface_hub import HfApi, ModelFilter
+
+api = HfApi()
+
+filter = ModelFilter(library="diffusers")
+
+models = api.list_models(filter=filter)
+
+downloads = sum(m.downloads for m in models)
+
+print(downloads)

monthly_per_class_downloads.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+from huggingface_hub import HfApi, ModelFilter
+
+api = HfApi()
+
+filter = ModelFilter(library="diffusers")
+
+models = api.list_models(filter=filter)
+
+import ipdb; ipdb.set_trace()
+
+downloads = sum(m.downloads for m in models)
+
+print(downloads)

new_loading.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import time
+import torch
+import safetensors.torch
+from diffusers import StableDiffusionXLPipeline
+
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    variant="fp16",
+    use_safetensors=True,
+    local_files_only=True,
+)
+
+pipe = pipe.to("cuda")
+
+# !wget https://civitai.com/api/download/models/135931 -O loras/pixel-art-xl.safetensors
+lora_weights = safetensors.torch.load_file(
+    "pixel-art-xl.safetensors", device="cpu"
+)
+
+for _ in range(5):
+    t0 = time.perf_counter()
+    pipe.load_lora_weights(lora_weights.copy())
+    pipe.unload_lora_weights()
+    print("Load + unload cycle took: ", time.perf_counter() - t0)

run_kandinsky.py

@@ -1,36 +1,7 @@
 #!/usr/bin/env python3
-from diffusers import DiffusionPipeline, KandinskyPriorPipeline, DDPMScheduler, DDIMScheduler
+from diffusers import AutoPipelineForInpainting, AutoPipelineForText2Image, DiffusionPipeline, KandinskyPriorPipeline, DDPMScheduler, DDIMScheduler
 import torch
 
-import os
-from huggingface_hub import HfApi
-from pathlib import Path
-
-api = HfApi()
-
-pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16)
-pipe_prior.to("cuda")
-
-prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting"
-negative_prompt = "low quality, bad quality"
-
-generator = torch.Generator(device="cuda").manual_seed(10)
-image_embeds, negative_image_embeds = pipe_prior(prompt, negative_prompt, guidance_scale=1.0, generator=generator).to_tuple()
-
-t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16)
-t2i_pipe.to("cuda")
-print(t2i_pipe.scheduler.config)
-
-images = t2i_pipe(prompt, num_images_per_prompt=4, image_embeds=image_embeds, negative_image_embeds=negative_image_embeds, negative_prompt=negative_prompt).images
-
-for i, image in enumerate(images):
-    path = os.path.join(Path.home(), "images", f"aa_{i}.png")
-    image.save(path)
-
-    api.upload_file(
-        path_or_fileobj=path,
-        path_in_repo=path.split("/")[-1],
-        repo_id="patrickvonplaten/images",
-        repo_type="dataset",
-    )
-    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa_{i}.png")
+cache_dir = DiffusionPipeline.download("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16, variant="fp16", load_connected_pipeline=True)
+print(cache_dir)
+pipe = DiffusionPipeline.from_pretrained(cache_dir, load_connected_pipeline=True)

run_local_fuse_xl.py

@@ -1,38 +1,37 @@
 #!/usr/bin/env python3
-from huggingface_hub import HfApi
 import torch
-from pathlib import Path
-import os
 import time
+import hf_image_uploader as hiu
 
-api = HfApi()
 start_time = time.time()
 
 from diffusers import DiffusionPipeline
 import torch
 
 pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16)
-pipe.load_lora_weights("stabilityai/stable-diffusion-xl-base-1.0", weight_name="sd_xl_offset_example-lora_1.0.safetensors")
-# pipe.unet.fuse_lora()
 
 pipe.to(torch_dtype=torch.float16)
 pipe.to("cuda")
 
 torch.manual_seed(33)
 
-prompt = "beautiful scenery nature glass bottle landscape, purple galaxy bottle"
+prompt = "a mecha robot"
 negative_prompt = "text, watermark"
 
+torch.manual_seed(0)
 image = pipe(prompt, negative_prompt=negative_prompt, num_inference_steps=25).images[0]
+hiu.upload(image, "patrickvonplaten/images")
 
-file_name = f"aaa"
-path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
-image.save(path)
-
-api.upload_file(
-    path_or_fileobj=path,
-    path_in_repo=path.split("/")[-1],
-    repo_id="patrickvonplaten/images",
-    repo_type="dataset",
-)
-print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+pipe.load_lora_weights("davizca87/sun-flower", weight_name="snfw3rXL-000004.safetensors")
+pipe.fuse_lora()
+pipe.unload_lora_weights()
+
+torch.manual_seed(0)
+image = pipe(prompt, negative_prompt=negative_prompt, num_inference_steps=25).images[0]
+hiu.upload(image, "patrickvonplaten/images")
+
+pipe.unfuse_lora()
+
+torch.manual_seed(0)
+image = pipe(prompt, negative_prompt=negative_prompt, num_inference_steps=25).images[0]
+hiu.upload(image, "patrickvonplaten/images")

run_local_img2img_xl.py

@@ -1,16 +1,11 @@
 #!/usr/bin/env python3
-from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler, StableDiffusionXLImg2ImgPipeline
+from diffusers import StableDiffusionXLImg2ImgPipeline, EulerDiscreteScheduler, UniPCMultistepScheduler, DEISMultistepScheduler, HeunDiscreteScheduler, DPMSolverMultistepScheduler
 import time
+import hf_image_uploader as hiu
 import numpy as np
-import os
 from huggingface_hub import HfApi
-# from compel import Compel
 import torch
 import sys
-from pathlib import Path
-import requests
-from PIL import Image
-from io import BytesIO
 
 path = sys.argv[1]
 
@@ -18,34 +13,30 @@ api = HfApi()
 start_time = time.time()
 pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16)
 pipe.to("cuda")
-pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
-# pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
-# pipe = StableDiffusionImg2ImgXLPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None
-
-# compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder)
 
 
 prompt = "An astronaut riding a green horse on Mars"
 
 
-# pipe.unet.to(memory_format=torch.channels_last)
-# pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
-# pipe(prompt=prompt, num_inference_steps=2).images[0]
-# url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
-# 
-# response = requests.get(url)
 init_image = torch.from_numpy(np.load("/home/patrick/images/xl_latents.npy")).to("cuda")
 
-image = pipe(prompt=prompt, image=init_image).images[0]
 
-file_name = f"aaa"
-path = os.path.join(Path.home(), "images", f"{file_name}.png")
-image.save(path)
 
-api.upload_file(
-    path_or_fileobj=path,
-    path_in_repo=path.split("/")[-1],
-    repo_id="patrickvonplaten/images",
-    repo_type="dataset",
-)
-print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+
+
+for scheduler_cls, kwargs in [
+        (EulerDiscreteScheduler, {}), 
+        (EulerDiscreteScheduler, {"use_karras_sigmas": True}), 
+        (UniPCMultistepScheduler, {}),
+        (UniPCMultistepScheduler, {"use_karras_sigmas": True}), 
+        (DEISMultistepScheduler, {}),
+        (DEISMultistepScheduler, {"use_karras_sigmas": True}), 
+        (HeunDiscreteScheduler, {}),
+        (HeunDiscreteScheduler, {"use_karras_sigmas": True}), 
+        (DPMSolverMultistepScheduler, {}),
+        (DPMSolverMultistepScheduler, {"use_karras_sigmas": True, "algorithm_type": "sd-dpmsolver++"}),
+]:
+    for num_steps in [11,12]:
+        pipe.scheduler = scheduler_cls.from_config(pipe.scheduler.config, **kwargs)
+        image = pipe(prompt=prompt, num_inference_steps=num_steps, image=init_image).images[0]
+        hiu.upload(image, "patrickvonplaten/images")

run_local_xl.py

@@ -22,12 +22,13 @@ use_refiner = False
 use_diffusers = True
 path = "stabilityai/stable-diffusion-xl-base-1.0"
 refiner_path = "stabilityai/stable-diffusion-xl-refiner-1.0"
-vae_path = "stabilityai/sdxl-vae"
+vae_path = "madebyollin/sdxl-vae-fp16-fix"
 
-vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16, force_upcast=True)
+vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16)
 if use_diffusers:
     # pipe = StableDiffusionXLPipeline.from_pretrained(path, vae=vae, torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
     pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, vae=vae, variant="fp16", use_safetensors=True, local_files_only=True, add_watermarker=False)
+    import ipdb; ipdb.set_trace()
     # pipe.enable_xformers_memory_efficient_attention()
     print(time.time() - start_time)
     pipe.to("cuda")

run_local_xl_img2img.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler
+from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, AutoencoderKL
+import time
+from pytorch_lightning import seed_everything
+import os
+from huggingface_hub import HfApi
+# from compel import Compel
+import torch
+import sys
+from pathlib import Path
+import requests
+from PIL import Image
+from io import BytesIO
+import xformers
+
+api = HfApi()
+start_time = time.time()
+
+# use_refiner = bool(int(sys.argv[1]))
+use_refiner = False
+use_diffusers = True
+path = "stabilityai/stable-diffusion-xl-base-1.0"
+refiner_path = "stabilityai/stable-diffusion-xl-refiner-1.0"
+vae_path = "madebyollin/sdxl-vae-fp16-fix"
+
+vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16)
+if use_diffusers:
+    # pipe = StableDiffusionXLPipeline.from_pretrained(path, vae=vae, torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
+    pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, vae=vae, variant="fp16", use_safetensors=True, local_files_only=True, add_watermarker=False)
+    import ipdb; ipdb.set_trace()
+    # pipe.enable_xformers_memory_efficient_attention()
+    print(time.time() - start_time)
+    pipe.to("cuda")
+
+    if use_refiner:
+        start_time = time.time()
+        refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained(refiner_path, vae=vae, torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
+        print(time.time() - start_time)
+        refiner.to("cuda")
+        # refiner.enable_sequential_cpu_offload()
+else:
+    start_time = time.time()
+    pipe = StableDiffusionXLPipeline.from_single_file("https://huggingface.co/stabilityai/stable-diffusion-xl-base-0.9/blob/main/sd_xl_base_0.9.safetensors", torch_dtype=torch.float16, use_safetensors=True)
+    print(time.time() - start_time)
+    pipe.to("cuda")
+
+    if use_refiner:
+        start_time = time.time()
+        refiner = StableDiffusionXLImg2ImgPipeline.from_single_file("https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-0.9/blob/main/sd_xl_refiner_0.9.safetensors", torch_dtype=torch.float16, use_safetensors=True)
+        print(time.time() - start_time)
+        refiner.to("cuda")
+
+
+prompt = "An astronaut riding a green horse on Mars"
+steps = 20
+seed = 0
+seed_everything(seed)
+start_time = time.time()
+image = pipe(prompt=prompt, num_inference_steps=steps, output_type="latent" if use_refiner else "pil").images[0]
+print(time.time() - start_time)
+
+if use_refiner:
+    image = refiner(prompt=prompt, num_inference_steps=steps - 10, image=image).images[0]
+
+file_name = f"aaa"
+path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
+image.save(path)
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")

run_local_xl_simple.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+from diffusers import DPMSolverMultistepScheduler, StableDiffusionXLPipeline, DPMSolverSDEScheduler
+import hf_image_uploader as hiu
+import torch
+
+path = "stabilityai/stable-diffusion-xl-base-1.0"
+vae_path = "madebyollin/sdxl-vae-fp16-fix"
+
+# vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16)
+# pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, vae=vae, variant="fp16", use_safetensors=True, local_files_only=True, add_watermarker=False)
+pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config, algorithm_type="sde-dpmsolver++")
+pipe.to("cuda")
+
+prompt = "An astronaut riding a green horse on Mars"
+steps = 20
+
+for i in range(2):
+    width = 512 * (i + 1)
+    height = 512 * (i + 1)
+    image = pipe(prompt=prompt, width=width, height=height, num_inference_steps=steps).images[0]
+    hiu.upload(image, "patrickvonplaten/images")
+
+pipe.scheduler = DPMSolverSDEScheduler.from_config(pipe.scheduler.config, algorithm_type="sde-dpmsolver++")
+
+for i in range(2):
+    width = 512 * (i + 1)
+    height = 512 * (i + 1)
+    image = pipe(prompt=prompt, width=width, height=height, num_inference_steps=steps).images[0]
+    hiu.upload(image, "patrickvonplaten/images")

run_lora.py

@@ -1,27 +1,25 @@
 #!/usr/bin/env python3
-from diffusers import AutoPipelineForText2Image, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler,  DPMSolverMultistepScheduler
+from diffusers import AutoPipelineForText2Image
 import time
 import os
 from huggingface_hub import HfApi
-# from compel import Compel
 import torch
-import sys
 from pathlib import Path
-import requests
-from PIL import Image
-from io import BytesIO
 
-path = "stabilityai/stable-diffusion-xl-base-0.9"
+path = "stabilityai/stable-diffusion-xl-base-1.0"
 
 api = HfApi()
 start_time = time.time()
 pipe = AutoPipelineForText2Image.from_pretrained(path, torch_dtype=torch.float16)
 pipe.enable_model_cpu_offload()
 lora_model_id = "hf-internal-testing/sdxl-0.9-kamepan-lora"
+lora_model_id = "TheLastBen/Papercut_SDXL"
 lora_filename = "kame_sdxl_v2-000020-16rank.safetensors"
+lora_filename = "papercut.safetensors"
 pipe.load_lora_weights(lora_model_id, weight_name=lora_filename)
 
 prompt = "masterpiece, best quality, mountain"
+prompt = "papercut sonic"
 
 images = pipe(prompt=prompt, 
     num_inference_steps=20, 

run_muse.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+from PIL import Image  
+import torch
+from muse import PipelineMuse, MaskGiTUViT
+from datasets import Dataset, Features
+from datasets import Image as ImageFeature
+from datasets import Value, load_dataset
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+pipe = PipelineMuse.from_pretrained(
+    transformer_path="valhalla/research-run",
+    text_encoder_path="openMUSE/clip-vit-large-patch14-text-enc",
+    vae_path="openMUSE/vqgan-f16-8192-laion",
+).to(device)
+
+pipe.transformer = MaskGiTUViT.from_pretrained("valhalla/research-run-finetuned-journeydb", revision="06bcd6ab6580a2ed3275ddfc17f463b8574457da", subfolder="ema_model").to(device)
+pipe.tokenizer.pad_token_id = 49407
+
+if device == "cuda":
+    pipe.transformer.enable_xformers_memory_efficient_attention()
+    pipe.text_encoder.to(torch.float16)
+    pipe.transformer.to(torch.float16)
+
+
+import PIL
+
+
+def main():
+    print("Loading dataset...")
+    parti_prompts = load_dataset("nateraw/parti-prompts", split="train")
+
+    print("Loading pipeline...")
+    seed = 0
+
+    device = "cuda"
+    torch.manual_seed(0)
+
+    ckpt_id = "openMUSE/muse-512"
+
+    scale = 10
+
+    print("Running inference...")
+    main_dict = {}
+    for i in range(len(parti_prompts)):
+        sample = parti_prompts[i]
+        prompt = sample["Prompt"]
+
+        image = pipe(
+            prompt,
+            timesteps=16,
+            negative_text=None,
+            guidance_scale=scale,
+            temperature=(2, 0),
+            orig_size=(512, 512),
+            crop_coords=(0, 0),
+            aesthetic_score=6,
+            use_fp16=device == "cuda",
+            transformer_seq_len=1024,
+            use_tqdm=False,
+        )[0]
+
+        image = image.resize((256, 256), resample=PIL.Image.Resampling.LANCZOS)
+        img_path = f"/home/patrick/muse_images/muse_512_{i}.png"
+        image.save(img_path)
+        main_dict.update(
+            {
+                prompt: {
+                    "img_path": img_path,
+                    "Category": sample["Category"],
+                    "Challenge": sample["Challenge"],
+                    "Note": sample["Note"],
+                    "model_name": ckpt_id,
+                    "seed": seed,
+                }
+            }
+        )
+
+    def generation_fn():
+        for prompt in main_dict:
+            prompt_entry = main_dict[prompt]
+            yield {
+                "Prompt": prompt,
+                "Category": prompt_entry["Category"],
+                "Challenge": prompt_entry["Challenge"],
+                "Note": prompt_entry["Note"],
+                "images": {"path": prompt_entry["img_path"]},
+                "model_name": prompt_entry["model_name"],
+                "seed": prompt_entry["seed"],
+            }
+
+    print("Preparing HF dataset...")
+    ds = Dataset.from_generator(
+        generation_fn,
+        features=Features(
+            Prompt=Value("string"),
+            Category=Value("string"),
+            Challenge=Value("string"),
+            Note=Value("string"),
+            images=ImageFeature(),
+            model_name=Value("string"),
+            seed=Value("int64"),
+        ),
+    )
+    ds_id = "diffusers-parti-prompts/muse_512"
+    ds.push_to_hub(ds_id)
+
+
+if __name__ == "__main__":
+    main()

run_muse_256.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+from PIL import Image  
+import torch
+from muse import PipelineMuse, MaskGiTUViT
+from datasets import Dataset, Features
+from datasets import Image as ImageFeature
+from datasets import Value, load_dataset
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+pipe = PipelineMuse.from_pretrained(
+    transformer_path="valhalla/research-run",
+    text_encoder_path="openMUSE/clip-vit-large-patch14-text-enc",
+    vae_path="openMUSE/vqgan-f16-8192-laion",
+).to(device)
+
+# pipe.transformer = MaskGiTUViT.from_pretrained("valhalla/research-run-finetuned-journeydb", revision="06bcd6ab6580a2ed3275ddfc17f463b8574457da", subfolder="ema_model").to(device)
+pipe.transformer = MaskGiTUViT.from_pretrained("valhalla/muse-research-run", subfolder="ema_model").to(device)
+pipe.tokenizer.pad_token_id = 49407
+
+if device == "cuda":
+    pipe.transformer.enable_xformers_memory_efficient_attention()
+    pipe.text_encoder.to(torch.float16)
+    pipe.transformer.to(torch.float16)
+
+
+import PIL
+
+
+def main():
+    print("Loading dataset...")
+    parti_prompts = load_dataset("nateraw/parti-prompts", split="train")
+
+    print("Loading pipeline...")
+    seed = 0
+
+    device = "cuda"
+    torch.manual_seed(0)
+
+    ckpt_id = "openMUSE/muse-256"
+
+    scale = 10
+
+    print("Running inference...")
+    main_dict = {}
+    for i in range(len(parti_prompts)):
+        sample = parti_prompts[i]
+        prompt = sample["Prompt"]
+
+        image = pipe(
+            prompt,
+            timesteps=16,
+            negative_text=None,
+            guidance_scale=scale,
+            temperature=(2, 0),
+            orig_size=(256, 256),
+            crop_coords=(0, 0),
+            aesthetic_score=6,
+            use_fp16=device == "cuda",
+            transformer_seq_len=256,
+            use_tqdm=False,
+        )[0]
+
+        image = image.resize((256, 256), resample=PIL.Image.Resampling.LANCZOS)
+        img_path = f"/home/patrick/muse_images/muse_256_{i}.png"
+        image.save(img_path)
+        main_dict.update(
+            {
+                prompt: {
+                    "img_path": img_path,
+                    "Category": sample["Category"],
+                    "Challenge": sample["Challenge"],
+                    "Note": sample["Note"],
+                    "model_name": ckpt_id,
+                    "seed": seed,
+                }
+            }
+        )
+
+    def generation_fn():
+        for prompt in main_dict:
+            prompt_entry = main_dict[prompt]
+            yield {
+                "Prompt": prompt,
+                "Category": prompt_entry["Category"],
+                "Challenge": prompt_entry["Challenge"],
+                "Note": prompt_entry["Note"],
+                "images": {"path": prompt_entry["img_path"]},
+                "model_name": prompt_entry["model_name"],
+                "seed": prompt_entry["seed"],
+            }
+
+    print("Preparing HF dataset...")
+    ds = Dataset.from_generator(
+        generation_fn,
+        features=Features(
+            Prompt=Value("string"),
+            Category=Value("string"),
+            Challenge=Value("string"),
+            Note=Value("string"),
+            images=ImageFeature(),
+            model_name=Value("string"),
+            seed=Value("int64"),
+        ),
+    )
+    ds_id = "diffusers-parti-prompts/muse_256"
+    ds.push_to_hub(ds_id)
+
+
+if __name__ == "__main__":
+    main()

run_sd_text.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import logging
+import torch
+import diffusers
+
+log = logging.getLogger("test")
+log.setLevel(logging.DEBUG)
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s')
+
+log.info(f'loaded: torch={torch.__version__} diffusers={diffusers.__version__}')
+
+prompt_positive = 'futuristic city'
+prompt_negative = 'grass'
+seeds = [42]
+model_path = "runwayml/stable-diffusion-v1-5"
+embedding_path_ok = "sd15_text_inv.pt"
+# embedding_path_err = "./sd21_text_inv.pt"
+device = 'cuda:0'
+load_args = {
+    "low_cpu_mem_usage": True,
+    "torch_dtype": torch.float16,
+    "variant": 'fp16',
+    "safety_checker": None,
+    "load_safety_checker": False,
+    # "local_files_only": False,
+}
+pipe = diffusers.StableDiffusionPipeline.from_pretrained(model_path, **load_args)
+pipe.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba')
+pipe.enable_model_cpu_offload()
+args = {
+    'prompt': [prompt_positive],
+    'negative_prompt': [prompt_negative],
+    'guidance_scale': 6,
+    'generator': [torch.Generator(device).manual_seed(s) for s in seeds],
+    'output_type': 'pil',
+    'num_inference_steps': 10,
+    'eta': 0.0,
+    'guidance_rescale': 0.7,
+    'height': 512,
+    'width': 512,
+}
+
+# pipe.load_lora_weights(embedding_path_ok, **load_args)
+try:
+    pipe.load_lora_weights(embedding_path_ok, **load_args)
+except Exception as e:
+    log.error(f'failed to load embeddings: {e}')
+
+output = pipe(**args)
+log.info(f'output: {output}') # this works fine
+

run_sdxl_text_inv.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import hf_image_uploader as hiu
+from diffusers import EulerDiscreteScheduler
+from diffusers import AutoPipelineForText2Image
+import torch
+
+pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", variant="fp16", torch_dtype=torch.float16)
+pipe.scheduler =  EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+pipe.to("cuda")
+
+prompt = "photo of a rhino dressed suit and tie sitting at a table in a bar with a bar stools, award winning photography, Elke vogelsang"
+image = pipe(prompt, guidance_scale=9.0, num_inference_steps=25).images[0]
+
+hiu.upload(image, repo_id="patrickvonplaten/images")

run_wuerst.py

@@ -1,37 +1,42 @@
 #!/usr/bin/env python3
+import hf_image_uploader as hiu
 import torch
-from diffusers import AutoPipelineForText2Image
-from huggingface_hub import HfApi
-from pathlib import Path
-import os
-
-from PIL import Image
-import numpy as np
-
-api = HfApi()
-
-pipe = AutoPipelineForText2Image.from_pretrained("warp-diffusion/WuerstchenGeneratorPipeline", torch_dtype=torch.float16).to("cuda")
-
-prompt = [
-    "An old destroyed car standing on a cliff in norway, cinematic photography",
-    "Western movie, closeup cinematic photography",
-    "Pink nike shoe commercial, closeup cinematic photography",
-    "Croatia, closeup cinematic photography",
-    "South Tyrol mountains at sunset, closeup cinematic photography",
-]
-
-
-images = pipe(prompt, guidance_scale=8.0, width=1024, height=1024).images
-
-for i, image in enumerate(images):
-    file_name = f"bb_1_{i}"
-    path = os.path.join(Path.home(), "images", f"{file_name}.png")
-    image.save(path)
-
-    api.upload_file(
-        path_or_fileobj=path,
-        path_in_repo=path.split("/")[-1],
-        repo_id="patrickvonplaten/images",
-        repo_type="dataset",
-    )
-    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+from diffusers import WuerstchenDecoderPipeline, WuerstchenPriorPipeline
+from diffusers.pipelines.wuerstchen import DEFAULT_STAGE_C_TIMESTEPS
+
+device = "cuda"
+dtype = torch.float16
+num_images_per_prompt = 2
+
+prior_pipeline = WuerstchenPriorPipeline.from_pretrained(
+    "warp-ai/wuerstchen-prior", torch_dtype=dtype
+).to(device)
+decoder_pipeline = WuerstchenDecoderPipeline.from_pretrained(
+    "warp-ai/wuerstchen", torch_dtype=dtype
+).to(device)
+
+caption = "Anthropomorphic cat dressed as a fire fighter"
+negative_prompt = ""
+
+prior_pipeline.prior = torch.compile(prior_pipeline.prior, mode="reduce-overhead", fullgraph=True)
+decoder_pipeline.decoder = torch.compile(decoder_pipeline.decoder, mode="reduce-overhead", fullgraph=True)
+
+prior_output = prior_pipeline(
+    prompt=caption,
+    height=1024,
+    width=1536,
+    timesteps=DEFAULT_STAGE_C_TIMESTEPS,
+    negative_prompt=negative_prompt,
+    guidance_scale=4.0,
+    num_images_per_prompt=num_images_per_prompt,
+)
+images = decoder_pipeline(
+    image_embeddings=prior_output.image_embeddings,
+    prompt=caption,
+    negative_prompt=negative_prompt,
+    guidance_scale=0.0,
+    output_type="pil",
+).images
+
+for image in images:
+    hiu.upload(image, repo_id="patrickvonplaten/images")

run_wuerstchen.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+import torch
+from datasets import Dataset, Features
+from datasets import Image as ImageFeature
+from datasets import Value, load_dataset
+from diffusers import AutoPipelineForText2Image
+
+import PIL
+
+
+def main():
+    print("Loading dataset...")
+    parti_prompts = load_dataset("nateraw/parti-prompts", split="train")
+
+    print("Loading pipeline...")
+    seed = 0
+
+    device = "cuda"
+    generator = torch.Generator(device).manual_seed(seed)
+    dtype = torch.float16
+
+    ckpt_id = "warp-diffusion/wuerstchen"
+
+    pipeline =  AutoPipelineForText2Image.from_pretrained(
+        ckpt_id, torch_dtype=dtype
+    ).to(device)
+
+    pipeline.prior_prior = torch.compile(pipeline.prior_prior, mode="reduce-overhead", fullgraph=True)
+    pipeline.decoder = torch.compile(pipeline.decoder, mode="reduce-overhead", fullgraph=True)
+
+    print("Running inference...")
+    main_dict = {}
+    for i in range(len(parti_prompts)):
+        sample = parti_prompts[i]
+        prompt = sample["Prompt"]
+
+        image = pipeline(
+            prompt=prompt,
+            height=1024,
+            width=1024,
+            prior_guidance_scale=4.0,
+            decoder_guidance_scale=0.0,
+            generator=generator,
+        ).images[0]
+
+        image = image.resize((256, 256), resample=PIL.Image.Resampling.LANCZOS)
+        img_path = f"wuerstchen_{i}.png"
+        image.save(img_path)
+        main_dict.update(
+            {
+                prompt: {
+                    "img_path": img_path,
+                    "Category": sample["Category"],
+                    "Challenge": sample["Challenge"],
+                    "Note": sample["Note"],
+                    "model_name": ckpt_id,
+                    "seed": seed,
+                }
+            }
+        )
+
+    def generation_fn():
+        for prompt in main_dict:
+            prompt_entry = main_dict[prompt]
+            yield {
+                "Prompt": prompt,
+                "Category": prompt_entry["Category"],
+                "Challenge": prompt_entry["Challenge"],
+                "Note": prompt_entry["Note"],
+                "images": {"path": prompt_entry["img_path"]},
+                "model_name": prompt_entry["model_name"],
+                "seed": prompt_entry["seed"],
+            }
+
+    print("Preparing HF dataset...")
+    ds = Dataset.from_generator(
+        generation_fn,
+        features=Features(
+            Prompt=Value("string"),
+            Category=Value("string"),
+            Challenge=Value("string"),
+            Note=Value("string"),
+            images=ImageFeature(),
+            model_name=Value("string"),
+            seed=Value("int64"),
+        ),
+    )
+    ds_id = "diffusers-parti-prompts/wuerstchen"
+    ds.push_to_hub(ds_id)
+
+
+if __name__ == "__main__":
+    main()

run_wuerstchen_new.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+import torch
+from diffusers import AutoPipelineForText2Image
+from diffusers.pipelines.wuerstchen import DEFAULT_STAGE_C_TIMESTEPS
+
+pipe = AutoPipelineForText2Image.from_pretrained("warp-ai/wuerstchen", torch_dtype=torch.float16).to("cuda")
+
+caption = "Anthropomorphic cat dressed as a fire fighter"
+images = pipe(
+    caption, 
+    width=1024,
+    height=1536,
+    prior_timesteps=DEFAULT_STAGE_C_TIMESTEPS,
+    prior_guidance_scale=4.0,
+    num_images_per_prompt=2,
+).images
+
+import ipdb; ipdb.set_trace()

run_xl.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import hf_image_uploader as hiu
+import torch
+from compel import Compel, ReturnedEmbeddingsType
+from diffusers import DiffusionPipeline
+
+pipe = DiffusionPipeline.from_pretrained(
+  "stabilityai/stable-diffusion-xl-base-1.0",
+  variant="fp16",
+  torch_dtype=torch.float16
+)
+pipe2 = DiffusionPipeline.from_pretrained(
+  "stabilityai/stable-diffusion-xl-refiner-1.0",
+  variant="fp16",
+  torch_dtype=torch.float16
+)
+pipe.enable_model_cpu_offload()
+pipe2.enable_model_cpu_offload()
+
+compel = Compel(
+  tokenizer=[pipe.tokenizer, pipe.tokenizer_2] ,
+  text_encoder=[pipe.text_encoder, pipe.text_encoder_2],
+  returned_embeddings_type=ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED,
+  requires_pooled=[False, True]
+)
+
+compel2 = Compel(
+  tokenizer=pipe.tokenizer_2,
+  text_encoder=pipe.text_encoder_2,
+  returned_embeddings_type=ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED,
+  requires_pooled=True,
+)
+
+# apply weights
+prompt = ["a red cat playing with a (ball)1.5", "a red cat playing with a (ball)0.6"]
+conditioning, pooled = compel(prompt)
+conditioning2, pooled2 = compel2(prompt)
+
+# generate image
+
+for _ in range(3):
+    generator = [torch.Generator().manual_seed(i) for i in range(len(prompt))]
+
+    image = pipe(prompt_embeds=conditioning, pooled_prompt_embeds=pooled, generator=generator, num_inference_steps=30, output_type="latent").images
+    image = pipe2(image=image, prompt_embeds=conditioning2, pooled_prompt_embeds=pooled2, generator=generator, num_inference_steps=20).images[0]
+    hiu.upload(image, "patrickvonplaten/images")

run_xl_lora.py

@@ -1,35 +1,45 @@
 #!/usr/bin/env python3
 from diffusers import DiffusionPipeline
+from safetensors.torch import load_file
 import torch
 from pathlib import Path
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, hf_hub_download
 import os
+import hf_image_uploader as hiu
+import time
 
 api = HfApi()
 
+
 pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16)
-pipe.load_lora_weights("stabilityai/stable-diffusion-xl-base-1.0", weight_name="sd_xl_offset_example-lora_1.0.safetensors")
-# pipe.unet.fuse_lora()
-# 7.8 it/s to beat
-#
-pipe.to(torch_dtype=torch.float16)
 pipe.to("cuda")
 
-torch.manual_seed(0)
+# pipe.load_lora_weights("stabilityai/stable-diffusion-xl-base-1.0", weight_name="sd_xl_offset_example-lora_1.0.safetensors", low_cpu_mem_usage=True)
+# file = hf_hub_download("TheLastBen/Papercut_SDXL", filename="papercut.safetensors")
+file = hf_hub_download("hf-internal-testing/sdxl-0.9-daiton-lora", filename="daiton-xl-lora-test.safetensors")
+
+start_time = time.time()
+pipe.load_lora_weights("Pclanglais/TintinIA")
+pipe.fuse_lora()
+pipe.unfuse_lora()
+pipe.unload_lora_weights()
+
+pipe.load_lora_weights("ProomptEngineer/pe-balloon-diffusion-style")
+pipe.fuse_lora()
+pipe.unload_lora_weights()
+pipe.unfuse_lora()
 
-prompt = "beautiful scenery nature glass bottle landscape, , purple galaxy bottle"
-negative_prompt = "text, watermark"
+pipe.load_lora_weights("ostris/crayon_style_lora_sdxl")
+pipe.fuse_lora()
+pipe.unload_lora_weights()
+pipe.unfuse_lora()
+print(time.time() - start_time)
 
-image = pipe(prompt, negative_prompt=negative_prompt, num_inference_steps=25).images[0]
+prompt = "masterpiece, best quality, mountain"
 
-file_name = f"aaa"
-path = os.path.join(Path.home(), "images", f"{file_name}.png")
-image.save(path)
+images = pipe(prompt=prompt, 
+    num_inference_steps=20, 
+    generator=torch.manual_seed(0)
+).images
 
-api.upload_file(
-    path_or_fileobj=path,
-    path_in_repo=path.split("/")[-1],
-    repo_id="patrickvonplaten/images",
-    repo_type="dataset",
-)
-print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+hiu.upload(images[0], "patrickvonplaten/images")

run_xl_pivotal.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import hf_image_uploader as hiu
+from safetensors.torch import load_file
+from huggingface_hub import snapshot_download
+
+from diffusers import DiffusionPipeline
+import torch
+import json
+import os
+
+model_id = "multimodalart/sdxl-emoji"
+
+folder = snapshot_download(model_id)
+
+with open(os.path.join(folder, "special_params.json"), 'r') as json_file:
+    data = json.load(json_file)
+    token, token_2 = data["TOK"][:4], data["TOK"][4:]
+
+state_dict = load_file(os.path.join(folder, "embeddings.pti"))
+
+text_encoder_sd = state_dict["text_encoders_0"]
+text_encoder_2_sd = state_dict["text_encoders_1"]
+
+pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16)
+
+pipe.load_textual_inversion(text_encoder_sd, token=token, text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer)
+pipe.load_textual_inversion(text_encoder_2_sd, token=token_2, text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2)
+pipe.load_lora_weights(os.path.join(folder, "lora.safetensors"))
+pipe.to(torch_dtype=torch.float16)
+pipe.to("cuda")
+
+prompt="A <s0><s1> emoji of a man"
+
+image = pipe(prompt, cross_attention_kwargs={"scale": 0.8}, num_inference_steps=20).images[0]
+hiu.upload(image, "patrickvonplaten/images")

sd_xl_inpaint.py

@@ -1,25 +1,24 @@
 #!/usr/bin/env python3
-from diffusers import AutoPipelineForInpainting, AutoPipelineForImage2Image
+from diffusers import AutoPipelineForInpainting, AutoPipelineForImage2Image, UNet2DConditionModel
 from diffusers.utils import load_image
 import torch
-from pathlib import Path
-import os
-from huggingface_hub import HfApi
+import hf_image_uploader as hfi
 
 torch.backends.cuda.matmul.allow_tf32 = True
 torch_device = "cuda" if torch.cuda.is_available() else "cpu"
-api = HfApi()
 
-pipe = AutoPipelineForInpainting.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, variant="fp16", use_safetensors=True) 
+# pipe = AutoPipelineForInpainting.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, variant="fp16", use_safetensors=True) 
+unet = UNet2DConditionModel.from_pretrained("valhalla/sdxl-inpaint-ema", torch_dtype=torch.float16)
+pipe = AutoPipelineForInpainting.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", unet=unet, torch_dtype=torch.float16, variant="fp16").to("cuda")
 
 pipe = pipe.to(torch_device)
-pipe.enable_xformers_memory_efficient_attention()
 
 img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
+img_url = "https://huggingface.co/datasets/patrickvonplaten/images_1024_1024/resolve/main/jdn_jacques_de_nuce-1024x1024.jpg"
 mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
 
-image = load_image(img_url)
-mask_image = load_image(mask_url)
+image = load_image(img_url).resize((1024, 1024))
+mask_image = load_image(mask_url).resize((1024, 1024))
 
 prompt = "dslr photography of an empty bench, high quality"
 generator = torch.Generator(device="cuda").manual_seed(0)
@@ -30,47 +29,10 @@ image = pipe(
     mask_image=mask_image,
     guidance_scale=8.0,
     num_inference_steps=20,
+    strength=0.90,
     generator=generator,
 ).images[0]
 
 
 image = image.resize((1024, 1024))
-
-pipe = AutoPipelineForInpainting.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True) 
-pipe.to("cuda")
-
-pipe.enable_xformers_memory_efficient_attention()
-
-image = pipe(
-    prompt=prompt,
-    image=image,
-    mask_image=mask_image,
-    guidance_scale=8.0,
-    num_inference_steps=100,
-    strength=0.2,
-    generator=generator,
-).images[0]
-
-pipe = AutoPipelineForImage2Image.from_pipe(pipe)
-pipe.enable_xformers_memory_efficient_attention()
-
-image = pipe(
-    prompt=prompt,
-    image=image,
-    guidance_scale=8.0,
-    num_inference_steps=100,
-    strength=0.2,
-    generator=generator,
-).images[0]
-
-file_name = f"aaa"
-path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
-image.save(path)
-
-api.upload_file(
-    path_or_fileobj=path,
-    path_in_repo=path.split("/")[-1],
-    repo_id="patrickvonplaten/images",
-    repo_type="dataset",
-)
-print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+hfi.upload(image, "patrickvonplaten/images")

upload_new_checkpoint.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+from huggingface_hub import HfApi
+from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, StableDiffusionInpaintPipeline
+from pathlib import Path
+import os
+import sys
+import requests
+from PIL import Image
+from io import BytesIO
+
+org_name = "lykon-models"
+org_name = "lykon-absolute-realism"
+
+file_path = sys.argv[1]
+file_name = file_path.split("/")[-1].split(".safetensors")[0].replace("_", "-").replace(".", "-").lower()
+
+if file_name.endswith("-"):
+    file_name = file_name[:-1]
+
+def download_image(url):
+    response = requests.get(url)
+    return Image.open(BytesIO(response.content)).convert("RGB")
+
+kwargs = {}
+if "xl" in file_name or "alpha2" in file_name:
+    pipe = StableDiffusionXLPipeline.from_single_file(file_path)
+elif "inpaint" in file_name:
+    pipe = StableDiffusionInpaintPipeline.from_single_file(file_path)
+    img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
+    mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
+
+    init_image = download_image(img_url).resize((512, 512))
+    mask_image = download_image(mask_url).resize((512, 512))
+    kwargs = {"image": init_image, "mask_image": mask_image}
+else:
+    pipe = StableDiffusionPipeline.from_single_file(file_path)
+
+print("Test...")
+prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k"
+pipe.to("cuda")
+images = pipe(prompt=prompt, num_inference_steps=25, **kwargs).images
+pipe.to("cpu")
+
+api = HfApi()
+for i, image in enumerate(images):
+    image_file_name = f"bb_1_{i}"
+    path = os.path.join(Path.home(), "images", f"{image_file_name}.png")
+    image.save(path)
+
+    api.upload_file(
+        path_or_fileobj=path,
+        path_in_repo=path.split("/")[-1],
+        repo_id="patrickvonplaten/images",
+        repo_type="dataset",
+    )
+    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{image_file_name}.png")
+
+print("Upload...")
+model_id = os.path.join(org_name, file_name)
+pipe.push_to_hub(model_id, private=True)