add some stuff

check_for_branches.py

@@ -26,5 +26,8 @@ if __name__ == "__main__":
     api = HfApi()
     branches = main(api, model_id)
 
-    if len(branches) > 0:
-        print(f"{model_id}: {branches}")
+    if "fp16" in branches:
+        print(model_id)
+#
+#    if len(branches) > 0:
+#        print(f"{model_id}: {branches}")

open_pr_version.py

@@ -0,0 +1,182 @@
+import argparse
+import json
+import os
+import torch
+import shutil
+from tempfile import TemporaryDirectory
+from typing import List, Optional
+
+from huggingface_hub import CommitInfo, CommitOperationAdd, Discussion, HfApi, hf_hub_download
+from huggingface_hub.file_download import repo_folder_name
+
+
+class AlreadyExists(Exception):
+    pass
+
+
+def is_index_stable_diffusion_like(config_dict):
+    if "_class_name" not in config_dict:
+        return False
+
+    compatible_classes = [
+        "AltDiffusionImg2ImgPipeline",
+        "AltDiffusionPipeline",
+        "CycleDiffusionPipeline",
+        "StableDiffusionImageVariationPipeline",
+        "StableDiffusionImg2ImgPipeline",
+        "StableDiffusionInpaintPipeline",
+        "StableDiffusionInpaintPipelineLegacy",
+        "StableDiffusionPipeline",
+        "StableDiffusionPipelineSafe",
+        "StableDiffusionUpscalePipeline",
+        "VersatileDiffusionDualGuidedPipeline",
+        "VersatileDiffusionImageVariationPipeline",
+        "VersatileDiffusionPipeline",
+        "VersatileDiffusionTextToImagePipeline",
+        "OnnxStableDiffusionImg2ImgPipeline",
+        "OnnxStableDiffusionInpaintPipeline",
+        "OnnxStableDiffusionInpaintPipelineLegacy",
+        "OnnxStableDiffusionPipeline",
+        "StableDiffusionOnnxPipeline",
+        "FlaxStableDiffusionPipeline",
+    ]
+    return config_dict["_class_name"] in compatible_classes
+
+
+def convert_single(model_id: str, folder: str) -> List["CommitOperationAdd"]:
+    pipe = DiffusionPipeline.from_pretrained(model_id)
+
+    try:
+        pipe.to(torch_dtype=torch.float16)
+        pipe.save_pretrained(model_id, variant="fp16")
+        pipe.save_pretrained(model_id, variant="fp16", safe_serialization=True)
+        import ipdb; ipdb.set_trace()
+
+        operations = [CommitOperationAdd(path_in_repo=config_file, path_or_fileobj=new_config_file)]
+    except:
+        import ipdb; ipdb.set_trace()
+
+    if success:
+        model_type = success
+        return operations, model_type
+    else:
+        return False, False
+
+
+def convert_file(
+    old_config: str,
+    new_config: str,
+):
+    with open(old_config, "r") as f:
+        old_dict = json.load(f)
+
+    old_dict["feature_extractor"][-1] = "CLIPImageProcessor"
+    # if "clip_sample" not in old_dict:
+    #     print("Make scheduler DDIM compatible")
+    #     old_dict["clip_sample"] = False
+    # else:
+    #     print("No matching config")
+    #     return False
+
+    with open(new_config, 'w') as f:
+        json_str = json.dumps(old_dict, indent=2, sort_keys=True) + "\n"
+        f.write(json_str)
+
+    return "Stable Diffusion"
+
+
+def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discussion"]:
+    try:
+        discussions = api.get_repo_discussions(repo_id=model_id)
+    except Exception:
+        return None
+    for discussion in discussions:
+        if discussion.status == "open" and discussion.is_pull_request and discussion.title == pr_title:
+            return discussion
+
+
+def convert(api: "HfApi", model_id: str, force: bool = False) -> Optional["CommitInfo"]:
+#    pr_title = "Correct `sample_size` of {}'s unet to have correct width and height default"
+    pr_title = "Fix deprecation warning by changing `CLIPFeatureExtractor` to `CLIPImageProcessor`."
+    info = api.model_info(model_id)
+    filenames = set(s.rfilename for s in info.siblings)
+
+    if "model_index.json" not in filenames:
+        print(f"Model: {model_id} has no model_index.json file to change")
+        return
+
+    # if "vae/config.json" not in filenames:
+    #     print(f"Model: {model_id} has no 'vae/config.json' file to change")
+    #     return
+
+    with TemporaryDirectory() as d:
+        folder = os.path.join(d, repo_folder_name(repo_id=model_id, repo_type="models"))
+        os.makedirs(folder)
+        new_pr = None
+        try:
+            operations = None
+            pr = previous_pr(api, model_id, pr_title)
+            if pr is not None and not force:
+                url = f"https://huggingface.co/{model_id}/discussions/{pr.num}"
+                new_pr = pr
+                raise AlreadyExists(f"Model {model_id} already has an open PR check out {url}")
+            else:
+                operations, model_type = convert_single(model_id, folder)
+
+            if operations:
+                pr_title = pr_title.format(model_type)
+#                if model_type == "Stable Diffusion 1":
+#                    sample_size = 64
+#                    image_size = 512
+#                elif model_type == "Stable Diffusion 2":
+#                    sample_size = 96
+#                    image_size = 768
+
+#                pr_description = (
+#                        f"Since `diffusers==0.9.0` the width and height is automatically inferred from the `sample_size` attribute of your unet's config. It seems like your diffusion model has the same architecture as {model_type} which means that when using this model, by default an image size of {image_size}x{image_size} should be generated. This in turn means the unet's sample size should be **{sample_size}**. \n\n In order to suppress to update your configuration on the fly and to suppress the deprecation warning added in this PR: https://github.com/huggingface/diffusers/pull/1406/files#r1035703505 it is strongly recommended to merge this PR."
+#                )
+                contributor = model_id.split("/")[0]
+                pr_description = (
+                        f"Hey {contributor} 👋, \n\n Your model repository seems to contain logic to load a feature extractor that is deprecated, which you should notice by seeing the warning: "
+                        "\n\n ```\ntransformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. "
+                        f"Please use CLIPImageProcessor instead. warnings.warn(\n``` \n\n when running `pipe = DiffusionPipeline.from_pretrained({model_id})`."
+                        "This PR makes sure that the warning does not show anymore by replacing `CLIPFeatureExtractor` with `CLIPImageProcessor`. This will certainly not change or break your checkpoint, but only" 
+                        "make sure that everything is up to date. \n\n Best, the 🧨 Diffusers team."
+                )
+                new_pr = api.create_commit(
+                    repo_id=model_id,
+                    operations=operations,
+                    commit_message=pr_title,
+                    commit_description=pr_description,
+                    create_pr=True,
+                )
+                print(f"Pr created at {new_pr.pr_url}")
+            else:
+                print(f"No files to convert for {model_id}")
+        finally:
+            shutil.rmtree(folder)
+        return new_pr
+
+
+if __name__ == "__main__":
+    DESCRIPTION = """
+    Simple utility tool to convert automatically some weights on the hub to `safetensors` format.
+    It is PyTorch exclusive for now.
+    It works by downloading the weights (PT), converting them locally, and uploading them back
+    as a PR on the hub.
+    """
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument(
+        "model_id",
+        type=str,
+        help="The name of the model on the hub to convert. E.g. `gpt2` or `facebook/wav2vec2-base-960h`",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Create the PR even if it already exists of if the model was already converted.",
+    )
+    args = parser.parse_args()
+    model_id = args.model_id
+    api = HfApi()
+    convert(api, model_id, force=args.force)

run_control_inpaint.py

@@ -1,10 +1,15 @@
 #!/usr/bin/env python3
 # !pip install transformers accelerate
-from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler
+import os
+import PIL
+from pathlib import Path
+from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler, StableDiffusionInpaintPipeline, StableDiffusionImg2ImgPipeline, StableDiffusionControlNetImg2ImgPipeline
 from diffusers.utils import load_image
 import numpy as np
+from huggingface_hub import HfApi
 import torch
 
+api = HfApi()
 init_image = load_image(
     "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png"
 )
@@ -31,8 +36,12 @@ def make_inpaint_condition(image, image_mask):
 
 control_image = make_inpaint_condition(init_image, mask_image)
 
+mask_image = PIL.Image.open("/home/patrick/images/mask.png").convert('RGB')
+init_image = PIL.Image.open("/home/patrick/images/init.png").convert('RGB')
+control_image = PIL.Image.open("/home/patrick/images/seg.png").convert('RGB')
+
 controlnet = ControlNetModel.from_pretrained(
-    "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16
+    "mfidabel/controlnet-segment-anything", torch_dtype=torch.float16
 )
 pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
@@ -44,12 +53,25 @@ pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
 
 # generate image
-image = pipe(
-    "a beautiful man",
-    num_inference_steps=20,
-    generator=generator,
-    eta=1.0,
-    image=init_image,
-    mask_image=mask_image,
-    control_image=control_image,
-).images[0]
+for t in [2]:
+    image = pipe(
+        "a bench in front of a beautiful lake and white mountain",
+        num_inference_steps=t,
+        generator=generator,
+        eta=1.0,
+        image=init_image,
+        mask_image=mask_image,
+        control_image=control_image,
+    ).images[0]
+
+    file_name = f"aa_{t}"
+    path = os.path.join(Path.home(), "images", f"{file_name}.png")
+    image.save(path)
+
+    api.upload_file(
+        path_or_fileobj=path,
+        path_in_repo=path.split("/")[-1],
+        repo_id="patrickvonplaten/images",
+        repo_type="dataset",
+    )
+    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")

run_kandinsky.py

@@ -1,30 +1,27 @@
 #!/usr/bin/env python3
-from diffusers import DiffusionPipeline
+from diffusers import DiffusionPipeline, KandinskyPriorPipeline, DDPMScheduler, DDIMScheduler
 import torch
-from diffusers.models.attention_processor import AttnAddedKVProcessor2_0, XFormersAttnAddedKVProcessor, AttnAddedKVProcessor
 
-import time
 import os
 from huggingface_hub import HfApi
 from pathlib import Path
 
-from diffusers import DiffusionPipeline
-import torch
-
 api = HfApi()
 
-pipe_prior = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16)
+pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16)
 pipe_prior.to("cuda")
 
-t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16)
-t2i_pipe.to("cuda")
-
 prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting"
 negative_prompt = "low quality, bad quality"
 
-generator = torch.Generator(device="cuda").manual_seed(12)
+generator = torch.Generator(device="cuda").manual_seed(10)
 image_embeds, negative_image_embeds = pipe_prior(prompt, negative_prompt, guidance_scale=1.0, generator=generator).to_tuple()
 
+scheduler = DDPMScheduler.from_pretrained("../kandinsky-2-1/", subfolder="ddpm_scheduler")
+t2i_pipe = DiffusionPipeline.from_pretrained("../kandinsky-2-1/", scheduler=scheduler, torch_dtype=torch.float16)
+t2i_pipe.to("cuda")
+print(t2i_pipe.scheduler.config)
+
 images = t2i_pipe(prompt, num_images_per_prompt=4, image_embeds=image_embeds, negative_image_embeds=negative_image_embeds, negative_prompt=negative_prompt).images
 
 for i, image in enumerate(images):

run_local.py

@@ -13,6 +13,7 @@ from io import BytesIO
 
 # path = sys.argv[1]
 path = "runwayml/stable-diffusion-v1-5"
+path = "ptx0/pseudo-journey-v2"
 # path = "stabilityai/stable-diffusion-2-1"
 
 api = HfApi()
@@ -45,7 +46,7 @@ for TIMESTEP_TYPE in ["trailing", "leading"]:
     for RESCALE_BETAS_ZEROS_SNR in [True, False]:
         for GUIDANCE_RESCALE in [0,0, 0.7]:
 
-            pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config, timestep_type=TIMESTEP_TYPE, rescale_betas_zero_snr=RESCALE_BETAS_ZEROS_SNR)
+            pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config, timestep_spacing=TIMESTEP_TYPE, rescale_betas_zero_snr=RESCALE_BETAS_ZEROS_SNR)
             generator = torch.Generator(device="cpu").manual_seed(0)
             images = pipe(prompt=prompt, generator=generator, num_images_per_prompt=4, num_inference_steps=40, guidance_rescale=GUIDANCE_RESCALE).images
 

run_watermark.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import tree_ring_watermark as trk
+from diffusers import DiffusionPipeline, DDIMScheduler
+from pathlib import Path
+from huggingface_hub import HfApi, login
+import torch
+
+
+# login()  # make sure you login it with on account that is connected to `trk-demo`
+trk.set_org("trk-demo")
+
+model_id = 'stabilityai/stable-diffusion-2-1-base'
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+# note that the model hash should be the latest commit hash of the repo's history: https://huggingface.co/stabilityai/stable-diffusion-2-base/commits/main
+model_hash = "dcd3ee64f0c1aba2eb9e0c0c16041c6cae40d780"
+
+pipe = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
+pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to(device)
+
+# get noise
+batch_size = 1
+n_channels = pipe.unet.config.in_channels
+sample_size = pipe.unet.config.sample_size 
+
+shape = (batch_size, n_channels, sample_size, sample_size)
+
+# get model hash from https://huggingface.co/stabilityai/stable-diffusion-2-1-base/commits/main
+latents = trk.get_noise(shape, model_hash=model_hash)
+latents = latents.to(device=pipe.device, dtype=torch.float16)
+
+# generation without watermarking
+image = pipe(prompt="an astronaut", latents=latents).images[0]
+
+is_watermarked = trk.detect(image, pipe, model_hash)
+print(f'is_watermarked: {is_watermarked}')