Upload 27 files

diffusion_webui/__init__.py

@@ -1 +1 @@
-__version__ = "1.4.1"
+__version__ = "1.6.0"

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_canny.py

@@ -0,0 +1,214 @@
+import cv2
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+from PIL import Image
+
+from diffusion_webui.utils.model_list import (
+    controlnet_canny_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+class StableDiffusionControlNetInpaintCannyGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_canny_inpaint(
+        self,
+        image_path: str,
+    ):
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+
+        image = cv2.Canny(image, 100, 200)
+        image = image[:, :, None]
+        image = np.concatenate([image, image, image], axis=2)
+        image = Image.fromarray(image)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_canny_inpaint(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_canny_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_canny_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_canny_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_canny_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_canny_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_canny_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_canny_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_canny_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_canny_model_list,
+                                    value=controlnet_canny_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_canny_inpaint_scheduler = (
+                                    gr.Dropdown(
+                                        choices=SCHEDULER_LIST,
+                                        value=SCHEDULER_LIST[0],
+                                        label="Scheduler",
+                                    )
+                                )
+                                controlnet_canny_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_canny_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_canny_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_canny_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintCannyGenerator().generate_image,
+                inputs=[
+                    controlnet_canny_inpaint_image_file,
+                    controlnet_canny_inpaint_stable_model_id,
+                    controlnet_canny_inpaint_model_id,
+                    controlnet_canny_inpaint_prompt,
+                    controlnet_canny_inpaint_negative_prompt,
+                    controlnet_canny_inpaint_num_images_per_prompt,
+                    controlnet_canny_inpaint_guidance_scale,
+                    controlnet_canny_inpaint_num_inference_step,
+                    controlnet_canny_inpaint_controlnet_conditioning_scale,
+                    controlnet_canny_inpaint_scheduler,
+                    controlnet_canny_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_depth.py

@@ -0,0 +1,211 @@
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+from PIL import Image
+from transformers import pipeline
+
+from diffusion_webui.utils.model_list import (
+    controlnet_depth_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+class StableDiffusionControlInpaintNetDepthGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_inpaint_depth(self, image_path: str):
+        depth_estimator = pipeline("depth-estimation")
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = depth_estimator(image)["depth"]
+        image = np.array(image)
+        image = image[:, :, None]
+        image = np.concatenate([image, image, image], axis=2)
+        image = Image.fromarray(image)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_inpaint_depth(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_depth_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_depth_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_depth_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_depth_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_depth_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_depth_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_depth_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_depth_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_depth_model_list,
+                                    value=controlnet_depth_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_depth_inpaint_scheduler = (
+                                    gr.Dropdown(
+                                        choices=SCHEDULER_LIST,
+                                        value=SCHEDULER_LIST[0],
+                                        label="Scheduler",
+                                    )
+                                )
+                                controlnet_depth_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_depth_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_depth_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_depth_inpaint_predict.click(
+                fn=StableDiffusionControlInpaintNetDepthGenerator().generate_image,
+                inputs=[
+                    controlnet_depth_inpaint_image_file,
+                    controlnet_depth_inpaint_stable_model_id,
+                    controlnet_depth_inpaint_model_id,
+                    controlnet_depth_inpaint_prompt,
+                    controlnet_depth_inpaint_negative_prompt,
+                    controlnet_depth_inpaint_num_images_per_prompt,
+                    controlnet_depth_inpaint_guidance_scale,
+                    controlnet_depth_inpaint_num_inference_step,
+                    controlnet_depth_inpaint_controlnet_conditioning_scale,
+                    controlnet_depth_inpaint_scheduler,
+                    controlnet_depth_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_hed.py

@@ -0,0 +1,205 @@
+import gradio as gr
+import numpy as np
+import torch
+from controlnet_aux import HEDdetector
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+
+from diffusion_webui.utils.model_list import (
+    controlnet_hed_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+class StableDiffusionControlNetInpaintHedGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_inpaint_hed(self, image_path: str):
+        hed = HEDdetector.from_pretrained("lllyasviel/ControlNet")
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+        image = hed(image)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_inpaint_hed(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_hed_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_hed_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_hed_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_hed_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_hed_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_hed_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_hed_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_hed_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_hed_model_list,
+                                    value=controlnet_hed_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_hed_inpaint_scheduler = gr.Dropdown(
+                                    choices=SCHEDULER_LIST,
+                                    value=SCHEDULER_LIST[0],
+                                    label="Scheduler",
+                                )
+                                controlnet_hed_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_hed_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_hed_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_hed_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintHedGenerator().generate_image,
+                inputs=[
+                    controlnet_hed_inpaint_image_file,
+                    controlnet_hed_inpaint_stable_model_id,
+                    controlnet_hed_inpaint_model_id,
+                    controlnet_hed_inpaint_prompt,
+                    controlnet_hed_inpaint_negative_prompt,
+                    controlnet_hed_inpaint_num_images_per_prompt,
+                    controlnet_hed_inpaint_guidance_scale,
+                    controlnet_hed_inpaint_num_inference_step,
+                    controlnet_hed_inpaint_controlnet_conditioning_scale,
+                    controlnet_hed_inpaint_scheduler,
+                    controlnet_hed_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_mlsd.py

@@ -0,0 +1,205 @@
+import gradio as gr
+import numpy as np
+import torch
+from controlnet_aux import MLSDdetector
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+
+from diffusion_webui.utils.model_list import (
+    controlnet_mlsd_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+class StableDiffusionControlNetInpaintMlsdGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_inpaint_mlsd(self, image_path: str):
+        mlsd = MLSDdetector.from_pretrained("lllyasviel/ControlNet")
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+        image = mlsd(image)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_inpaint_mlsd(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_mlsd_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_mlsd_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_mlsd_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_mlsd_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_mlsd_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_mlsd_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_mlsd_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_mlsd_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_mlsd_model_list,
+                                    value=controlnet_mlsd_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_mlsd_inpaint_scheduler = gr.Dropdown(
+                                    choices=SCHEDULER_LIST,
+                                    value=SCHEDULER_LIST[0],
+                                    label="Scheduler",
+                                )
+                                controlnet_mlsd_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_mlsd_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_mlsd_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_mlsd_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintMlsdGenerator().generate_image,
+                inputs=[
+                    controlnet_mlsd_inpaint_image_file,
+                    controlnet_mlsd_inpaint_stable_model_id,
+                    controlnet_mlsd_inpaint_model_id,
+                    controlnet_mlsd_inpaint_prompt,
+                    controlnet_mlsd_inpaint_negative_prompt,
+                    controlnet_mlsd_inpaint_num_images_per_prompt,
+                    controlnet_mlsd_inpaint_guidance_scale,
+                    controlnet_mlsd_inpaint_num_inference_step,
+                    controlnet_mlsd_inpaint_controlnet_conditioning_scale,
+                    controlnet_mlsd_inpaint_scheduler,
+                    controlnet_mlsd_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_pose.py

@@ -0,0 +1,207 @@
+import gradio as gr
+import numpy as np
+import torch
+from controlnet_aux import OpenposeDetector
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+
+from diffusion_webui.utils.model_list import (
+    controlnet_pose_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+class StableDiffusionControlNetInpaintPoseGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_pose_inpaint(self, image_path: str):
+        openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
+
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+        image = openpose(image)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_pose_inpaint(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_pose_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_pose_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_pose_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_pose_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_pose_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_pose_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_pose_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_pose_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_pose_model_list,
+                                    value=controlnet_pose_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_pose_inpaint_scheduler = gr.Dropdown(
+                                    choices=SCHEDULER_LIST,
+                                    value=SCHEDULER_LIST[0],
+                                    label="Scheduler",
+                                )
+                                controlnet_pose_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_pose_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_pose_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_pose_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintPoseGenerator().generate_image,
+                inputs=[
+                    controlnet_pose_inpaint_image_file,
+                    controlnet_pose_inpaint_stable_model_id,
+                    controlnet_pose_inpaint_model_id,
+                    controlnet_pose_inpaint_prompt,
+                    controlnet_pose_inpaint_negative_prompt,
+                    controlnet_pose_inpaint_num_images_per_prompt,
+                    controlnet_pose_inpaint_guidance_scale,
+                    controlnet_pose_inpaint_num_inference_step,
+                    controlnet_pose_inpaint_controlnet_conditioning_scale,
+                    controlnet_pose_inpaint_scheduler,
+                    controlnet_pose_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_scribble.py

@@ -0,0 +1,210 @@
+import gradio as gr
+import numpy as np
+import torch
+from controlnet_aux import HEDdetector
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+
+from diffusion_webui.utils.model_list import (
+    controlnet_scribble_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+class StableDiffusionControlNetInpaintScribbleGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_inpaint_scribble(self, image_path: str):
+        hed = HEDdetector.from_pretrained("lllyasviel/ControlNet")
+
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+        image = hed(image, scribble=True)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_inpaint_scribble(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_scribble_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_scribble_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_scribble_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_scribble_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_scribble_inpaint_guidance_scale = (
+                                gr.Slider(
+                                    minimum=0.1,
+                                    maximum=15,
+                                    step=0.1,
+                                    value=7.5,
+                                    label="Guidance Scale",
+                                )
+                            )
+
+                            controlnet_scribble_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_scribble_inpaint_num_images_per_prompt = gr.Slider(
+                                minimum=1,
+                                maximum=10,
+                                step=1,
+                                value=1,
+                                label="Number Of Images",
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_scribble_inpaint_model_id = (
+                                    gr.Dropdown(
+                                        choices=controlnet_scribble_model_list,
+                                        value=controlnet_scribble_model_list[0],
+                                        label="Controlnet Model Id",
+                                    )
+                                )
+                                controlnet_scribble_inpaint_scheduler = (
+                                    gr.Dropdown(
+                                        choices=SCHEDULER_LIST,
+                                        value=SCHEDULER_LIST[0],
+                                        label="Scheduler",
+                                    )
+                                )
+                                controlnet_scribble_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_scribble_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_scribble_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_scribble_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintScribbleGenerator().generate_image,
+                inputs=[
+                    controlnet_scribble_inpaint_image_file,
+                    controlnet_scribble_inpaint_stable_model_id,
+                    controlnet_scribble_inpaint_model_id,
+                    controlnet_scribble_inpaint_prompt,
+                    controlnet_scribble_inpaint_negative_prompt,
+                    controlnet_scribble_inpaint_num_images_per_prompt,
+                    controlnet_scribble_inpaint_guidance_scale,
+                    controlnet_scribble_inpaint_num_inference_step,
+                    controlnet_scribble_inpaint_controlnet_conditioning_scale,
+                    controlnet_scribble_inpaint_scheduler,
+                    controlnet_scribble_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_inpaint/controlnet_inpaint_seg.py

@@ -0,0 +1,390 @@
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+from PIL import Image
+from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
+
+from diffusion_webui.utils.model_list import (
+    controlnet_seg_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_LIST,
+    get_scheduler_list,
+)
+
+# https://github.com/mikonvergence/ControlNetInpaint
+
+
+def ade_palette():
+    """ADE20K palette that maps each class to RGB values."""
+    return [
+        [120, 120, 120],
+        [180, 120, 120],
+        [6, 230, 230],
+        [80, 50, 50],
+        [4, 200, 3],
+        [120, 120, 80],
+        [140, 140, 140],
+        [204, 5, 255],
+        [230, 230, 230],
+        [4, 250, 7],
+        [224, 5, 255],
+        [235, 255, 7],
+        [150, 5, 61],
+        [120, 120, 70],
+        [8, 255, 51],
+        [255, 6, 82],
+        [143, 255, 140],
+        [204, 255, 4],
+        [255, 51, 7],
+        [204, 70, 3],
+        [0, 102, 200],
+        [61, 230, 250],
+        [255, 6, 51],
+        [11, 102, 255],
+        [255, 7, 71],
+        [255, 9, 224],
+        [9, 7, 230],
+        [220, 220, 220],
+        [255, 9, 92],
+        [112, 9, 255],
+        [8, 255, 214],
+        [7, 255, 224],
+        [255, 184, 6],
+        [10, 255, 71],
+        [255, 41, 10],
+        [7, 255, 255],
+        [224, 255, 8],
+        [102, 8, 255],
+        [255, 61, 6],
+        [255, 194, 7],
+        [255, 122, 8],
+        [0, 255, 20],
+        [255, 8, 41],
+        [255, 5, 153],
+        [6, 51, 255],
+        [235, 12, 255],
+        [160, 150, 20],
+        [0, 163, 255],
+        [140, 140, 140],
+        [250, 10, 15],
+        [20, 255, 0],
+        [31, 255, 0],
+        [255, 31, 0],
+        [255, 224, 0],
+        [153, 255, 0],
+        [0, 0, 255],
+        [255, 71, 0],
+        [0, 235, 255],
+        [0, 173, 255],
+        [31, 0, 255],
+        [11, 200, 200],
+        [255, 82, 0],
+        [0, 255, 245],
+        [0, 61, 255],
+        [0, 255, 112],
+        [0, 255, 133],
+        [255, 0, 0],
+        [255, 163, 0],
+        [255, 102, 0],
+        [194, 255, 0],
+        [0, 143, 255],
+        [51, 255, 0],
+        [0, 82, 255],
+        [0, 255, 41],
+        [0, 255, 173],
+        [10, 0, 255],
+        [173, 255, 0],
+        [0, 255, 153],
+        [255, 92, 0],
+        [255, 0, 255],
+        [255, 0, 245],
+        [255, 0, 102],
+        [255, 173, 0],
+        [255, 0, 20],
+        [255, 184, 184],
+        [0, 31, 255],
+        [0, 255, 61],
+        [0, 71, 255],
+        [255, 0, 204],
+        [0, 255, 194],
+        [0, 255, 82],
+        [0, 10, 255],
+        [0, 112, 255],
+        [51, 0, 255],
+        [0, 194, 255],
+        [0, 122, 255],
+        [0, 255, 163],
+        [255, 153, 0],
+        [0, 255, 10],
+        [255, 112, 0],
+        [143, 255, 0],
+        [82, 0, 255],
+        [163, 255, 0],
+        [255, 235, 0],
+        [8, 184, 170],
+        [133, 0, 255],
+        [0, 255, 92],
+        [184, 0, 255],
+        [255, 0, 31],
+        [0, 184, 255],
+        [0, 214, 255],
+        [255, 0, 112],
+        [92, 255, 0],
+        [0, 224, 255],
+        [112, 224, 255],
+        [70, 184, 160],
+        [163, 0, 255],
+        [153, 0, 255],
+        [71, 255, 0],
+        [255, 0, 163],
+        [255, 204, 0],
+        [255, 0, 143],
+        [0, 255, 235],
+        [133, 255, 0],
+        [255, 0, 235],
+        [245, 0, 255],
+        [255, 0, 122],
+        [255, 245, 0],
+        [10, 190, 212],
+        [214, 255, 0],
+        [0, 204, 255],
+        [20, 0, 255],
+        [255, 255, 0],
+        [0, 153, 255],
+        [0, 41, 255],
+        [0, 255, 204],
+        [41, 0, 255],
+        [41, 255, 0],
+        [173, 0, 255],
+        [0, 245, 255],
+        [71, 0, 255],
+        [122, 0, 255],
+        [0, 255, 184],
+        [0, 92, 255],
+        [184, 255, 0],
+        [0, 133, 255],
+        [255, 214, 0],
+        [25, 194, 194],
+        [102, 255, 0],
+        [92, 0, 255],
+    ]
+
+
+class StableDiffusionControlNetInpaintSegGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(
+        self,
+        stable_model_path,
+        controlnet_model_path,
+        scheduler,
+    ):
+
+        if self.pipe is None:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+
+        self.pipe = get_scheduler_list(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def controlnet_seg_inpaint(self, image_path: str):
+        image_processor = AutoImageProcessor.from_pretrained(
+            "openmmlab/upernet-convnext-small"
+        )
+        image_segmentor = UperNetForSemanticSegmentation.from_pretrained(
+            "openmmlab/upernet-convnext-small"
+        )
+
+        image = image_path["image"].convert("RGB").resize((512, 512))
+        image = np.array(image)
+        pixel_values = image_processor(image, return_tensors="pt").pixel_values
+
+        with torch.no_grad():
+            outputs = image_segmentor(pixel_values)
+
+        seg = image_processor.post_process_semantic_segmentation(
+            outputs, target_sizes=[image.size[::-1]]
+        )[0]
+
+        color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
+        palette = np.array(ade_palette())
+
+        for label, color in enumerate(palette):
+            color_seg[seg == label, :] = color
+
+        color_seg = color_seg.astype(np.uint8)
+        image = Image.fromarray(color_seg)
+
+        return image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+
+        image = self.controlnet_seg_inpaint(image_path=image_path)
+
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_seg_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    )
+
+                    controlnet_seg_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+
+                    controlnet_seg_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        show_label=False,
+                        placeholder="Negative Prompt",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_seg_inpaint_stable_model_id = (
+                                gr.Dropdown(
+                                    choices=stable_model_list,
+                                    value=stable_model_list[0],
+                                    label="Stable Model Id",
+                                )
+                            )
+
+                            controlnet_seg_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            controlnet_seg_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+                            controlnet_seg_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_seg_inpaint_model_id = gr.Dropdown(
+                                    choices=controlnet_seg_model_list,
+                                    value=controlnet_seg_model_list[0],
+                                    label="Controlnet Model Id",
+                                )
+                                controlnet_seg_inpaint_scheduler = gr.Dropdown(
+                                    choices=SCHEDULER_LIST,
+                                    value=SCHEDULER_LIST[0],
+                                    label="Scheduler",
+                                )
+                                controlnet_seg_inpaint_controlnet_conditioning_scale = gr.Slider(
+                                    minimum=0.1,
+                                    maximum=1.0,
+                                    step=0.1,
+                                    value=0.5,
+                                    label="Controlnet Conditioning Scale",
+                                )
+
+                                controlnet_seg_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed Generator",
+                                    )
+                                )
+
+                    controlnet_seg_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            controlnet_seg_inpaint_predict.click(
+                fn=StableDiffusionControlNetInpaintSegGenerator().generate_image,
+                inputs=[
+                    controlnet_seg_inpaint_image_file,
+                    controlnet_seg_inpaint_stable_model_id,
+                    controlnet_seg_inpaint_model_id,
+                    controlnet_seg_inpaint_prompt,
+                    controlnet_seg_inpaint_negative_prompt,
+                    controlnet_seg_inpaint_num_images_per_prompt,
+                    controlnet_seg_inpaint_guidance_scale,
+                    controlnet_seg_inpaint_num_inference_step,
+                    controlnet_seg_inpaint_controlnet_conditioning_scale,
+                    controlnet_seg_inpaint_scheduler,
+                    controlnet_seg_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/controlnet/controlnet_seg.py

@@ -203,7 +203,7 @@ class StableDiffusionControlNetSegGenerator:
             "openmmlab/upernet-convnext-small"
         )
 
-        image = Image.open(image_path).convert("RGB")
+        image = image_path["image"].convert("RGB").resize((512, 512))
         pixel_values = image_processor(image, return_tensors="pt").pixel_values
 
         with torch.no_grad():

diffusion_webui/helpers.py

@@ -7,8 +7,26 @@ from diffusion_webui.diffusion_models.controlnet.controlnet_depth import (
 from diffusion_webui.diffusion_models.controlnet.controlnet_hed import (
     StableDiffusionControlNetHEDGenerator,
 )
-from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_app import (
-    StableDiffusionControlInpaintNetCannyGenerator,
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_canny import (
+    StableDiffusionControlNetInpaintCannyGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_depth import (
+    StableDiffusionControlInpaintNetDepthGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_hed import (
+    StableDiffusionControlNetInpaintHedGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_mlsd import (
+    StableDiffusionControlNetInpaintMlsdGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_pose import (
+    StableDiffusionControlNetInpaintPoseGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_scribble import (
+    StableDiffusionControlNetInpaintScribbleGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet.controlnet_inpaint.controlnet_inpaint_seg import (
+    StableDiffusionControlNetInpaintSegGenerator,
 )
 from diffusion_webui.diffusion_models.controlnet.controlnet_mlsd import (
     StableDiffusionControlNetMLSDGenerator,

diffusion_webui/utils/model_list.py

@@ -1,7 +1,7 @@
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
     "stabilityai/stable-diffusion-2-1",
-    "prompthero/openjourney"
+    "prompthero/openjourney-v4",
 ]
 
 controlnet_canny_model_list = [
@@ -32,3 +32,11 @@ stable_inpiant_model_list = [
     "stabilityai/stable-diffusion-2-inpainting",
     "runwayml/stable-diffusion-inpainting",
 ]
+
+controlnet_mlsd_model_list = [
+    "lllyasviel/sd-controlnet-mlsd",
+]
+
+controlnet_seg_model_list = [
+    "lllyasviel/sd-controlnet-seg",
+]