Update README.md
Browse files
README.md
CHANGED
@@ -38,4 +38,263 @@ support any aspect ratio and any times upscale, followings are 3 * 3 times
|
|
38 |
|
39 |
![images_7)](./000053.webp)
|
40 |
|
41 |
-
![images_8)](./000053_scribble.webp)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
|
39 |
![images_7)](./000053.webp)
|
40 |
|
41 |
+
![images_8)](./000053_scribble.webp)
|
42 |
+
|
43 |
+
# Code to Use Tile blur
|
44 |
+
|
45 |
+
code reference: https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic/blob/main/TTP_tile_preprocessor_v5.py
|
46 |
+
https://github.com/lllyasviel/ControlNet-v1-1-nightly/blob/main/gradio_tile.py
|
47 |
+
|
48 |
+
```python
|
49 |
+
from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
|
50 |
+
from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler
|
51 |
+
from PIL import Image
|
52 |
+
from guided_filter import FastGuidedFilter # I have upload this file in this repo
|
53 |
+
import torch
|
54 |
+
import numpy as np
|
55 |
+
import cv2
|
56 |
+
|
57 |
+
def resize_image_control(control_image, resolution):
|
58 |
+
HH, WW, _ = control_image.shape
|
59 |
+
crop_h = random.randint(0, HH - resolution[1])
|
60 |
+
crop_w = random.randint(0, WW - resolution[0])
|
61 |
+
crop_image = control_image[crop_h:crop_h+resolution[1], crop_w:crop_w+resolution[0], :]
|
62 |
+
return crop_image, crop_w, crop_h
|
63 |
+
|
64 |
+
def apply_gaussian_blur(image_np, ksize=5, sigmaX=1.0):
|
65 |
+
if ksize % 2 == 0:
|
66 |
+
ksize += 1 # ksize must be odd
|
67 |
+
blurred_image = cv2.GaussianBlur(image_np, (ksize, ksize), sigmaX=sigmaX)
|
68 |
+
return blurred_image
|
69 |
+
|
70 |
+
def apply_guided_filter(image_np, radius, eps, scale):
|
71 |
+
filter = FastGuidedFilter(image_np, radius, eps, scale)
|
72 |
+
return filter.filter(image_np)
|
73 |
+
|
74 |
+
|
75 |
+
controlnet_conditioning_scale = 1.0
|
76 |
+
prompt = "your prompt, the longer the better, you can describe it as detail as possible"
|
77 |
+
negative_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
|
78 |
+
|
79 |
+
eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
|
80 |
+
|
81 |
+
|
82 |
+
controlnet = ControlNetModel.from_pretrained(
|
83 |
+
"xinsir/controlnet-tile-sdxl-1.0",
|
84 |
+
torch_dtype=torch.float16
|
85 |
+
)
|
86 |
+
|
87 |
+
# when test with other base model, you need to change the vae also.
|
88 |
+
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
|
89 |
+
|
90 |
+
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
|
91 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
92 |
+
controlnet=controlnet,
|
93 |
+
vae=vae,
|
94 |
+
safety_checker=None,
|
95 |
+
torch_dtype=torch.float16,
|
96 |
+
scheduler=eulera_scheduler,
|
97 |
+
)
|
98 |
+
|
99 |
+
controlnet_img = cv2.imread("your original image path")
|
100 |
+
height, width, _ = controlnet_img.shape
|
101 |
+
ratio = np.sqrt(1024. * 1024. / (width * height))
|
102 |
+
W, H = int(width * ratio), int(height * ratio)
|
103 |
+
|
104 |
+
crop_w, crop_h = 0, 0
|
105 |
+
controlnet_img = cv2.resize(controlnet_img, (W, H))
|
106 |
+
|
107 |
+
|
108 |
+
blur_strength = random.sample([i / 10. for i in range(10, 201, 2)], k=1)[0]
|
109 |
+
radius = random.sample([i for i in range(1, 40, 2)], k=1)[0]
|
110 |
+
eps = random.sample([i / 1000. for i in range(1, 101, 2)], k=1)[0]
|
111 |
+
scale_factor = random.sample([i / 10. for i in range(10, 181, 5)], k=1)[0]
|
112 |
+
|
113 |
+
|
114 |
+
if random.random() > 0.5:
|
115 |
+
controlnet_img = apply_gaussian_blur(controlnet_img, ksize=int(blur_strength), sigmaX=blur_strength / 2)
|
116 |
+
|
117 |
+
if random.random() > 0.5:
|
118 |
+
# Apply Guided Filter
|
119 |
+
controlnet_img = apply_guided_filter(controlnet_img, radius, eps, scale_factor)
|
120 |
+
|
121 |
+
# Resize image
|
122 |
+
controlnet_img = cv2.resize(controlnet_img, (int(W / scale_factor), int(H / scale_factor)), interpolation=cv2.INTER_AREA)
|
123 |
+
controlnet_img = cv2.resize(controlnet_img, (W, H), interpolation=cv2.INTER_CUBIC)
|
124 |
+
|
125 |
+
controlnet_img = cv2.cvtColor(controlnet_img, cv2.COLOR_BGR2RGB)
|
126 |
+
controlnet_img = Image.fromarray(controlnet_img)
|
127 |
+
|
128 |
+
# need to resize the image resolution to 1024 * 1024 or same bucket resolution to get the best performance
|
129 |
+
|
130 |
+
images = pipe(
|
131 |
+
prompt,
|
132 |
+
negative_prompt=negative_prompt,
|
133 |
+
image=controlnet_img,
|
134 |
+
controlnet_conditioning_scale=controlnet_conditioning_scale,
|
135 |
+
width=new_width,
|
136 |
+
height=new_height,
|
137 |
+
num_inference_steps=30,
|
138 |
+
).images
|
139 |
+
|
140 |
+
images[0].save(f"your image save path, png format is usually better than jpg or webp in terms of image quality but got much bigger")
|
141 |
+
|
142 |
+
```
|
143 |
+
# Code to Use Tile var
|
144 |
+
|
145 |
+
Use more detail prompt to regerate can help!
|
146 |
+
|
147 |
+
```python
|
148 |
+
from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
|
149 |
+
from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler
|
150 |
+
from PIL import Image
|
151 |
+
import torch
|
152 |
+
import numpy as np
|
153 |
+
import cv2
|
154 |
+
|
155 |
+
controlnet_conditioning_scale = 1.0
|
156 |
+
prompt = "your prompt, the longer the better, you can describe it as detail as possible"
|
157 |
+
negative_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
|
158 |
+
|
159 |
+
eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
|
160 |
+
|
161 |
+
|
162 |
+
controlnet = ControlNetModel.from_pretrained(
|
163 |
+
"xinsir/controlnet-tile-sdxl-1.0",
|
164 |
+
torch_dtype=torch.float16
|
165 |
+
)
|
166 |
+
|
167 |
+
# when test with other base model, you need to change the vae also.
|
168 |
+
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
|
169 |
+
|
170 |
+
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
|
171 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
172 |
+
controlnet=controlnet,
|
173 |
+
vae=vae,
|
174 |
+
safety_checker=None,
|
175 |
+
torch_dtype=torch.float16,
|
176 |
+
scheduler=eulera_scheduler,
|
177 |
+
)
|
178 |
+
|
179 |
+
controlnet_img = cv2.imread("your original image path")
|
180 |
+
height, width, _ = controlnet_img.shape
|
181 |
+
ratio = np.sqrt(1024. * 1024. / (width * height))
|
182 |
+
W, H = int(width * ratio), int(height * ratio)
|
183 |
+
|
184 |
+
crop_w, crop_h = 0, 0
|
185 |
+
controlnet_img = cv2.resize(controlnet_img, (W, H))
|
186 |
+
controlnet_img = cv2.cvtColor(controlnet_img, cv2.COLOR_BGR2RGB)
|
187 |
+
controlnet_img = Image.fromarray(controlnet_img)
|
188 |
+
|
189 |
+
# need to resize the image resolution to 1024 * 1024 or same bucket resolution to get the best performance
|
190 |
+
images = pipe(
|
191 |
+
prompt,
|
192 |
+
negative_prompt=negative_prompt,
|
193 |
+
image=controlnet_img,
|
194 |
+
controlnet_conditioning_scale=controlnet_conditioning_scale,
|
195 |
+
width=new_width,
|
196 |
+
height=new_height,
|
197 |
+
num_inference_steps=30,
|
198 |
+
).images
|
199 |
+
|
200 |
+
images[0].save(f"your image save path, png format is usually better than jpg or webp in terms of image quality but got much bigger")
|
201 |
+
|
202 |
+
```
|
203 |
+
|
204 |
+
|
205 |
+
# Code to Use Tile super
|
206 |
+
|
207 |
+
performance may unstable and next version is optimizing!
|
208 |
+
|
209 |
+
```python
|
210 |
+
from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
|
211 |
+
from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler
|
212 |
+
from PIL import Image
|
213 |
+
import torch
|
214 |
+
import numpy as np
|
215 |
+
import cv2
|
216 |
+
|
217 |
+
controlnet_conditioning_scale = 1.0
|
218 |
+
prompt = "your prompt, the longer the better, you can describe it as detail as possible"
|
219 |
+
negative_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
|
220 |
+
|
221 |
+
eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
|
222 |
+
|
223 |
+
|
224 |
+
controlnet = ControlNetModel.from_pretrained(
|
225 |
+
"xinsir/controlnet-tile-sdxl-1.0",
|
226 |
+
torch_dtype=torch.float16
|
227 |
+
)
|
228 |
+
|
229 |
+
# when test with other base model, you need to change the vae also.
|
230 |
+
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
|
231 |
+
|
232 |
+
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
|
233 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
234 |
+
controlnet=controlnet,
|
235 |
+
vae=vae,
|
236 |
+
safety_checker=None,
|
237 |
+
torch_dtype=torch.float16,
|
238 |
+
scheduler=eulera_scheduler,
|
239 |
+
)
|
240 |
+
|
241 |
+
controlnet_img = cv2.imread("your original image path")
|
242 |
+
height, width, _ = controlnet_img.shape
|
243 |
+
ratio = np.sqrt(1024. * 1024. / (width * height))
|
244 |
+
W, H = int(width * ratio) // 48 * 48, int(height * ratio) // 48 * 48
|
245 |
+
controlnet_img = cv2.resize(controlnet_img, (W, H))
|
246 |
+
controlnet_img = cv2.cvtColor(controlnet_img, cv2.COLOR_BGR2RGB)
|
247 |
+
controlnet_img = Image.fromarray(controlnet_img)
|
248 |
+
|
249 |
+
# need to resize the image resolution to 1024 * 1024 or same bucket resolution to get the best performance
|
250 |
+
target_width = W // 3
|
251 |
+
target_height = H // 3
|
252 |
+
|
253 |
+
for i in range(3): # 两行
|
254 |
+
for j in range(3): # 两列
|
255 |
+
left = j * target_width
|
256 |
+
top = i * target_height
|
257 |
+
right = left + target_width
|
258 |
+
bottom = top + target_height
|
259 |
+
|
260 |
+
# 根据计算的边界裁剪图像
|
261 |
+
cropped_image = controlnet_img.crop((left, top, right, bottom))
|
262 |
+
cropped_image = cropped_image.resize((W, H))
|
263 |
+
|
264 |
+
images.append(cropped_image)
|
265 |
+
|
266 |
+
seed = random.randint(0, 2147483647)
|
267 |
+
generator = torch.Generator('cuda').manual_seed(seed)
|
268 |
+
|
269 |
+
result_images = []
|
270 |
+
for sub_img in images:
|
271 |
+
new_width, new_height = W, H
|
272 |
+
out = sd_model(prompt=[prompt]*1,
|
273 |
+
image=sub_img,
|
274 |
+
control_image=sub_img,
|
275 |
+
negative_prompt=[negative_prompt]*1,
|
276 |
+
generator=generator,
|
277 |
+
width=new_width,
|
278 |
+
height=new_height,
|
279 |
+
num_inference_steps=30,
|
280 |
+
crops_coords_top_left=(W, H),
|
281 |
+
target_size=(W, H),
|
282 |
+
original_size=(W * 2, H * 2),
|
283 |
+
)
|
284 |
+
result_images.append(out.images[0])
|
285 |
+
|
286 |
+
new_im = Image.new('RGB', (new_width*3, new_height*3))
|
287 |
+
# 拼接图片到新的图像上
|
288 |
+
new_im.paste(result_images[0], (0, 0))
|
289 |
+
new_im.paste(result_images[1], (new_width, 0))
|
290 |
+
new_im.paste(result_images[2], (new_width * 2, 0))
|
291 |
+
new_im.paste(result_images[3], (0, new_height))
|
292 |
+
new_im.paste(result_images[4], (new_width, new_height))
|
293 |
+
new_im.paste(result_images[5], (new_width * 2, new_height))
|
294 |
+
new_im.paste(result_images[6], (0, new_height * 2))
|
295 |
+
new_im.paste(result_images[7], (new_width, new_height * 2))
|
296 |
+
new_im.paste(result_images[8], (new_width * 2, new_height * 2))
|
297 |
+
|
298 |
+
new_im.save(f"your image save path, png format is usually better than jpg or webp in terms of image quality but got much bigger")
|
299 |
+
|
300 |
+
```
|