Spaces:
Runtime error
Runtime error
VikramSingh178
commited on
Commit
β’
0a7de9a
1
Parent(s):
a8d1f41
commit
Browse filesFormer-commit-id: 0e58c57fa85967602d1f836634abad55d585c997
- logs/app_debug.log +18 -0
- logs/app_info.log +18 -0
- masks/mask.jpg +0 -0
- masks/mask_0.jpg +0 -0
- masks/mask_1.jpg +0 -0
- masks/mask_2.jpg +0 -0
- masks/mask_3.jpg +0 -0
- masks/mask_4.jpg +0 -0
- masks/pot_inverted_mask.jpg +0 -0
- masks/pot_original_mask.jpg +0 -0
- output/pot_output.jpg +0 -0
- scripts/__pycache__/mask_generator.cpython-310.pyc +0 -0
- scripts/__pycache__/models.cpython-310.pyc +0 -0
- scripts/__pycache__/pipeline.cpython-310.pyc +0 -0
- scripts/invert_mask.jpg +0 -0
- scripts/mask.jpg +0 -0
- scripts/mask_generator.py +13 -27
- scripts/models.py +8 -23
- scripts/output.jpg +0 -0
- scripts/run.py +45 -0
logs/app_debug.log
CHANGED
@@ -1371,3 +1371,21 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
|
|
1371 |
2024-03-22 06:39:26,709 [INFO] models - Kandinsky Inpainting Inference
|
1372 |
2024-03-22 06:43:26,086 [INFO] models - Kandinsky Inpainting Inference
|
1373 |
2024-03-22 06:53:54,562 [INFO] models - Kandinsky Inpainting Inference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1371 |
2024-03-22 06:39:26,709 [INFO] models - Kandinsky Inpainting Inference
|
1372 |
2024-03-22 06:43:26,086 [INFO] models - Kandinsky Inpainting Inference
|
1373 |
2024-03-22 06:53:54,562 [INFO] models - Kandinsky Inpainting Inference
|
1374 |
+
2024-03-22 07:03:17,990 [INFO] models - Kandinsky Inpainting Inference
|
1375 |
+
2024-03-22 07:09:38,955 [INFO] models - Kandinsky Inpainting Inference
|
1376 |
+
2024-03-22 07:17:59,975 [INFO] models - Kandinsky Inpainting Inference
|
1377 |
+
2024-03-22 07:28:54,339 [INFO] models - Kandinsky Inpainting Inference
|
1378 |
+
2024-03-22 07:41:45,300 [INFO] models - Kandinsky Inpainting Inference
|
1379 |
+
2024-03-22 07:50:46,880 [INFO] models - Kandinsky Inpainting Inference
|
1380 |
+
2024-03-22 08:05:52,674 [INFO] models - Kandinsky Inpainting Inference
|
1381 |
+
2024-03-22 08:11:07,093 [INFO] models - Kandinsky Inpainting Inference
|
1382 |
+
2024-03-22 08:49:32,092 [INFO] models - Kandinsky Inpainting Inference
|
1383 |
+
2024-03-22 08:56:43,084 [INFO] models - Kandinsky Inpainting Inference
|
1384 |
+
2024-03-22 09:13:03,681 [INFO] models - Kandinsky Inpainting Inference
|
1385 |
+
2024-03-22 09:23:45,335 [INFO] models - Kandinsky Inpainting Inference
|
1386 |
+
2024-03-22 09:29:54,960 [INFO] models - Kandinsky Inpainting Inference
|
1387 |
+
2024-03-22 09:31:28,680 [INFO] models - Kandinsky Inpainting Inference
|
1388 |
+
2024-03-22 09:39:32,351 [INFO] models - Kandinsky Inpainting Inference
|
1389 |
+
2024-03-22 09:47:05,180 [INFO] models - Kandinsky Inpainting Inference
|
1390 |
+
2024-03-22 09:51:28,523 [INFO] models - Kandinsky Inpainting Inference
|
1391 |
+
2024-03-22 09:53:18,039 [INFO] models - Kandinsky Inpainting Inference
|
logs/app_info.log
CHANGED
@@ -1371,3 +1371,21 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
|
|
1371 |
2024-03-22 06:39:26,709 [INFO] models - Kandinsky Inpainting Inference
|
1372 |
2024-03-22 06:43:26,086 [INFO] models - Kandinsky Inpainting Inference
|
1373 |
2024-03-22 06:53:54,562 [INFO] models - Kandinsky Inpainting Inference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1371 |
2024-03-22 06:39:26,709 [INFO] models - Kandinsky Inpainting Inference
|
1372 |
2024-03-22 06:43:26,086 [INFO] models - Kandinsky Inpainting Inference
|
1373 |
2024-03-22 06:53:54,562 [INFO] models - Kandinsky Inpainting Inference
|
1374 |
+
2024-03-22 07:03:17,990 [INFO] models - Kandinsky Inpainting Inference
|
1375 |
+
2024-03-22 07:09:38,955 [INFO] models - Kandinsky Inpainting Inference
|
1376 |
+
2024-03-22 07:17:59,975 [INFO] models - Kandinsky Inpainting Inference
|
1377 |
+
2024-03-22 07:28:54,339 [INFO] models - Kandinsky Inpainting Inference
|
1378 |
+
2024-03-22 07:41:45,300 [INFO] models - Kandinsky Inpainting Inference
|
1379 |
+
2024-03-22 07:50:46,880 [INFO] models - Kandinsky Inpainting Inference
|
1380 |
+
2024-03-22 08:05:52,674 [INFO] models - Kandinsky Inpainting Inference
|
1381 |
+
2024-03-22 08:11:07,093 [INFO] models - Kandinsky Inpainting Inference
|
1382 |
+
2024-03-22 08:49:32,092 [INFO] models - Kandinsky Inpainting Inference
|
1383 |
+
2024-03-22 08:56:43,084 [INFO] models - Kandinsky Inpainting Inference
|
1384 |
+
2024-03-22 09:13:03,681 [INFO] models - Kandinsky Inpainting Inference
|
1385 |
+
2024-03-22 09:23:45,335 [INFO] models - Kandinsky Inpainting Inference
|
1386 |
+
2024-03-22 09:29:54,960 [INFO] models - Kandinsky Inpainting Inference
|
1387 |
+
2024-03-22 09:31:28,680 [INFO] models - Kandinsky Inpainting Inference
|
1388 |
+
2024-03-22 09:39:32,351 [INFO] models - Kandinsky Inpainting Inference
|
1389 |
+
2024-03-22 09:47:05,180 [INFO] models - Kandinsky Inpainting Inference
|
1390 |
+
2024-03-22 09:51:28,523 [INFO] models - Kandinsky Inpainting Inference
|
1391 |
+
2024-03-22 09:53:18,039 [INFO] models - Kandinsky Inpainting Inference
|
masks/mask.jpg
DELETED
Binary file (5.96 kB)
|
|
masks/mask_0.jpg
DELETED
Binary file (22.3 kB)
|
|
masks/mask_1.jpg
DELETED
Binary file (34 kB)
|
|
masks/mask_2.jpg
DELETED
Binary file (20.9 kB)
|
|
masks/mask_3.jpg
DELETED
Binary file (50.7 kB)
|
|
masks/mask_4.jpg
DELETED
Binary file (21.3 kB)
|
|
masks/pot_inverted_mask.jpg
ADDED
masks/pot_original_mask.jpg
ADDED
output/pot_output.jpg
ADDED
scripts/__pycache__/mask_generator.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/mask_generator.cpython-310.pyc and b/scripts/__pycache__/mask_generator.cpython-310.pyc differ
|
|
scripts/__pycache__/models.cpython-310.pyc
ADDED
Binary file (6.39 kB). View file
|
|
scripts/__pycache__/pipeline.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/pipeline.cpython-310.pyc and b/scripts/__pycache__/pipeline.cpython-310.pyc differ
|
|
scripts/invert_mask.jpg
DELETED
Binary file (18.8 kB)
|
|
scripts/mask.jpg
DELETED
Binary file (18.7 kB)
|
|
scripts/mask_generator.py
CHANGED
@@ -4,10 +4,8 @@ from logger import rich_logger as l
|
|
4 |
from ultralytics import YOLO
|
5 |
import cv2
|
6 |
from config import yolo_model
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
|
12 |
|
13 |
|
@@ -15,43 +13,31 @@ from config import yolo_model
|
|
15 |
def generate_mask(image_path: str) -> np.ndarray:
|
16 |
"""Method to segment image
|
17 |
Args:
|
18 |
-
|
19 |
Returns:
|
20 |
-
|
21 |
"""
|
22 |
-
model = YOLO(model=yolo_model)
|
23 |
-
results = model(image_path)
|
24 |
for result in results:
|
25 |
orig_img = result.orig_img
|
26 |
masks = result.masks.xy
|
27 |
height, width = result.orig_img.shape[:2]
|
28 |
-
|
29 |
|
30 |
for mask in masks:
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
orig_img = np.array(orig_img)
|
36 |
-
|
37 |
-
return mask_img, orig_img
|
38 |
|
39 |
-
def invert_mask(mask_image:
|
40 |
"""Method to invert mask
|
41 |
Args:
|
42 |
mask_image (np.ndarray): input mask image
|
43 |
Returns:
|
44 |
np.ndarray: inverted mask image
|
45 |
"""
|
46 |
-
inverted_mask_image =
|
47 |
-
cv2.imwrite('invert_mask.jpg', inverted_mask_image)
|
48 |
return inverted_mask_image
|
49 |
|
50 |
-
|
51 |
-
if __name__ == "__main__":
|
52 |
-
image = Image.open("../sample_data/example1.jpg")
|
53 |
-
mask_img,orig_image = generate_mask(image_path='../sample_data/example1.jpg')
|
54 |
-
invert_mask(mask_image=mask_img)
|
55 |
-
|
56 |
-
|
57 |
-
|
|
|
4 |
from ultralytics import YOLO
|
5 |
import cv2
|
6 |
from config import yolo_model
|
7 |
+
from pathlib import Path
|
8 |
+
import PIL.ImageOps
|
|
|
|
|
9 |
|
10 |
|
11 |
|
|
|
13 |
def generate_mask(image_path: str) -> np.ndarray:
|
14 |
"""Method to segment image
|
15 |
Args:
|
16 |
+
image_path (str): path to input image
|
17 |
Returns:
|
18 |
+
np.ndarray: segmented image mask
|
19 |
"""
|
20 |
+
model = YOLO(model=yolo_model) # Initialize YOLO model
|
21 |
+
results = model(image_path) # Perform object detection
|
22 |
for result in results:
|
23 |
orig_img = result.orig_img
|
24 |
masks = result.masks.xy
|
25 |
height, width = result.orig_img.shape[:2]
|
26 |
+
mask_img = np.ones((height, width), dtype=np.uint8) * 255 # Initialize mask with white background
|
27 |
|
28 |
for mask in masks:
|
29 |
+
mask = mask.astype(int)
|
30 |
+
cv2.fillPoly(mask_img, [mask], 0) # Fill mask with detected object areas
|
31 |
+
|
32 |
+
return mask_img
|
|
|
|
|
|
|
33 |
|
34 |
+
def invert_mask(mask_image: Image) -> np.ndarray:
|
35 |
"""Method to invert mask
|
36 |
Args:
|
37 |
mask_image (np.ndarray): input mask image
|
38 |
Returns:
|
39 |
np.ndarray: inverted mask image
|
40 |
"""
|
41 |
+
inverted_mask_image =PIL.ImageOps.invert(mask_image)
|
|
|
42 |
return inverted_mask_image
|
43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
scripts/models.py
CHANGED
@@ -5,13 +5,13 @@ from clear_memory import clear_memory
|
|
5 |
from typing import List
|
6 |
import numpy as np
|
7 |
import torch
|
8 |
-
from PIL import Image
|
9 |
from mask_generator import invert_mask
|
10 |
from diffusers.utils import load_image
|
11 |
from pipeline import fetch_control_pipeline,fetch_kandinsky_pipeline,fetch_kandinsky_prior_pipeline,fetch_kandinsky_img2img_pipeline
|
12 |
from config import controlnet_adapter_model_name,controlnet_base_model_name,kandinsky_model_name
|
13 |
import cv2
|
14 |
-
|
15 |
from transformers import pipeline
|
16 |
|
17 |
|
@@ -126,14 +126,12 @@ def kandinsky_inpainting_inference(prompt, negative_prompt, image, mask_image):
|
|
126 |
Returns:
|
127 |
PIL.Image.Image: The output inpainted image.
|
128 |
"""
|
129 |
-
pipe = fetch_kandinsky_pipeline(controlnet_adapter_model_name, controlnet_base_model_name, kandinsky_model_name, image)
|
130 |
-
output_image = pipe(prompt=prompt, negative_prompt=negative_prompt, image=image, mask_image=mask_image).images[0]
|
131 |
-
return output_image
|
132 |
-
def kandinsky_inpainting_inference(prompt,negative_prompt,image,mask_image):
|
133 |
pipe = fetch_kandinsky_pipeline(controlnet_adapter_model_name, controlnet_base_model_name,kandinsky_model_name, image)
|
134 |
-
output_image = pipe(prompt=prompt,negative_prompt=negative_prompt,image=image,mask_image=mask_image).images[0]
|
135 |
return output_image
|
136 |
|
|
|
|
|
137 |
def kandinsky_controlnet_inpainting_inference(prompt, negative_prompt, image, hint, generator=torch.Generator(device="cuda").manual_seed(43)):
|
138 |
"""
|
139 |
Perform inpainting inference using the Kandinsky ControlNet model.
|
@@ -150,28 +148,15 @@ def kandinsky_controlnet_inpainting_inference(prompt, negative_prompt, image, hi
|
|
150 |
|
151 |
"""
|
152 |
prior_pipe = fetch_kandinsky_prior_pipeline(controlnet_adapter_model_name, controlnet_base_model_name, kandinsky_model_name, image)
|
153 |
-
img_embed = prior_pipe(prompt=prompt, image=image, strength=0
|
154 |
negative_embed = prior_pipe(prompt=negative_prompt, image=image, strength=1, generator=generator)
|
155 |
controlnet_pipe = fetch_kandinsky_img2img_pipeline(controlnet_adapter_model_name, controlnet_base_model_name, kandinsky_model_name, image)
|
156 |
-
image = controlnet_pipe(image=image, strength=0
|
157 |
return image
|
158 |
|
159 |
|
160 |
|
161 |
-
|
162 |
-
l.info("Kandinsky Inpainting Inference")
|
163 |
-
image = load_image('/home/product_diffusion_api/sample_data/example2.jpg')
|
164 |
-
image = image.resize((768, 768))
|
165 |
-
mask_image = load_image('/home/product_diffusion_api/scripts/invert_mask.jpg')
|
166 |
-
mask_image = mask_image.resize((768,768))
|
167 |
-
prompt = "Product in a GYM 8k ultrarealistic "
|
168 |
-
negative_prompt="lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature"
|
169 |
-
output_image = kandinsky_inpainting_inference(prompt,negative_prompt,image,mask_image)
|
170 |
-
output_image=output_image.resize((768,768))
|
171 |
-
depth_estimator = pipeline("depth-estimation")
|
172 |
-
hint = make_hint(output_image, depth_estimator).unsqueeze(0).half().to("cuda")
|
173 |
-
final_output_image = kandinsky_controlnet_inpainting_inference(prompt,negative_prompt,image, hint)
|
174 |
-
|
175 |
|
176 |
|
177 |
|
|
|
5 |
from typing import List
|
6 |
import numpy as np
|
7 |
import torch
|
8 |
+
from PIL import Image,ImageFilter,ImageOps
|
9 |
from mask_generator import invert_mask
|
10 |
from diffusers.utils import load_image
|
11 |
from pipeline import fetch_control_pipeline,fetch_kandinsky_pipeline,fetch_kandinsky_prior_pipeline,fetch_kandinsky_img2img_pipeline
|
12 |
from config import controlnet_adapter_model_name,controlnet_base_model_name,kandinsky_model_name
|
13 |
import cv2
|
14 |
+
|
15 |
from transformers import pipeline
|
16 |
|
17 |
|
|
|
126 |
Returns:
|
127 |
PIL.Image.Image: The output inpainted image.
|
128 |
"""
|
|
|
|
|
|
|
|
|
129 |
pipe = fetch_kandinsky_pipeline(controlnet_adapter_model_name, controlnet_base_model_name,kandinsky_model_name, image)
|
130 |
+
output_image = pipe(prompt=prompt,negative_prompt=negative_prompt,image=image,mask_image=mask_image,num_inference_steps=200,strength=1.0).images[0]
|
131 |
return output_image
|
132 |
|
133 |
+
|
134 |
+
|
135 |
def kandinsky_controlnet_inpainting_inference(prompt, negative_prompt, image, hint, generator=torch.Generator(device="cuda").manual_seed(43)):
|
136 |
"""
|
137 |
Perform inpainting inference using the Kandinsky ControlNet model.
|
|
|
148 |
|
149 |
"""
|
150 |
prior_pipe = fetch_kandinsky_prior_pipeline(controlnet_adapter_model_name, controlnet_base_model_name, kandinsky_model_name, image)
|
151 |
+
img_embed = prior_pipe(prompt=prompt, image=image, strength=1.0, generator=generator)
|
152 |
negative_embed = prior_pipe(prompt=negative_prompt, image=image, strength=1, generator=generator)
|
153 |
controlnet_pipe = fetch_kandinsky_img2img_pipeline(controlnet_adapter_model_name, controlnet_base_model_name, kandinsky_model_name, image)
|
154 |
+
image = controlnet_pipe(image=image, strength=1.0, image_embeds=img_embed.image_embeds, negative_image_embeds=negative_embed.image_embeds, hint=hint, num_inference_steps=200, generator=generator, height=768, width=768).images[0]
|
155 |
return image
|
156 |
|
157 |
|
158 |
|
159 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
160 |
|
161 |
|
162 |
|
scripts/output.jpg
DELETED
Binary file (41.1 kB)
|
|
scripts/run.py
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import argparse
|
2 |
+
import os
|
3 |
+
from mask_generator import generate_mask, invert_mask
|
4 |
+
from models import kandinsky_inpainting_inference, load_image
|
5 |
+
from PIL import Image
|
6 |
+
|
7 |
+
def main(args):
|
8 |
+
# Generate mask
|
9 |
+
mask = generate_mask(args.image_path)
|
10 |
+
mask_image = Image.fromarray(mask)
|
11 |
+
|
12 |
+
# Save original mask
|
13 |
+
original_mask_path = os.path.join(args.mask_dir, f'{args.uid}_original_mask.jpg')
|
14 |
+
mask_image.save(original_mask_path)
|
15 |
+
|
16 |
+
|
17 |
+
# Invert mask
|
18 |
+
mask_image = load_image(original_mask_path)
|
19 |
+
inverted_mask = invert_mask(mask_image)
|
20 |
+
inverted_mask_path = os.path.join(args.mask_dir, f'{args.uid}_inverted_mask.jpg')
|
21 |
+
inverted_mask.save(inverted_mask_path)
|
22 |
+
|
23 |
+
# Load mask and image
|
24 |
+
invert_mask_image = load_image(inverted_mask_path)
|
25 |
+
image = load_image(args.image_path)
|
26 |
+
|
27 |
+
# Perform inpainting
|
28 |
+
output_image = kandinsky_inpainting_inference(args.prompt, args.negative_prompt, image, mask_image)
|
29 |
+
|
30 |
+
# Save output image
|
31 |
+
output_image_path = os.path.join(args.output_dir, f'{args.uid}_output.jpg')
|
32 |
+
output_image.save(output_image_path)
|
33 |
+
|
34 |
+
|
35 |
+
if __name__ == "__main__":
|
36 |
+
parser = argparse.ArgumentParser(description='Perform Kandinsky inpainting on an image.')
|
37 |
+
parser.add_argument('--image_path', type=str, required=True, help='Path to the input image.')
|
38 |
+
parser.add_argument('--prompt', type=str, required=True, help='Prompt for the Kandinsky inpainting.')
|
39 |
+
parser.add_argument('--negative_prompt', type=str, required=True, help='Negative prompt for the Kandinsky inpainting.')
|
40 |
+
parser.add_argument('--output_dir', type=str, required=True, help='Directory to save the output image.')
|
41 |
+
parser.add_argument('--mask_dir', type=str, required=True, help='Directory to save the mask image.')
|
42 |
+
parser.add_argument('--uid', type=str, required=True, help='Unique identifier for the image and mask.')
|
43 |
+
|
44 |
+
args = parser.parse_args()
|
45 |
+
main(args)
|