Spaces:
Runtime error
Runtime error
VikramSingh178
commited on
Commit
β’
fa32203
1
Parent(s):
d1a4430
chore: Update .gitignore and add new files for inpainting pipeline
Browse filesFormer-commit-id: 98433cc2ee6b23538fc629e82c96c608a7ace21d [formerly e830430e6009854c8f7ef28751729b821abdb4b8]
Former-commit-id: f17fae1701fed57adad6c5861841805ef068dec4
- .gitignore +2 -1
- configs/inpainting.yaml +13 -0
- outputs/output.jpg +0 -0
- scripts/__pycache__/config.cpython-310.pyc +0 -0
- scripts/__pycache__/utils.cpython-310.pyc +0 -0
- scripts/config.py +1 -1
- scripts/extended_image.png +0 -0
- scripts/mask.png +0 -0
- scripts/pipeline.py +77 -0
- scripts/utils.py +20 -1
- scripts/yolov8l.pt.REMOVED.git-id +1 -0
.gitignore
CHANGED
@@ -4,4 +4,5 @@ variables.tf
|
|
4 |
.terraform
|
5 |
config.env
|
6 |
/scripts/yolov8s*
|
7 |
-
/scripts/*jpg
|
|
|
|
4 |
.terraform
|
5 |
config.env
|
6 |
/scripts/yolov8s*
|
7 |
+
/scripts/*jpg
|
8 |
+
/scripts/outputs
|
configs/inpainting.yaml
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
segmentation_model : 'facebook/sam-vit-large'
|
2 |
+
detection_model : 'yolov8l'
|
3 |
+
model : 'kandinsky-community/kandinsky-2-2-decoder-inpaint'
|
4 |
+
target_width : 1920
|
5 |
+
target_height : 1080
|
6 |
+
prompt : 'product is on the kitchen floor , ultrarealistic lighting , commercial, award , winning photography'
|
7 |
+
negative_prompt : 'low resolution , bad resolution'
|
8 |
+
roi_scale : 0.5
|
9 |
+
strength : 0.7
|
10 |
+
guidance_scale : 7.5
|
11 |
+
num_inference_steps : 600
|
12 |
+
output_path : '../outputs'
|
13 |
+
|
outputs/output.jpg
ADDED
scripts/__pycache__/config.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/config.cpython-310.pyc and b/scripts/__pycache__/config.cpython-310.pyc differ
|
|
scripts/__pycache__/utils.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/utils.cpython-310.pyc and b/scripts/__pycache__/utils.cpython-310.pyc differ
|
|
scripts/config.py
CHANGED
@@ -7,7 +7,7 @@ PROJECT_NAME = "Product Photography"
|
|
7 |
PRODUCTS_10k_DATASET = "VikramSingh178/Products-10k-BLIP-captions"
|
8 |
CAPTIONING_MODEL_NAME = "Salesforce/blip-image-captioning-base"
|
9 |
SEGMENTATION_MODEL_NAME = "facebook/sam-vit-large"
|
10 |
-
DETECTION_MODEL_NAME = "
|
11 |
|
12 |
|
13 |
|
|
|
7 |
PRODUCTS_10k_DATASET = "VikramSingh178/Products-10k-BLIP-captions"
|
8 |
CAPTIONING_MODEL_NAME = "Salesforce/blip-image-captioning-base"
|
9 |
SEGMENTATION_MODEL_NAME = "facebook/sam-vit-large"
|
10 |
+
DETECTION_MODEL_NAME = "yolov8l"
|
11 |
|
12 |
|
13 |
|
scripts/extended_image.png
DELETED
Binary file (86.2 kB)
|
|
scripts/mask.png
DELETED
Binary file (2.82 kB)
|
|
scripts/pipeline.py
ADDED
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
from diffusers import AutoPipelineForInpainting
|
3 |
+
from diffusers.utils import load_image
|
4 |
+
from utils import (accelerator, ImageAugmentation, clear_memory)
|
5 |
+
import hydra
|
6 |
+
from omegaconf import OmegaConf, DictConfig
|
7 |
+
from PIL import Image
|
8 |
+
import lightning.pytorch as pl
|
9 |
+
pl.seed_everything(42)
|
10 |
+
generator = torch.Generator("cuda").manual_seed(92)
|
11 |
+
|
12 |
+
class AutoPaintingPipeline:
|
13 |
+
"""
|
14 |
+
AutoPaintingPipeline class represents a pipeline for auto painting using an inpainting model from diffusers.
|
15 |
+
|
16 |
+
Args:
|
17 |
+
model_name (str): The name of the pretrained inpainting model.
|
18 |
+
image (Image): The input image to be processed.
|
19 |
+
mask_image (Image): The mask image indicating the areas to be inpainted.
|
20 |
+
"""
|
21 |
+
|
22 |
+
def __init__(self, model_name: str, image: Image, mask_image: Image):
|
23 |
+
self.model_name = model_name
|
24 |
+
self.device = accelerator()
|
25 |
+
self.pipeline = AutoPipelineForInpainting.from_pretrained(self.model_name, torch_dtype=torch.float16)
|
26 |
+
self.image = load_image(image)
|
27 |
+
self.mask_image = load_image(mask_image)
|
28 |
+
self.pipeline.to(self.device)
|
29 |
+
self.pipeline.unet = torch.compile(self.pipeline.unet, mode="reduce-overhead", fullgraph=True)
|
30 |
+
|
31 |
+
|
32 |
+
def run_inference(self, prompt: str, negative_prompt: str, num_inference_steps: int, strength: float, guidance_scale: float):
|
33 |
+
"""
|
34 |
+
Runs the inference on the input image using the inpainting pipeline.
|
35 |
+
|
36 |
+
Returns:
|
37 |
+
Image: The output image after inpainting.
|
38 |
+
"""
|
39 |
+
|
40 |
+
image = load_image(self.image)
|
41 |
+
mask_image = load_image(self.mask_image)
|
42 |
+
output = self.pipeline(prompt=prompt,negative_prompt=negative_prompt,image=image,mask_image=mask_image,num_inference_steps=num_inference_steps,strength=strength,guidance_scale =guidance_scale,height = 1472, width = 2560).images[0]
|
43 |
+
clear_memory()
|
44 |
+
return output
|
45 |
+
|
46 |
+
|
47 |
+
@hydra.main(version_base=None ,config_path="../configs", config_name="inpainting")
|
48 |
+
def inference(cfg: DictConfig):
|
49 |
+
"""
|
50 |
+
Load the configuration file for the inpainting pipeline.
|
51 |
+
|
52 |
+
Args:
|
53 |
+
cfg (DictConfig): The configuration file for the inpainting pipeline.
|
54 |
+
"""
|
55 |
+
augmenter = ImageAugmentation(target_width=cfg.target_width, target_height=cfg.target_height, roi_scale=cfg.roi_scale)
|
56 |
+
model_name = cfg.model
|
57 |
+
image_path = "../sample_data/example3.jpg"
|
58 |
+
image = Image.open(image_path)
|
59 |
+
extended_image = augmenter.extend_image(image)
|
60 |
+
mask_image = augmenter.generate_mask_from_bbox(extended_image, cfg.segmentation_model, cfg.detection_model)
|
61 |
+
mask_image = augmenter.invert_mask(mask_image)
|
62 |
+
prompt = cfg.prompt
|
63 |
+
negative_prompt = cfg.negative_prompt
|
64 |
+
num_inference_steps = cfg.num_inference_steps
|
65 |
+
strength = cfg.strength
|
66 |
+
guidance_scale = cfg.guidance_scale
|
67 |
+
pipeline = AutoPaintingPipeline(model_name=model_name, image=extended_image, mask_image=mask_image)
|
68 |
+
output = pipeline.run_inference(prompt=prompt, negative_prompt=negative_prompt, num_inference_steps=num_inference_steps, strength=strength, guidance_scale=guidance_scale)
|
69 |
+
output.save(f'{cfg.output_path}/output.jpg')
|
70 |
+
return output
|
71 |
+
|
72 |
+
if __name__ == "__main__":
|
73 |
+
inference()
|
74 |
+
|
75 |
+
|
76 |
+
|
77 |
+
|
scripts/utils.py
CHANGED
@@ -5,6 +5,25 @@ import numpy as np
|
|
5 |
from PIL import Image, ImageOps
|
6 |
from config import SEGMENTATION_MODEL_NAME, DETECTION_MODEL_NAME
|
7 |
from diffusers.utils import load_image
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
|
10 |
|
@@ -93,7 +112,7 @@ class ImageAugmentation:
|
|
93 |
|
94 |
if __name__ == "__main__":
|
95 |
augmenter = ImageAugmentation(target_width=2560, target_height=1440, roi_scale=0.7)
|
96 |
-
image_path = "
|
97 |
image = Image.open(image_path)
|
98 |
extended_image = augmenter.extend_image(image)
|
99 |
mask = augmenter.generate_mask_from_bbox(extended_image, SEGMENTATION_MODEL_NAME, DETECTION_MODEL_NAME)
|
|
|
5 |
from PIL import Image, ImageOps
|
6 |
from config import SEGMENTATION_MODEL_NAME, DETECTION_MODEL_NAME
|
7 |
from diffusers.utils import load_image
|
8 |
+
import gc
|
9 |
+
|
10 |
+
|
11 |
+
|
12 |
+
|
13 |
+
|
14 |
+
|
15 |
+
|
16 |
+
def clear_memory():
|
17 |
+
"""
|
18 |
+
Clears the memory by collecting garbage and emptying the CUDA cache.
|
19 |
+
|
20 |
+
This function is useful when dealing with memory-intensive operations in Python, especially when using libraries like PyTorch.
|
21 |
+
|
22 |
+
"""
|
23 |
+
gc.collect()
|
24 |
+
torch.cuda.empty_cache()
|
25 |
+
|
26 |
+
|
27 |
|
28 |
|
29 |
|
|
|
112 |
|
113 |
if __name__ == "__main__":
|
114 |
augmenter = ImageAugmentation(target_width=2560, target_height=1440, roi_scale=0.7)
|
115 |
+
image_path = "../sample_data/example3.jpg"
|
116 |
image = Image.open(image_path)
|
117 |
extended_image = augmenter.extend_image(image)
|
118 |
mask = augmenter.generate_mask_from_bbox(extended_image, SEGMENTATION_MODEL_NAME, DETECTION_MODEL_NAME)
|
scripts/yolov8l.pt.REMOVED.git-id
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
4b11a02a7599e520f9f14a4703f4991237d6ce50
|