Spaces:
Runtime error
Runtime error
Commit
•
fe4ead9
1
Parent(s):
d96c8f1
Update image and video pipelines
Browse filesFormer-commit-id: 79174593572d581d0b2fa473d348ea58ef25fcb5
- README.md +2 -2
- logs/app_debug.log +4 -0
- logs/app_info.log +4 -0
- scripts/__pycache__/config.cpython-310.pyc +0 -0
- scripts/__pycache__/pipeline.cpython-310.pyc +0 -0
- scripts/__pycache__/video_pipeline.cpython-310.pyc +0 -0
- scripts/config.py +1 -1
- scripts/extended_image.jpg +0 -0
- scripts/models.py +32 -7
- scripts/video_pipeline.py +25 -0
- scripts/yolov8s-seg.pt.REMOVED.git-id +0 -1
README.md
CHANGED
@@ -42,8 +42,8 @@ EXPERIMENTATION WITH THE FOLLOWING models
|
|
42 |
![cooker_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/1228718b-5ef7-44a1-81f6-2953ffdc767c)
|
43 |
![toaster_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/06e12aea-cdc2-4ab8-97e0-be77bc49a238)
|
44 |
![chair](https://github.com/VikramxD/product_diffusion_api/assets/72499426/65bcd04f-a715-43c3-8928-a9669f8eda85)
|
45 |
-
![
|
46 |
-
![
|
47 |
|
48 |
## Some Improvements
|
49 |
- Working on API to deploy this model in batch mode adding loggers from prompt and generated output
|
|
|
42 |
![cooker_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/1228718b-5ef7-44a1-81f6-2953ffdc767c)
|
43 |
![toaster_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/06e12aea-cdc2-4ab8-97e0-be77bc49a238)
|
44 |
![chair](https://github.com/VikramxD/product_diffusion_api/assets/72499426/65bcd04f-a715-43c3-8928-a9669f8eda85)
|
45 |
+
![tent output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/dd6af644-1c07-424a-8ba6-0715a5611094)
|
46 |
+
![cycle](https://github.com/VikramxD/product_diffusion_api/assets/72499426/b1b8c745-deb4-41ff-a93a-77fa06f55cc3)
|
47 |
|
48 |
## Some Improvements
|
49 |
- Working on API to deploy this model in batch mode adding loggers from prompt and generated output
|
logs/app_debug.log
CHANGED
@@ -1464,3 +1464,7 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
|
|
1464 |
2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
|
1465 |
2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
|
1466 |
2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
|
|
|
|
|
|
|
|
|
|
1464 |
2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
|
1465 |
2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
|
1466 |
2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
|
1467 |
+
2024-03-26 19:58:18,622 [INFO] clear_memory - Memory Cleared
|
1468 |
+
2024-03-26 19:58:18,633 [INFO] models - I2VGenXL pipeline Inference ->
|
1469 |
+
2024-03-26 20:22:04,466 [INFO] clear_memory - Memory Cleared
|
1470 |
+
2024-03-26 20:22:04,471 [INFO] models - Stable Video Diffusion Image 2 Video pipeline Inference ->
|
logs/app_info.log
CHANGED
@@ -1464,3 +1464,7 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
|
|
1464 |
2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
|
1465 |
2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
|
1466 |
2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
|
|
|
|
|
|
|
|
|
|
1464 |
2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
|
1465 |
2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
|
1466 |
2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
|
1467 |
+
2024-03-26 19:58:18,622 [INFO] clear_memory - Memory Cleared
|
1468 |
+
2024-03-26 19:58:18,633 [INFO] models - I2VGenXL pipeline Inference ->
|
1469 |
+
2024-03-26 20:22:04,466 [INFO] clear_memory - Memory Cleared
|
1470 |
+
2024-03-26 20:22:04,471 [INFO] models - Stable Video Diffusion Image 2 Video pipeline Inference ->
|
scripts/__pycache__/config.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/config.cpython-310.pyc and b/scripts/__pycache__/config.cpython-310.pyc differ
|
|
scripts/__pycache__/pipeline.cpython-310.pyc
CHANGED
Binary files a/scripts/__pycache__/pipeline.cpython-310.pyc and b/scripts/__pycache__/pipeline.cpython-310.pyc differ
|
|
scripts/__pycache__/video_pipeline.cpython-310.pyc
ADDED
Binary file (731 Bytes). View file
|
|
scripts/config.py
CHANGED
@@ -8,4 +8,4 @@ mask_dir = '../masks'
|
|
8 |
controlnet_adapter_model_name= 'lllyasviel/control_v11p_sd15_inpaint'
|
9 |
controlnet_base_model_name = "runwayml/stable-diffusion-inpainting"
|
10 |
kandinsky_model_name = 'kandinsky-community/kandinsky-2-2-decoder-inpaint'
|
11 |
-
|
|
|
8 |
controlnet_adapter_model_name= 'lllyasviel/control_v11p_sd15_inpaint'
|
9 |
controlnet_base_model_name = "runwayml/stable-diffusion-inpainting"
|
10 |
kandinsky_model_name = 'kandinsky-community/kandinsky-2-2-decoder-inpaint'
|
11 |
+
video_model_name = 'stabilityai/stable-video-diffusion-img2vid-xt'
|
scripts/extended_image.jpg
DELETED
Binary file (112 kB)
|
|
scripts/models.py
CHANGED
@@ -4,13 +4,16 @@ from config import Project_Name
|
|
4 |
from clear_memory import clear_memory
|
5 |
import numpy as np
|
6 |
import torch
|
7 |
-
from
|
8 |
-
from mask_generator import invert_mask
|
9 |
-
from diffusers.utils import load_image
|
10 |
from pipeline import fetch_kandinsky_pipeline
|
11 |
from config import controlnet_adapter_model_name,controlnet_base_model_name,kandinsky_model_name
|
12 |
-
import cv2
|
13 |
from diffusers import StableDiffusionInpaintPipeline, DPMSolverMultistepScheduler
|
|
|
|
|
|
|
|
|
|
|
|
|
14 |
|
15 |
|
16 |
|
@@ -90,11 +93,33 @@ def sd2_inpainting_inference(prompt, img, mask, repo_id="stabilityai/stable-diff
|
|
90 |
return image
|
91 |
|
92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
93 |
|
94 |
|
95 |
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
|
100 |
|
|
|
4 |
from clear_memory import clear_memory
|
5 |
import numpy as np
|
6 |
import torch
|
7 |
+
from diffusers.utils import load_image,export_to_video
|
|
|
|
|
8 |
from pipeline import fetch_kandinsky_pipeline
|
9 |
from config import controlnet_adapter_model_name,controlnet_base_model_name,kandinsky_model_name
|
|
|
10 |
from diffusers import StableDiffusionInpaintPipeline, DPMSolverMultistepScheduler
|
11 |
+
from video_pipeline import fetch_video_pipeline
|
12 |
+
from diffusers.utils import export_to_gif,load_image
|
13 |
+
from config import video_model_name
|
14 |
+
|
15 |
+
|
16 |
+
|
17 |
|
18 |
|
19 |
|
|
|
93 |
return image
|
94 |
|
95 |
|
96 |
+
def image_to_video_pipeline(image, video_model_name, decode_chunk_size, motion_bucket_id, generator=torch.manual_seed(42)):
|
97 |
+
"""
|
98 |
+
Converts an image to a video using a specified video model.
|
99 |
+
|
100 |
+
Args:
|
101 |
+
image (Image): The input image to convert to video.
|
102 |
+
video_model_name (str): The name of the video model to use.
|
103 |
+
decode_chunk_size (int): The size of the chunks to decode.
|
104 |
+
motion_bucket_id (str): The ID of the motion bucket.
|
105 |
+
generator (torch.Generator, optional): The random number generator. Defaults to torch.manual_seed(42).
|
106 |
+
|
107 |
+
Returns:
|
108 |
+
list: The frames of the generated video.
|
109 |
+
"""
|
110 |
+
clear_memory()
|
111 |
+
l.info("Stable Video Diffusion Image 2 Video pipeline Inference ->")
|
112 |
+
pipe = fetch_video_pipeline(video_model_name)
|
113 |
+
frames = pipe(image=image, decode_chunk_size=decode_chunk_size, motion_bucket_id=motion_bucket_id, generator=generator).frames[0]
|
114 |
+
return frames
|
115 |
+
|
116 |
+
|
117 |
+
if __name__ == "__main__":
|
118 |
+
image = load_image("https://github.com/VikramxD/product_diffusion_api/assets/72499426/dd6af644-1c07-424a-8ba6-0715a5611094")
|
119 |
+
frames = image_to_video_pipeline(image, video_model_name,decode_chunk_size=8,motion_bucket_id=180)
|
120 |
+
export_to_video(frames, "output.mp4")
|
121 |
|
122 |
|
123 |
|
|
|
|
|
|
|
124 |
|
125 |
|
scripts/video_pipeline.py
ADDED
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
from diffusers import StableVideoDiffusionPipeline
|
3 |
+
from diffusers.utils import load_image
|
4 |
+
|
5 |
+
|
6 |
+
|
7 |
+
|
8 |
+
|
9 |
+
|
10 |
+
def fetch_video_pipeline(video_model_name):
|
11 |
+
"""
|
12 |
+
Fetches the video pipeline for image processing.
|
13 |
+
|
14 |
+
Args:
|
15 |
+
video_model_name (str): The name of the video model.
|
16 |
+
|
17 |
+
Returns:
|
18 |
+
pipe (StableVideoDiffusionPipeline): The video pipeline.
|
19 |
+
|
20 |
+
"""
|
21 |
+
pipe = StableVideoDiffusionPipeline.from_pretrained(
|
22 |
+
video_model_name, torch_dtype=torch.float16, variant="fp16"
|
23 |
+
)
|
24 |
+
pipe.to('cuda')
|
25 |
+
return pipe
|
scripts/yolov8s-seg.pt.REMOVED.git-id
DELETED
@@ -1 +0,0 @@
|
|
1 |
-
6e924a316b5623dd99eedf5f9988b66ee4f9dfbe
|
|
|
|