Update image and video pipelines

Former-commit-id: 79174593572d581d0b2fa473d348ea58ef25fcb5

README.md

@@ -42,8 +42,8 @@ EXPERIMENTATION WITH THE FOLLOWING models
 ![cooker_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/1228718b-5ef7-44a1-81f6-2953ffdc767c)
 ![toaster_output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/06e12aea-cdc2-4ab8-97e0-be77bc49a238)
 ![chair](https://github.com/VikramxD/product_diffusion_api/assets/72499426/65bcd04f-a715-43c3-8928-a9669f8eda85)
-![Generated Image Pipeline Call 1](https://github.com/VikramxD/product_diffusion_api/assets/72499426/dd6af644-1c07-424a-8ba6-0715a5611094)
-![Generated Image Pipeline Call (1)](https://github.com/VikramxD/product_diffusion_api/assets/72499426/b1b8c745-deb4-41ff-a93a-77fa06f55cc3)
+![tent output](https://github.com/VikramxD/product_diffusion_api/assets/72499426/dd6af644-1c07-424a-8ba6-0715a5611094)
+![cycle](https://github.com/VikramxD/product_diffusion_api/assets/72499426/b1b8c745-deb4-41ff-a93a-77fa06f55cc3)
 
 ## Some Improvements 
 - Working on API to deploy this model in batch mode adding loggers from prompt and generated output 

logs/app_debug.log

@@ -1464,3 +1464,7 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
 2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
 2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
 2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
+2024-03-26 19:58:18,622 [INFO] clear_memory - Memory Cleared
+2024-03-26 19:58:18,633 [INFO] models - I2VGenXL pipeline Inference ->
+2024-03-26 20:22:04,466 [INFO] clear_memory - Memory Cleared
+2024-03-26 20:22:04,471 [INFO] models - Stable Video Diffusion Image 2 Video pipeline Inference ->

logs/app_info.log

@@ -1464,3 +1464,7 @@ speed: {'preprocess': 1.9655227661132812, 'inference': 86.20810508728027, 'postp
 2024-03-24 21:44:47,907 [INFO] clear_memory - Memory Cleared
 2024-03-24 21:48:43,724 [INFO] clear_memory - Memory Cleared
 2024-03-24 21:52:50,583 [INFO] clear_memory - Memory Cleared
+2024-03-26 19:58:18,622 [INFO] clear_memory - Memory Cleared
+2024-03-26 19:58:18,633 [INFO] models - I2VGenXL pipeline Inference ->
+2024-03-26 20:22:04,466 [INFO] clear_memory - Memory Cleared
+2024-03-26 20:22:04,471 [INFO] models - Stable Video Diffusion Image 2 Video pipeline Inference ->

scripts/config.py

@@ -8,4 +8,4 @@ mask_dir = '../masks'
 controlnet_adapter_model_name= 'lllyasviel/control_v11p_sd15_inpaint'
 controlnet_base_model_name = "runwayml/stable-diffusion-inpainting"
 kandinsky_model_name = 'kandinsky-community/kandinsky-2-2-decoder-inpaint'
-
+video_model_name = 'stabilityai/stable-video-diffusion-img2vid-xt'

scripts/models.py

@@ -4,13 +4,16 @@ from config import Project_Name
 from clear_memory import clear_memory
 import numpy as np
 import torch
-from PIL import Image
-from mask_generator import invert_mask
-from diffusers.utils import load_image
+from diffusers.utils import load_image,export_to_video
 from pipeline import fetch_kandinsky_pipeline
 from config import controlnet_adapter_model_name,controlnet_base_model_name,kandinsky_model_name
-import cv2 
 from diffusers import StableDiffusionInpaintPipeline, DPMSolverMultistepScheduler
+from video_pipeline import fetch_video_pipeline
+from diffusers.utils import export_to_gif,load_image
+from config import video_model_name
+
+
+
 
 
 
@@ -90,11 +93,33 @@ def sd2_inpainting_inference(prompt, img, mask, repo_id="stabilityai/stable-diff
     return image
 
     
+def image_to_video_pipeline(image, video_model_name, decode_chunk_size, motion_bucket_id, generator=torch.manual_seed(42)):
+    """
+    Converts an image to a video using a specified video model.
+
+    Args:
+        image (Image): The input image to convert to video.
+        video_model_name (str): The name of the video model to use.
+        decode_chunk_size (int): The size of the chunks to decode.
+        motion_bucket_id (str): The ID of the motion bucket.
+        generator (torch.Generator, optional): The random number generator. Defaults to torch.manual_seed(42).
+
+    Returns:
+        list: The frames of the generated video.
+    """
+    clear_memory()
+    l.info("Stable Video Diffusion Image 2 Video pipeline Inference ->")
+    pipe = fetch_video_pipeline(video_model_name)
+    frames = pipe(image=image, decode_chunk_size=decode_chunk_size, motion_bucket_id=motion_bucket_id, generator=generator).frames[0]
+    return frames
+
+
+if __name__ == "__main__":
+    image = load_image("https://github.com/VikramxD/product_diffusion_api/assets/72499426/dd6af644-1c07-424a-8ba6-0715a5611094")
+    frames = image_to_video_pipeline(image, video_model_name,decode_chunk_size=8,motion_bucket_id=180)
+    export_to_video(frames, "output.mp4")
     
     
     
-   
-    
-    
     
 

scripts/video_pipeline.py

@@ -0,0 +1,25 @@
+import torch
+from diffusers import StableVideoDiffusionPipeline
+from diffusers.utils import load_image
+
+
+
+
+
+
+def fetch_video_pipeline(video_model_name):
+    """
+    Fetches the video pipeline for image processing.
+
+    Args:
+        video_model_name (str): The name of the video model.
+
+    Returns:
+        pipe (StableVideoDiffusionPipeline): The video pipeline.
+
+    """
+    pipe = StableVideoDiffusionPipeline.from_pretrained(
+        video_model_name, torch_dtype=torch.float16, variant="fp16"
+    )
+    pipe.to('cuda')
+    return pipe

scripts/yolov8s-seg.pt.REMOVED.git-id

@@ -1 +0,0 @@
-6e924a316b5623dd99eedf5f9988b66ee4f9dfbe