feat: add API endpoint for clothing generation without mask; implement functionality within pipeline

app.py

@@ -19,7 +19,7 @@ from pydantic import BaseModel
 import replicate
 
 pipeline = Pipeline()
-app = FastAPI(title="Magical Mirror Web Plugin")
+app = FastAPI(title="Magical Mirror Web Plugin", description="API for Magical Mirror Application ", version="1.0")
 
 app.add_middleware(
     CORSMiddleware,
@@ -33,9 +33,56 @@ key: str = os.getenv("SUPABASE_KEY")
 supabase: Client = create_client(url, key)
 
 
-class ClothingTryOnEntity(BaseModel):
-    image: str
-    mask: str
+class NecklaceTryOnIDEntity(BaseModel):
+    necklaceImageId: str
+    necklaceCategory: str
+    storename: str
+    api_token: str
+
+
+@app.post("/clothingTryOnV2")
+async def clothing_try_on_v2(image: UploadFile = File(...), clothing_type: str = Form(...)):
+    image_bytes = await image.read()
+    image = Image.open(BytesIO(image_bytes)).convert("RGB")
+
+    mask = await pipeline.shoulderPointMaskGeneration_(image=image)
+
+    mask_img_base_64, act_img_base_64 = BytesIO(), BytesIO()
+    mask.save(mask_img_base_64, format="WEBP")
+    image.save(act_img_base_64, format="WEBP")
+    mask_bytes_ = base64.b64encode(mask_img_base_64.getvalue()).decode("utf-8")
+    image_bytes_ = base64.b64encode(act_img_base_64.getvalue()).decode("utf-8")
+
+    mask_data_uri = f"data:image/webp;base64,{mask_bytes_}"
+    image_data_uri = f"data:image/webp;base64,{image_bytes_}"
+
+    input = {
+        "mask": mask_data_uri,
+        "image": image_data_uri,
+        "prompt": f"Dull {clothing_type}, non-reflective clothing, properly worn, natural setting, elegant, natural look, neckline without jewellery, simple, perfect eyes, perfect face, perfect body, high quality, realistic, photorealistic, high resolution,traditional full sleeve blouse",
+        "negative_prompt": "necklaces, jewellery, jewelry, necklace, neckpiece, garland, chain, neck wear, jewelled neck, jeweled neck, necklace on neck, jewellery on neck, accessories, watermark, text, changed background, wider body, narrower body, bad proportions, extra limbs, mutated hands, changed sizes, altered proportions, unnatural body proportions, blury, ugly",
+        "num_inference_steps": 25
+    }
+
+    output = replicate.run(
+        "stability-ai/stable-diffusion-inpainting:95b7223104132402a9ae91cc677285bc5eb997834bd2349fa486f53910fd68b3",
+        input=input
+    )
+    image_url = output[0]
+
+    response = requests.get(image_url)
+    result = Image.open(BytesIO(response.content)).resize(image.size)
+
+    in_mem_file = BytesIO()
+    result.save(in_mem_file, format="WEBP", quality=85)
+    base_64_output = base64.b64encode(in_mem_file.getvalue()).decode('utf-8')
+
+    response = {
+        "output": f"data:image/WEBP;base64,{base_64_output}",
+        'code': 200
+    }
+
+    return JSONResponse(content=response, status_code=200)
 
 
 @app.post("/clothingTryOn")
@@ -237,13 +284,6 @@ async def product_data(
         raise HTTPException(status_code=500, detail=f"Failed to fetch or process data: {e}")
 
 
-class NecklaceTryOnIDEntity(BaseModel):
-    necklaceImageId: str
-    necklaceCategory: str
-    storename: str
-    api_token: str
-
-
 async def parse_necklace_try_on_id(necklaceImageId: str = Form(...),
                                    necklaceCategory: str = Form(...),
                                    storename: str = Form(...),
@@ -281,8 +321,8 @@ async def necklace_try_on_id(necklace_try_on_id: NecklaceTryOnIDEntity = Depends
 
             return JSONResponse(content=error_message, status_code=404)
 
-        result, headetText, mask = await pipeline.necklaceTryOn(image=image, jewellery=jewellery,
-                                                                storename=necklace_try_on_id.storename)
+        result, headetText, mask = await pipeline.necklaceTryOn_(image=image, jewellery=jewellery,
+                                                                 storename=necklace_try_on_id.storename)
 
         inMemFile = BytesIO()
         inMemFileMask = BytesIO()

logs/running_logs.log

@@ -0,0 +1,30 @@
+[2024-10-10 20:52:18,370: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 20:52:18,695: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 20:52:18,695: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 20:56:13,263: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 20:56:13,569: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 20:56:13,569: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 20:56:13,569: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 20:57:20,291: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 20:57:20,597: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 20:57:20,597: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 20:57:20,597: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:01:04,285: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 21:01:04,583: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 21:01:04,583: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:01:04,583: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:02:43,963: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 21:02:44,284: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 21:02:44,284: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:02:44,284: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:07:16,992: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 21:07:17,246: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 21:07:17,246: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:08:15,909: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 21:08:16,206: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 21:08:16,206: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:09:24,365: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: detecting pose and landmarks]
+[2024-10-10 21:09:24,665: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: estimating neck points]
+[2024-10-10 21:09:24,665: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:09:24,665: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: generating shoulder point mask]
+[2024-10-10 21:09:24,671: INFO: necklaceTryOn: SHOULDER POINT MASK GENERATION :: mask generated successfully]

src/components/necklaceTryOn.py

@@ -1,3 +1,5 @@
+import time
+
 from cvzone.FaceMeshModule import FaceMeshDetector
 from src.utils import addWatermark, returnBytesData
 from src.utils.exceptions import CustomException
@@ -251,3 +253,50 @@ class NecklaceTryOn:
         except Exception as e:
             logger.error(f"{CustomException(e)}:: {storename}")
             raise CustomException(e)
+
+    def shoulderPointMaskGeneration(self, image: Image.Image) -> Image.Image:
+        image = np.array(image)
+        copy_image = image.copy()
+
+        logger.info(f"SHOULDER POINT MASK GENERATION :: detecting pose and landmarks")
+
+        image = self.detector.findPose(image)
+        lmList, _ = self.detector.findPosition(image, bboxWithHands=False, draw=False)
+
+        img, faces = self.meshDetector.findFaceMesh(image, draw=False)
+        leftLandmarkIndex = 172
+        rightLandmarkIndex = 397
+
+        leftLandmark, rightLandmark = faces[0][leftLandmarkIndex], faces[0][rightLandmarkIndex]
+        landmarksDistance = int(
+            ((leftLandmark[0] - rightLandmark[0]) ** 2 + (leftLandmark[1] - rightLandmark[1]) ** 2) ** 0.5)
+
+        logger.info(f"SHOULDER POINT MASK GENERATION :: estimating neck points")
+
+        avg_x1 = int(leftLandmark[0] - landmarksDistance * 0.12)
+        avg_x2 = int(rightLandmark[0] + landmarksDistance * 0.12)
+
+        avg_y1 = int(leftLandmark[1] + landmarksDistance * 0.5)
+        avg_y2 = int(rightLandmark[1] + landmarksDistance * 0.5)
+
+        offset = 50
+        avg_y1 -= offset
+        avg_y2 -= offset
+
+        logger.info(f"SHOULDER POINT MASK GENERATION :: generating shoulder point mask")
+
+        logger.info("SHOULDER POINT MASK GENERATION :: generating shoulder point mask")
+
+        mask = np.zeros_like(image[:, :, 0])
+
+        mask[avg_y1:, :] = 255
+
+        pts = np.array([[0, 0], [image.shape[1], 0], [avg_x2, avg_y2], [avg_x1, avg_y1]], np.int32)
+        pts = pts.reshape((-1, 1, 2))
+        cv2.fillPoly(mask, [pts], 0)
+
+        black_n_white_mask = np.zeros_like(image[:, :, 0])
+        black_n_white_mask[avg_y1:, :] = 255
+        logger.info("SHOULDER POINT MASK GENERATION :: mask generated successfully")
+
+        return Image.fromarray(black_n_white_mask.astype(np.uint8))

src/pipelines/completePipeline.py

@@ -7,8 +7,12 @@ class Pipeline:
     def __init__(self) -> None:
         self.necklaceTryOnObj = NecklaceTryOn()
 
-    async def necklaceTryOn(self, image: Image.Image, jewellery: Image.Image, storename: str) -> list[
+    async def necklaceTryOn_(self, image: Image.Image, jewellery: Image.Image, storename: str) -> list[
         Union[Image.Image, str]]:
         result, headerText, mask = self.necklaceTryOnObj.necklaceTryOn(image=image, jewellery=jewellery,
                                                                        storename=storename)
         return [result, headerText, mask]
+
+    async def shoulderPointMaskGeneration_(self, image: Image.Image) -> Image.Image:
+        mask = self.necklaceTryOnObj.shoulderPointMaskGeneration(image=image)
+        return mask