app.py

import spaces
import gradio as gr
import subprocess
from PIL import Image,ImageOps,ImageDraw,ImageFilter
import json
import os
import time

from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate
from glibvision.draw_utils import points_to_box,box_to_xy,plus_point
import mp_constants
import mp_box
import io
import numpy as np
from glibvision.pil_utils import fill_points,create_color_image,draw_points,draw_box

from gradio_utils import save_image,save_buffer,clear_old_files ,read_file
import opencvinpaint

'''
Face landmark detection based Face Detection.
https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker
from model card
https://storage.googleapis.com/mediapipe-assets/MediaPipe%20BlazeFace%20Model%20Card%20(Short%20Range).pdf
Licensed Apache License, Version 2.0
Train with google's dataset(more detail see model card)

'''

def picker_color_to_rgba(picker_color):
   
    color_value = picker_color.strip("rgba()").split(",")
    color_value[0] = int(float(color_value[0]))
    color_value[1] = int(float(color_value[1]))
    color_value[2] = int(float(color_value[2]))
    color_value[3] = int(float(color_value[3])*255)
    print(f"picker_color = {picker_color} color_value={color_value}")
    return color_value

#@spaces.GPU(duration=120)
'''

    innner_eyes_blur - inner eyes blur
    iris_mask_blur - final iris edge blur
'''


def process_images(image,eyes_slide_x_ratio,eyes_slide_y_ratio,innner_eyes_blur_ratio=0.1,iris_mask_blur_ratio=0.1,pupil_offset_ratio=0.08,draw_eye_pupil_ratio=1.1,iris_color_value="rgba(20,20,20,255)",eyes_white_erode_ratio=0.2,eyes_ball_mask_ratio=0.9,
                   output_important_only=True,progress=gr.Progress(track_tqdm=True)):
    clear_old_files()
    if image == None:
        raise gr.Error("Need Image")
    
    

    iris_color =  tuple(picker_color_to_rgba(iris_color_value))
    #print(iris_color)
    #return None,None
    
    


    # TODO resize max 2048

    white_image = create_color_image(image.width,image.height,(255,255,255))
    ## Mediapipe landmark
    progress(0, desc="Start Making Animation")
    mp_image,face_landmarker_result = extract_landmark(image)
    larndmarks=face_landmarker_result.face_landmarks


    ## eyes cordinates
    left_iris_points = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_LEFT_IRIS,image.width,image.height)
    right_iris_points = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_RIGHT_IRIS,image.width,image.height)

    left_box = points_to_box(left_iris_points)
    right_box = points_to_box(right_iris_points)
    left_eye_radius = (left_box[2] if left_box[2]>left_box[3] else left_box[3])/2
    right_eye_radius = (right_box[2] if right_box[2]>right_box[3] else right_box[3])/2


    innner_eyes_blur = int(left_eye_radius*innner_eyes_blur_ratio)
    iris_mask_blur = int(left_eye_radius*iris_mask_blur_ratio)
    

    eyes_slide_x = int(left_eye_radius*2 * eyes_slide_x_ratio)
    eyes_slide_y = int(left_eye_radius*2 * eyes_slide_y_ratio)
    
    
    pupil_offset_y = right_eye_radius * pupil_offset_ratio

    point_right_pupil = get_pixel_cordinate(larndmarks,mp_constants.POINT_RIGHT_PUPIL,image.width,image.height)
    point_right_pupil = plus_point(point_right_pupil,[0,pupil_offset_y])

    point_left_pupil = get_pixel_cordinate(larndmarks,mp_constants.POINT_LEFT_PUPIL,image.width,image.height)
    point_left_pupil = plus_point(point_left_pupil,[0,pupil_offset_y])



    left_inner_eyes = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_RIGHT_UPPER_INNER_EYE+mp_constants.LINE_RIGHT_LOWER_INNER_EYE,image.width,image.height)
    right_inner_eyes = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_LEFT_UPPER_INNER_EYE+mp_constants.LINE_LEFT_LOWER_INNER_EYE ,image.width,image.height)
    
    left_white_eyes = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_LEFT_EYES_WHITE,image.width,image.height)
    fill_points(white_image,left_white_eyes,(0,0,0))

    right_white_eyes = get_pixel_cordinate_list(larndmarks,mp_constants.LINE_RIGHT_EYES_WHITE ,image.width,image.height)
    fill_points(white_image,right_white_eyes,(0,0,0))

    left_eyes_box = points_to_box(left_white_eyes)
    right_eyes_box = points_to_box(right_white_eyes)

    black_image = create_color_image(image.width,image.height,(0,0,0))
    draw_box(black_image,left_eyes_box,fill=(255,255,255))
    draw_box(black_image,right_eyes_box,fill=(255,255,255))


    eyes_mask_area_image = black_image.convert("L")
    eyes_mask_image = white_image.convert("L") #eyes-white-area-hole painted black

    galleries = []
    
    progressed = 0

    def add_webp(add_image,label,important=False):
        nonlocal progressed
        progressed += .038
        progress(progressed)
        if important ==False and output_important_only == True:
            return
        
        file_path = save_image(add_image,"webp")
        galleries.append((file_path,label))
    
    # Create EYE LINE IMAGE
    eyes_line_image = image.copy()
    draw_points(eyes_line_image,left_inner_eyes,outline=(200,200,255),fill=None,width=3)
    draw_points(eyes_line_image,right_inner_eyes,outline=(200,200,255),fill=None,width=3)

    draw_points(eyes_line_image,left_white_eyes,outline=(255,0,0),fill=None,width=4)
    draw_points(eyes_line_image,right_white_eyes,outline=(255,0,0),fill=None,width=4)
    draw_points(eyes_line_image,left_iris_points,outline=(0,255,0),fill=None,width=4)
    draw_points(eyes_line_image,right_iris_points,outline=(0,255,0),fill=None,width=4)
    add_webp(eyes_line_image,"eyes-line",True)
    

    # eyes socket(face) image
    rgba_image = image.convert("RGBA")
    rgba_image.putalpha(eyes_mask_image)
    eyes_socket = rgba_image
    add_webp(eyes_socket,"eyes-socket",True)
    eyes_socket_mask = eyes_mask_image

    # Save Eyes mask and area
    eyes_white_mask = ImageOps.invert(eyes_mask_image)
    add_webp(eyes_white_mask,"eyes-mask")
    add_webp(eyes_mask_area_image,"eyes-box")


    # Remove Edge,
    
    erode_size = int(left_box[3]*eyes_white_erode_ratio) # eyes-height base #TODO take care right eyes
    if erode_size%2==0:
        erode_size+=1
    eyes_white_mask=eyes_white_mask.filter(ImageFilter.MinFilter(erode_size))


    # eyes_only_image inner-white-eyes - erode
    rgba_image = image.convert("RGBA")
    rgba_image.putalpha(eyes_white_mask)
    eyes_only_image = rgba_image
    add_webp(eyes_only_image,"eyes-only")
    eyes_only_image_mask = eyes_white_mask.copy()

    

    draw = ImageDraw.Draw(eyes_white_mask)
    draw.circle(point_right_pupil,left_eye_radius*draw_eye_pupil_ratio,fill=(0))
    draw.circle(point_left_pupil,left_eye_radius*draw_eye_pupil_ratio,fill=(0))

    rgba_image = image.convert("RGBA")
    rgba_image.putalpha(eyes_white_mask)
    add_webp(rgba_image,"white-inapint-image",True)

    eyes_mask_area_image.paste(ImageOps.invert(eyes_white_mask),None,mask=eyes_white_mask)
    add_webp(eyes_mask_area_image,"white-inapint-mask")


    cropped_right_eye = rgba_image.crop(box_to_xy(right_eyes_box))
    add_webp(cropped_right_eye,"right-eye")
    cropped_right_eye_mask = eyes_mask_area_image.crop(box_to_xy(right_eyes_box))
    add_webp(cropped_right_eye_mask,"right-eye-mask")


    cropped_left_eye = rgba_image.crop(box_to_xy(left_eyes_box))
    add_webp(cropped_left_eye,"left-eye")
    cropped_left_eye_mask = eyes_mask_area_image.crop(box_to_xy(left_eyes_box))
    add_webp(cropped_left_eye_mask,"left-eye-mask")


    inpaint_radius  = 20
    blur_radius = 15
    edge_expand = 4

    inpaint_mode = "Telea"
    inner_eyes_image = create_color_image(image.width,image.height,color=(0,0,0,0))
    inpaint_right,tmp_mask=opencvinpaint.process_cvinpaint(cropped_right_eye,cropped_right_eye_mask.convert("RGB"),inpaint_radius,blur_radius,edge_expand,inpaint_mode)
    add_webp(inpaint_right,"right-eye")
    inpaint_left,tmp_mask=opencvinpaint.process_cvinpaint(cropped_left_eye,cropped_left_eye_mask.convert("RGB"),inpaint_radius,blur_radius,edge_expand,inpaint_mode)
    add_webp(inpaint_left,"left-eye")
    
    inner_eyes_image.paste(inpaint_right,box_to_xy(right_eyes_box))
    inner_eyes_image.paste(inpaint_left,box_to_xy(left_eyes_box))
    add_webp(inner_eyes_image,"inpainted-eyes",True)
    eyes_blank = inner_eyes_image.copy()
    eyes_blank.paste(eyes_socket,eyes_socket_mask)
    add_webp(eyes_blank,"eyes_blank",True)
    
    eyes_move_pt = (eyes_slide_x,eyes_slide_y)
    draw_pupil_border=2
    
    draw_left_pupil_radius = int(left_eye_radius*draw_eye_pupil_ratio)
    draw_right_pupil_radius = int(right_eye_radius*draw_eye_pupil_ratio)

    eyes_ball_image = eyes_only_image.convert("RGBA")#create_color_image(image.width,image.height,color=(0,0,0,0))
    draw = ImageDraw.Draw(eyes_ball_image)
   
    
    draw.circle(point_right_pupil,draw_right_pupil_radius,outline=iris_color,width=draw_pupil_border)
    draw.circle(point_left_pupil,draw_left_pupil_radius,outline=iris_color,width=draw_pupil_border)
    add_webp(eyes_ball_image,"eyes-ball-inpaint-base",True)

    
    #draw mask too
    

    eyes_ball_image_mask = create_color_image(image.width,image.height,color=(0,0,0))
    draw = ImageDraw.Draw(eyes_ball_image_mask)
    draw.circle(point_right_pupil,draw_right_pupil_radius-draw_pupil_border,fill=(255,255,255))
    draw.circle(point_left_pupil,draw_left_pupil_radius-draw_pupil_border,fill=(255,255,255))
    add_webp(eyes_ball_image_mask,"eyes-ball-image-mask")
    eyes_ball_image_mask = eyes_ball_image_mask.convert("L")

    eyes_ball_image_inpaint_mask = eyes_ball_image_mask.copy()
    eyes_ball_image_inpaint_mask.paste(ImageOps.invert(eyes_only_image_mask),mask=eyes_only_image_mask)
    add_webp(eyes_ball_image_inpaint_mask,"eyes_ball_image_inpaint_mask")

    ### create inpaint and replace
    pupil_inpaint_radius  = 5
    pupil_blur_radius = 2
    pupil_edge_expand = 5

    inpaint_eyes_ball_image,tmp_mask=opencvinpaint.process_cvinpaint(eyes_ball_image,eyes_ball_image_inpaint_mask.convert("RGB"),pupil_inpaint_radius,pupil_blur_radius,pupil_edge_expand,inpaint_mode)
    inpaint_eyes_ball_image.putalpha(eyes_ball_image_mask)
    add_webp(inpaint_eyes_ball_image,"inpaint_eyes_ball_image")
    eyes_ball_image = inpaint_eyes_ball_image


    eyes_and_ball_mask = eyes_only_image_mask.copy()
    eyes_and_ball_mask.paste(eyes_ball_image_mask,mask=eyes_ball_image_mask)
    add_webp(eyes_and_ball_mask,"eyes-ball-mask")

    eyes_ball_image.paste(eyes_only_image,mask=eyes_only_image_mask)
    add_webp(eyes_ball_image,"eyes-ball",True)



    inner_eyes_image.paste(eyes_ball_image,eyes_move_pt,mask=eyes_and_ball_mask)
    add_webp(inner_eyes_image,"inner-eyes")

    inner_eyes_image.paste(eyes_socket,None,mask=eyes_socket_mask)

    #ImageFilter.BLUR,"Smooth More":ImageFilter.SMOOTH_MORE,"Smooth":ImageFilter.SMOOTH
    filtered_image = inner_eyes_image.filter(ImageFilter.GaussianBlur(radius=innner_eyes_blur))
    add_webp(filtered_image,"bluerd_inner_face",True)
    

    #filtered_image.paste(eyes_only_image,eyes_move_pt,mask=eyes_ball_image_mask.convert("L"))
    


    ### create innner mask minus eyeballs
    white_image = create_color_image(image.width,image.height,color=(255,255,255))
    draw = ImageDraw.Draw(white_image)
    right_eyes_xy = get_pixel_cordinate(larndmarks,mp_constants.POINT_RIGHT_PUPIL,image.width,image.height)
    left_eyes_xy = get_pixel_cordinate(larndmarks,mp_constants.POINT_LEFT_PUPIL,image.width,image.height)
    draw.circle(plus_point(left_eyes_xy,eyes_move_pt),left_eye_radius*eyes_ball_mask_ratio,fill=(0,0,0,255))
    draw.circle(plus_point(right_eyes_xy,eyes_move_pt),right_eye_radius*eyes_ball_mask_ratio,fill=(0,0,0,255))
    add_webp(white_image,"eyes_ball_mask")

    eyes_socket_mask_invert = ImageOps.invert(eyes_socket_mask)
    eyes_socket_mask_invert.paste(white_image,eyes_socket_mask_invert)
    add_webp(eyes_socket_mask_invert,"inner_mask_without_eyesball")


    ### final paste eyes-ball and outer-faces on blured inner
    eyes_socket_mask_invert = eyes_socket_mask_invert.filter(ImageFilter.GaussianBlur(radius=iris_mask_blur))
    add_webp(eyes_socket_mask_invert,"inner_mask_without_eyesball-blur",True)

    filtered_image.paste(inner_eyes_image,None,mask=ImageOps.invert(eyes_socket_mask_invert))
    filtered_image.paste(eyes_socket,None,mask=eyes_socket_mask)
    file_path = save_image(filtered_image,"webp")

    return filtered_image,galleries
    



css="""
#col-left {
    margin: 0 auto;
    max-width: 640px;
}
#col-right {
    margin: 0 auto;
    max-width: 640px;
}
.grid-container {
  display: flex;
  align-items: center;
  justify-content: center;
  gap:10px
}

.image {
  width: 128px; 
  height: 128px; 
  object-fit: cover; 
}

.text {
  font-size: 16px;
}
"""

#css=css,



with gr.Blocks(css=css, elem_id="demo-container") as demo:
    with gr.Column():
        gr.HTML(read_file("demo_header.html"))
        gr.HTML(read_file("demo_tools.html"))
    with gr.Row():
                with gr.Column():
                    image = gr.Image(height=800,sources=['upload','clipboard'],image_mode='RGB',elem_id="image_upload", type="pil", label="Image")
                    with gr.Row(elem_id="prompt-container",  equal_height=False):
                        with gr.Row():
                            btn = gr.Button("Slide Eyes Direction", elem_id="run_button",variant="primary")
                    
                    with gr.Accordion(label="Eyes Slide", open=True):
                        with gr.Row(equal_height=False):
                            eyes_slide_x_ratio = gr.Slider(
                                    label="Horizontal Slide (Iris based size)",
                                    minimum=-2,
                                    maximum=2,
                                    step=0.01,
                                    value=0,info="Based iris size minus to left,plus to right")
                            eyes_slide_y_ratio = gr.Slider(
                                    label="Vertical Slide (Iris based size)",
                                    minimum=-1.5,
                                    maximum=1.5,
                                    step=0.01,
                                    value=0.25,info="Based iris size minus to up,plus to down")
                        
                    with gr.Accordion(label="Advanced Settings", open=False):
                        
                        with gr.Row( equal_height=True):
                            innner_eyes_blur_ratio = gr.Slider(
                                label="Inner Eyes Blur Ratio",
                                minimum=0,
                                maximum=1,
                                step=0.01,
                                value=0.2,info="increse valueinnser eyes make flat")
                            iris_mask_blur_ratio = gr.Slider(
                                label="Iris Mask Blur Ratio",
                                minimum=0,
                                maximum=1,
                                step=0.01,
                                value=0.15,info="mask edge smooth")
                        with gr.Row( equal_height=True):
                            pupil_offset_ratio = gr.Slider(
                                label="Pupil center Offset Y",
                                minimum=-0.5,
                                maximum=0.5,
                                step=0.01,
                                value=-0.08,info="mediapipe detection is not middle")
                            draw_eye_pupil_ratio = gr.Slider(
                                label="Draw Pupil radius ratio",
                                minimum=0.5,
                                maximum=1.5,
                                step=0.01,
                                value=1.1,info="mediapipe detection is usually small")
                            iris_color_value = gr.ColorPicker(value="rgba(20,20,20,1)",label="Iris Border Color")
                        with gr.Row( equal_height=True):
                            eyes_white_erode_ratio = gr.Slider(
                                label="Eye Erode erode ratio",
                                minimum=0,
                                maximum=0.5,
                                step=0.01,
                                value=0.1,info="eyes edge is pink")
                            eyes_ball_mask_ratio = gr.Slider(
                                label="Eye Ball Mask ratio",
                                minimum=0,
                                maximum=1,
                                step=0.01,
                                value=0.9,info="iris blur and mask for img2img")
                        with gr.Row( equal_height=True):
                            output_important_only=gr.Checkbox(label="output important image only",value=True)        

                                         
                with gr.Column():
                    animation_out = gr.Image(height=760,label="Result", elem_id="output-animation")
                    image_out = gr.Gallery(label="Output", elem_id="output-img",preview=True)
                    

    btn.click(fn=process_images, inputs=[image,eyes_slide_x_ratio,eyes_slide_y_ratio,innner_eyes_blur_ratio,iris_mask_blur_ratio,
                                         pupil_offset_ratio,draw_eye_pupil_ratio,iris_color_value,eyes_white_erode_ratio,eyes_ball_mask_ratio,output_important_only
                                         ],outputs=[animation_out,image_out] ,api_name='infer')
    gr.Examples(
                examples =[
                     ["examples/02316230.jpg"],
                    ["examples/00003245_00.jpg"],
                   ["examples/00827009.jpg"],
                   
                     ["examples/00002062.jpg"],
                   

                    ["examples/00824008.jpg"],
                    ["examples/00825000.jpg"],
                    ["examples/00826007.jpg"],
                     ["examples/00824006.jpg"],
                    ["examples/00828003.jpg"],

                     ["examples/00002200.jpg"],
                    ["examples/00005259.jpg"],
                    ["examples/00018022.jpg"],
                    ["examples/img-above.jpg"],
                     ["examples/00100265.jpg"],
                ],
                #examples =["examples/00003245_00.jpg","examples/00002062.jpg","examples/00100265.jpg","examples/00824006.jpg","examples/00824008.jpg",
                #           "examples/00825000.jpg","examples/00826007.jpg","examples/00827009.jpg","examples/00828003.jpg",],
                inputs=[image],examples_per_page=5
    )
    gr.HTML(read_file("demo_footer.html"))

    if __name__ == "__main__":
        demo.launch()