|
import colorsys |
|
import math |
|
from enum import IntEnum |
|
|
|
import cv2 |
|
import numpy as np |
|
import numpy.linalg as npla |
|
|
|
from face_detect.core import imagelib |
|
from face_detect.core import mathlib |
|
from face_detect.core.mathlib.umeyama import umeyama |
|
from face_detect.FaceType import FaceType |
|
|
|
mesh_33=[70,63,105,66,107,336,296,334,293,300,168,197,5,4,240,99,2,328,460,33,160,158,133,153,144,362,385,387,263,373,380,57,287] |
|
landmarks_2D_4=np.array([ |
|
[0.224152 , 0.2119465], |
|
[0.75610125, 0.2119465], |
|
[0.490127, 0.515625], |
|
[0.4901265, 0.780233] |
|
]) |
|
landmarks_2D_4_bottom=np.array([ |
|
[0.2218305, 0.244588 ], |
|
[0.7584225, 0.244588], |
|
[0.490127, 0.515625], |
|
[0.4901265, 0.780233] |
|
]) |
|
landmarks_2D = np.array([ |
|
[0.000213256, 0.106454], |
|
[0.0752622, 0.038915], |
|
[0.18113, 0.0187482], |
|
[0.29077, 0.0344891], |
|
[0.393397, 0.0773906], |
|
[0.586856, 0.0773906], |
|
[0.689483, 0.0344891], |
|
[0.799124, 0.0187482], |
|
[0.904991, 0.038915], |
|
[0.98004, 0.106454], |
|
[0.490127, 0.203352], |
|
[0.490127, 0.307009], |
|
[0.490127, 0.409805], |
|
[0.490127, 0.515625], |
|
[0.36688, 0.587326], |
|
[0.426036, 0.609345], |
|
[0.490127, 0.628106], |
|
[0.554217, 0.609345], |
|
[0.613373, 0.587326], |
|
[0.121737, 0.216423], |
|
[0.187122, 0.178758], |
|
[0.265825, 0.179852], |
|
[0.334606, 0.231733], |
|
[0.260918, 0.245099], |
|
[0.182743, 0.244077], |
|
[0.645647, 0.231733], |
|
[0.714428, 0.179852], |
|
[0.793132, 0.178758], |
|
[0.858516, 0.216423], |
|
[0.79751, 0.244077], |
|
[0.719335, 0.245099], |
|
[0.254149, 0.780233], |
|
[0.340985, 0.745405], |
|
[0.428858, 0.727388], |
|
[0.490127, 0.742578], |
|
[0.551395, 0.727388], |
|
[0.639268, 0.745405], |
|
[0.726104, 0.780233], |
|
[0.642159, 0.864805], |
|
[0.556721, 0.902192], |
|
[0.490127, 0.909281], |
|
[0.423532, 0.902192], |
|
[0.338094, 0.864805], |
|
[0.290379, 0.784792], |
|
[0.428096, 0.778746], |
|
[0.490127, 0.785343], |
|
[0.552157, 0.778746], |
|
[0.689874, 0.784792], |
|
[0.553364, 0.824182], |
|
[0.490127, 0.831803], |
|
[0.42689, 0.824182] |
|
], dtype=np.float32) |
|
|
|
landmarks_2D_new = np.array([ |
|
[0.000213256, 0.106454], |
|
[0.0752622, 0.038915], |
|
[0.18113, 0.0187482], |
|
[0.29077, 0.0344891], |
|
[0.393397, 0.0773906], |
|
[0.586856, 0.0773906], |
|
[0.689483, 0.0344891], |
|
[0.799124, 0.0187482], |
|
[0.904991, 0.038915], |
|
[0.98004, 0.106454], |
|
[0.490127, 0.203352], |
|
[0.490127, 0.307009], |
|
[0.490127, 0.409805], |
|
[0.490127, 0.515625], |
|
[0.36688, 0.587326], |
|
[0.426036, 0.609345], |
|
[0.490127, 0.628106], |
|
[0.554217, 0.609345], |
|
[0.613373, 0.587326], |
|
[0.121737, 0.216423], |
|
[0.187122, 0.178758], |
|
[0.265825, 0.179852], |
|
[0.334606, 0.231733], |
|
[0.260918, 0.245099], |
|
[0.182743, 0.244077], |
|
[0.645647, 0.231733], |
|
[0.714428, 0.179852], |
|
[0.793132, 0.178758], |
|
[0.858516, 0.216423], |
|
[0.79751, 0.244077], |
|
[0.719335, 0.245099], |
|
[0.254149, 0.780233], |
|
[0.726104, 0.780233], |
|
], dtype=np.float32) |
|
landmarks_2D_new_mesh = np.array([ |
|
[ 0.000213256, 0.106454 ], |
|
[ 0.0752622, 0.038915 ], |
|
[0.1281961, 0.0288316], |
|
[ 0.29077, 0.0144891 ], |
|
[ 0.393397, 0.0773906 ], |
|
[ 0.586856, 0.0773906 ], |
|
[ 0.689483, 0.0144891 ], |
|
[0.8520575, 0.0288316], |
|
[ 0.904991, 0.038915 ], |
|
[ 0.98004, 0.106454 ], |
|
[ 0.490127, 0.203352 ], |
|
[ 0.490127, 0.307009 ], |
|
[ 0.490127, 0.409805 ], |
|
[ 0.490127, 0.515625 ], |
|
[0.396458 , 0.5983355], |
|
[ 0.426036, 0.609345 ], |
|
[ 0.490127, 0.628106 ], |
|
[ 0.554217, 0.609345 ], |
|
[ 0.613373, 0.587326 ], |
|
[ 0.071737, 0.136423 ], |
|
[ 0.137122, 0.118758 ], |
|
[ 0.215825, 0.119852 ], |
|
[ 0.334606, 0.151733 ], |
|
[ 0.210918, 0.165099 ], |
|
[ 0.132743, 0.164077 ], |
|
[ 0.645647, 0.151733 ], |
|
[ 0.764428, 0.119852 ], |
|
[ 0.743132, 0.118758 ], |
|
[ 0.908516, 0.136423 ], |
|
[ 0.84751, 0.164077 ], |
|
[ 0.769335, 0.165099 ], |
|
[ 0.254149, 0.780233 ], |
|
[ 0.726104, 0.780233 ], |
|
], dtype=np.float32) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
landmarks_68_pt = {"mouth": (48, 68), |
|
"right_eyebrow": (17, 22), |
|
"left_eyebrow": (22, 27), |
|
"right_eye": (36, 42), |
|
"left_eye": (42, 48), |
|
"nose": (27, 36), |
|
"jaw": (0, 17)} |
|
|
|
landmarks_68_3D = np.array([ |
|
[-73.393523, -29.801432, 47.667532], |
|
[-72.775014, -10.949766, 45.909403], |
|
[-70.533638, 7.929818, 44.842580], |
|
[-66.850058, 26.074280, 43.141114], |
|
[-59.790187, 42.564390, 38.635298], |
|
[-48.368973, 56.481080, 30.750622], |
|
[-34.121101, 67.246992, 18.456453], |
|
[-17.875411, 75.056892, 3.609035], |
|
[0.098749, 77.061286, -0.881698], |
|
[17.477031, 74.758448, 5.181201], |
|
[32.648966, 66.929021, 19.176563], |
|
[46.372358, 56.311389, 30.770570], |
|
[57.343480, 42.419126, 37.628629], |
|
[64.388482, 25.455880, 40.886309], |
|
[68.212038, 6.990805, 42.281449], |
|
[70.486405, -11.666193, 44.142567], |
|
[71.375822, -30.365191, 47.140426], |
|
[-61.119406, -49.361602, 14.254422], |
|
[-51.287588, -58.769795, 7.268147], |
|
[-37.804800, -61.996155, 0.442051], |
|
[-24.022754, -61.033399, -6.606501], |
|
[-11.635713, -56.686759, -11.967398], |
|
[12.056636, -57.391033, -12.051204], |
|
[25.106256, -61.902186, -7.315098], |
|
[38.338588, -62.777713, -1.022953], |
|
[51.191007, -59.302347, 5.349435], |
|
[60.053851, -50.190255, 11.615746], |
|
[0.653940, -42.193790, -13.380835], |
|
[0.804809, -30.993721, -21.150853], |
|
[0.992204, -19.944596, -29.284036], |
|
[1.226783, -8.414541, -36.948060], |
|
[-14.772472, 2.598255, -20.132003], |
|
[-7.180239, 4.751589, -23.536684], |
|
[0.555920, 6.562900, -25.944448], |
|
[8.272499, 4.661005, -23.695741], |
|
[15.214351, 2.643046, -20.858157], |
|
[-46.047290, -37.471411, 7.037989], |
|
[-37.674688, -42.730510, 3.021217], |
|
[-27.883856, -42.711517, 1.353629], |
|
[-19.648268, -36.754742, -0.111088], |
|
[-28.272965, -35.134493, -0.147273], |
|
[-38.082418, -34.919043, 1.476612], |
|
[19.265868, -37.032306, -0.665746], |
|
[27.894191, -43.342445, 0.247660], |
|
[37.437529, -43.110822, 1.696435], |
|
[45.170805, -38.086515, 4.894163], |
|
[38.196454, -35.532024, 0.282961], |
|
[28.764989, -35.484289, -1.172675], |
|
[-28.916267, 28.612716, -2.240310], |
|
[-17.533194, 22.172187, -15.934335], |
|
[-6.684590, 19.029051, -22.611355], |
|
[0.381001, 20.721118, -23.748437], |
|
[8.375443, 19.035460, -22.721995], |
|
[18.876618, 22.394109, -15.610679], |
|
[28.794412, 28.079924, -3.217393], |
|
[19.057574, 36.298248, -14.987997], |
|
[8.956375, 39.634575, -22.554245], |
|
[0.381549, 40.395647, -23.591626], |
|
[-7.428895, 39.836405, -22.406106], |
|
[-18.160634, 36.677899, -15.121907], |
|
[-24.377490, 28.677771, -4.785684], |
|
[-6.897633, 25.475976, -20.893742], |
|
[0.340663, 26.014269, -22.220479], |
|
[8.444722, 25.326198, -21.025520], |
|
[24.474473, 28.323008, -5.712776], |
|
[8.449166, 30.596216, -20.671489], |
|
[0.205322, 31.408738, -21.903670], |
|
[-7.198266, 30.844876, -20.328022] |
|
], dtype=np.float32) |
|
|
|
FaceType_to_padding_remove_align = { |
|
FaceType.HALF: (0.0, False), |
|
FaceType.MID_FULL: (0.0675, False), |
|
FaceType.FULL: (0.2109375, False), |
|
FaceType.FULL_NO_ALIGN: (0.2109375, True), |
|
FaceType.WHOLE_FACE: (0.40, False), |
|
FaceType.HEAD: (0.70, False), |
|
FaceType.HEAD_NO_ALIGN: (0.70, True), |
|
} |
|
|
|
|
|
def convert_98_to_68(lmrks): |
|
|
|
result = [lmrks[0]] |
|
for i in range(2, 16, 2): |
|
result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2] |
|
result += [lmrks[16]] |
|
for i in range(18, 32, 2): |
|
result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2] |
|
result += [lmrks[32]] |
|
|
|
|
|
result += [lmrks[33], |
|
(lmrks[34] + lmrks[41]) / 2, |
|
(lmrks[35] + lmrks[40]) / 2, |
|
(lmrks[36] + lmrks[39]) / 2, |
|
(lmrks[37] + lmrks[38]) / 2, |
|
] |
|
|
|
result += [(lmrks[42] + lmrks[50]) / 2, |
|
(lmrks[43] + lmrks[49]) / 2, |
|
(lmrks[44] + lmrks[48]) / 2, |
|
(lmrks[45] + lmrks[47]) / 2, |
|
lmrks[46] |
|
] |
|
|
|
|
|
result += list(lmrks[51:60]) |
|
|
|
|
|
result += [lmrks[60], |
|
lmrks[61], |
|
lmrks[63], |
|
lmrks[64], |
|
lmrks[65], |
|
lmrks[67]] |
|
|
|
|
|
result += [lmrks[68], |
|
lmrks[69], |
|
lmrks[71], |
|
lmrks[72], |
|
lmrks[73], |
|
lmrks[75]] |
|
|
|
|
|
result += list(lmrks[76:96]) |
|
|
|
return np.concatenate(result).reshape((68, 2)) |
|
|
|
|
|
def transform_points(points, mat, invert=False): |
|
if invert: |
|
mat = cv2.invertAffineTransform(mat) |
|
points = np.expand_dims(points, axis=1) |
|
points = cv2.transform(points, mat, points.shape) |
|
points = np.squeeze(points) |
|
return points |
|
|
|
|
|
def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0): |
|
if not isinstance(image_landmarks, np.ndarray): |
|
image_landmarks = np.array(image_landmarks) |
|
|
|
|
|
mat = umeyama(np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]), landmarks_2D_new, True)[0:2] |
|
|
|
|
|
g_p = transform_points(np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True) |
|
g_c = g_p[4] |
|
|
|
|
|
tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32) |
|
tb_diag_vec /= npla.norm(tb_diag_vec) |
|
bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32) |
|
bt_diag_vec /= npla.norm(bt_diag_vec) |
|
|
|
|
|
|
|
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) |
|
mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5)) |
|
|
|
if face_type == FaceType.WHOLE_FACE: |
|
|
|
vec = (g_p[0] - g_p[3]).astype(np.float32) |
|
vec_len = npla.norm(vec) |
|
vec /= vec_len |
|
g_c += vec * vec_len * 0.07 |
|
|
|
|
|
|
|
if not remove_align: |
|
l_t = np.array([g_c - tb_diag_vec * mod, |
|
g_c + bt_diag_vec * mod, |
|
g_c + tb_diag_vec * mod]) |
|
else: |
|
|
|
l_t = np.array([g_c - tb_diag_vec * mod, |
|
g_c + bt_diag_vec * mod, |
|
g_c + tb_diag_vec * mod, |
|
g_c - bt_diag_vec * mod, |
|
]) |
|
|
|
|
|
area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1]) |
|
|
|
|
|
side = np.float32(math.sqrt(area) / 2) |
|
|
|
|
|
l_t = np.array([g_c + [-side, -side], |
|
g_c + [side, -side], |
|
g_c + [side, side]]) |
|
|
|
|
|
pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size))) |
|
mat = cv2.getAffineTransform(l_t, pts2) |
|
return mat |
|
|
|
|
|
def get_rect_from_landmarks(image_landmarks): |
|
mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN) |
|
|
|
g_p = transform_points(np.float32([(0, 0), (255, 255)]), mat, True) |
|
|
|
(l, t, r, b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1] |
|
|
|
return (l, t, r, b) |
|
|
|
def get_transform_mat_all(image_landmarks,uni_landmarks,output_size,scale=1,gcx=-0.02,gcy=0.15,face_type=FaceType.WHOLE_FACE): |
|
if not isinstance(image_landmarks, np.ndarray): |
|
image_landmarks = np.array (image_landmarks) |
|
|
|
|
|
mat = umeyama(image_landmarks, uni_landmarks, True)[0:2] |
|
|
|
|
|
g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True) |
|
g_c = g_p[4] |
|
|
|
|
|
|
|
tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32) |
|
tb_diag_vec /= npla.norm(tb_diag_vec) |
|
bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32) |
|
bt_diag_vec /= npla.norm(bt_diag_vec) |
|
|
|
|
|
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) |
|
mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5)) |
|
|
|
vec = (g_p[0]-g_p[3]).astype(np.float32) |
|
vec_len = npla.norm(vec) |
|
vec /= vec_len |
|
g_c += vec*vec_len*[gcx,gcy] |
|
|
|
|
|
|
|
if not remove_align: |
|
l_t = np.array([g_c - tb_diag_vec * mod, |
|
g_c + bt_diag_vec * mod, |
|
g_c + tb_diag_vec * mod]) |
|
else: |
|
|
|
l_t = np.array([g_c - tb_diag_vec * mod, |
|
g_c + bt_diag_vec * mod, |
|
g_c + tb_diag_vec * mod, |
|
g_c - bt_diag_vec * mod, |
|
]) |
|
|
|
|
|
area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1]) |
|
|
|
|
|
side = np.float32(math.sqrt(area) / 2) |
|
|
|
|
|
l_t = np.array([g_c + [-side, -side], |
|
g_c + [side, -side], |
|
g_c + [side, side]]) |
|
|
|
|
|
pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size))) |
|
mat = cv2.getAffineTransform(l_t, pts2) |
|
return mat |
|
|
|
|
|
def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0): |
|
|
|
if len(lmrks) != 68: |
|
raise Exception('works only with 68 landmarks') |
|
lmrks = np.array(lmrks.copy(), dtype=np.int) |
|
|
|
|
|
ml_pnt = (lmrks[36] + lmrks[0]) // 2 |
|
mr_pnt = (lmrks[16] + lmrks[45]) // 2 |
|
|
|
|
|
ql_pnt = (lmrks[36] + ml_pnt) // 2 |
|
qr_pnt = (lmrks[45] + mr_pnt) // 2 |
|
|
|
|
|
bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39])) |
|
bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt)) |
|
|
|
|
|
top_l = lmrks[17:22] |
|
top_r = lmrks[22:27] |
|
|
|
|
|
lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l) |
|
lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r) |
|
return lmrks |
|
|
|
|
|
def get_image_hull_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0): |
|
hull_mask = np.zeros(image_shape[0:2] + (1,), dtype=np.float32) |
|
|
|
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) |
|
|
|
r_jaw = (lmrks[0:9], lmrks[17:18]) |
|
l_jaw = (lmrks[8:17], lmrks[26:27]) |
|
r_cheek = (lmrks[17:20], lmrks[8:9]) |
|
l_cheek = (lmrks[24:27], lmrks[8:9]) |
|
nose_ridge = (lmrks[19:25], lmrks[8:9],) |
|
r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9]) |
|
l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9]) |
|
nose = (lmrks[27:31], lmrks[31:36]) |
|
parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose] |
|
|
|
for item in parts: |
|
merged = np.concatenate(item) |
|
cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,)) |
|
|
|
return hull_mask |
|
|
|
|
|
def get_image_eye_mask(image_shape, image_landmarks): |
|
if len(image_landmarks) != 68: |
|
raise Exception('get_image_eye_mask works only with 68 landmarks') |
|
|
|
h, w, c = image_shape |
|
|
|
hull_mask = np.zeros((h, w, 1), dtype=np.float32) |
|
|
|
image_landmarks = image_landmarks.astype(np.int) |
|
|
|
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[36:42]), (1,)) |
|
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[42:48]), (1,)) |
|
|
|
dilate = h // 32 |
|
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1) |
|
|
|
blur = h // 16 |
|
blur = blur + (1 - blur % 2) |
|
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0) |
|
hull_mask = hull_mask[..., None] |
|
|
|
return hull_mask |
|
|
|
|
|
def get_image_mouth_mask(image_shape, image_landmarks): |
|
if len(image_landmarks) != 68: |
|
raise Exception('get_image_eye_mask works only with 68 landmarks') |
|
|
|
h, w, c = image_shape |
|
|
|
hull_mask = np.zeros((h, w, 1), dtype=np.float32) |
|
|
|
image_landmarks = image_landmarks.astype(np.int) |
|
|
|
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[60:]), (1,)) |
|
|
|
dilate = h // 32 |
|
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1) |
|
|
|
blur = h // 16 |
|
blur = blur + (1 - blur % 2) |
|
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0) |
|
hull_mask = hull_mask[..., None] |
|
|
|
return hull_mask |
|
|
|
|
|
def alpha_to_color(img_alpha, color): |
|
if len(img_alpha.shape) == 2: |
|
img_alpha = img_alpha[..., None] |
|
h, w, c = img_alpha.shape |
|
result = np.zeros((h, w, len(color)), dtype=np.float32) |
|
result[:, :] = color |
|
|
|
return result * img_alpha |
|
|
|
|
|
def get_cmask(image_shape, lmrks, eyebrows_expand_mod=1.0): |
|
h, w, c = image_shape |
|
|
|
hull = get_image_hull_mask(image_shape, lmrks, eyebrows_expand_mod) |
|
|
|
result = np.zeros((h, w, 3), dtype=np.float32) |
|
|
|
def process(w, h, data): |
|
d = {} |
|
cur_lc = 0 |
|
all_lines = [] |
|
for s, pts_loop_ar in data: |
|
lines = [] |
|
for pts, loop in pts_loop_ar: |
|
pts_len = len(pts) |
|
lines.append([[pts[i], pts[(i + 1) % pts_len]] for i in range(pts_len - (0 if loop else 1))]) |
|
lines = np.concatenate(lines) |
|
|
|
lc = lines.shape[0] |
|
all_lines.append(lines) |
|
d[s] = cur_lc, cur_lc + lc |
|
cur_lc += lc |
|
all_lines = np.concatenate(all_lines, 0) |
|
|
|
|
|
line_count = all_lines.shape[0] |
|
pts_count = w * h |
|
|
|
all_lines = np.repeat(all_lines[None, ...], pts_count, axis=0).reshape((pts_count * line_count, 2, 2)) |
|
|
|
pts = np.empty((h, w, line_count, 2), dtype=np.float32) |
|
pts[..., 1] = np.arange(h)[:, None, None] |
|
pts[..., 0] = np.arange(w)[:, None] |
|
pts = pts.reshape((h * w * line_count, -1)) |
|
|
|
a = all_lines[:, 0, :] |
|
b = all_lines[:, 1, :] |
|
pa = pts - a |
|
ba = b - a |
|
ph = np.clip(np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1) |
|
dists = npla.norm(pa - ba * ph[..., None], axis=1).reshape((h, w, line_count)) |
|
|
|
def get_dists(name, thickness=0): |
|
s, e = d[name] |
|
result = dists[..., s:e] |
|
if thickness != 0: |
|
result = np.abs(result) - thickness |
|
return np.min(result, axis=-1) |
|
|
|
return get_dists |
|
|
|
l_eye = lmrks[42:48] |
|
r_eye = lmrks[36:42] |
|
l_brow = lmrks[22:27] |
|
r_brow = lmrks[17:22] |
|
mouth = lmrks[48:60] |
|
|
|
up_nose = np.concatenate((lmrks[27:31], lmrks[33:34])) |
|
down_nose = lmrks[31:36] |
|
nose = np.concatenate((up_nose, down_nose)) |
|
|
|
gdf = process(w, h, |
|
( |
|
('eyes', ((l_eye, True), (r_eye, True))), |
|
('brows', ((l_brow, False), (r_brow, False))), |
|
('up_nose', ((up_nose, False),)), |
|
('down_nose', ((down_nose, False),)), |
|
('mouth', ((mouth, True),)), |
|
) |
|
) |
|
|
|
eyes_fall_dist = w // 32 |
|
eyes_thickness = max(w // 64, 1) |
|
|
|
brows_fall_dist = w // 32 |
|
brows_thickness = max(w // 256, 1) |
|
|
|
nose_fall_dist = w / 12 |
|
nose_thickness = max(w // 96, 1) |
|
|
|
mouth_fall_dist = w // 32 |
|
mouth_thickness = max(w // 64, 1) |
|
|
|
eyes_mask = gdf('eyes', eyes_thickness) |
|
eyes_mask = 1 - np.clip(eyes_mask / eyes_fall_dist, 0, 1) |
|
|
|
|
|
|
|
brows_mask = gdf('brows', brows_thickness) |
|
brows_mask = 1 - np.clip(brows_mask / brows_fall_dist, 0, 1) |
|
|
|
|
|
mouth_mask = gdf('mouth', mouth_thickness) |
|
mouth_mask = 1 - np.clip(mouth_mask / mouth_fall_dist, 0, 1) |
|
|
|
|
|
|
|
def blend(a, b, k): |
|
x = np.clip(0.5 + 0.5 * (b - a) / k, 0.0, 1.0) |
|
return (a - b) * x + b - k * x * (1.0 - x) |
|
|
|
|
|
|
|
|
|
|
|
|
|
nose_mask = blend(gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness * 3) |
|
nose_mask = 1 - np.clip(nose_mask / nose_fall_dist, 0, 1) |
|
|
|
up_nose_mask = gdf('up_nose', nose_thickness) |
|
up_nose_mask = 1 - np.clip(up_nose_mask / nose_fall_dist, 0, 1) |
|
|
|
|
|
down_nose_mask = gdf('down_nose', nose_thickness) |
|
down_nose_mask = 1 - np.clip(down_nose_mask / nose_fall_dist, 0, 1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eyes_mask = eyes_mask * (1 - mouth_mask) |
|
nose_mask = nose_mask * (1 - eyes_mask) |
|
|
|
hull_mask = hull[..., 0].copy() |
|
hull_mask = hull_mask * (1 - eyes_mask) * (1 - brows_mask) * (1 - nose_mask) * (1 - mouth_mask) |
|
|
|
|
|
|
|
mouth_mask = mouth_mask * (1 - nose_mask) |
|
|
|
brows_mask = brows_mask * (1 - nose_mask) * (1 - eyes_mask) |
|
|
|
hull_mask = alpha_to_color(hull_mask, (0, 1, 0)) |
|
eyes_mask = alpha_to_color(eyes_mask, (1, 0, 0)) |
|
brows_mask = alpha_to_color(brows_mask, (0, 0, 1)) |
|
nose_mask = alpha_to_color(nose_mask, (0, 1, 1)) |
|
mouth_mask = alpha_to_color(mouth_mask, (0, 0, 1)) |
|
|
|
|
|
|
|
result = hull_mask + mouth_mask + nose_mask + brows_mask + eyes_mask |
|
result *= hull |
|
|
|
return result |
|
|
|
|
|
def blur_image_hull_mask(hull_mask): |
|
maxregion = np.argwhere(hull_mask == 1.0) |
|
miny, minx = maxregion.min(axis=0)[:2] |
|
maxy, maxx = maxregion.max(axis=0)[:2] |
|
lenx = maxx - minx; |
|
leny = maxy - miny; |
|
masky = int(minx + (lenx // 2)) |
|
maskx = int(miny + (leny // 2)) |
|
lowest_len = min(lenx, leny) |
|
ero = int(lowest_len * 0.085) |
|
blur = int(lowest_len * 0.10) |
|
|
|
hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) |
|
hull_mask = cv2.blur(hull_mask, (blur, blur)) |
|
hull_mask = np.expand_dims(hull_mask, -1) |
|
|
|
return hull_mask |
|
|
|
|
|
mirror_idxs = [ |
|
[0, 16], |
|
[1, 15], |
|
[2, 14], |
|
[3, 13], |
|
[4, 12], |
|
[5, 11], |
|
[6, 10], |
|
[7, 9], |
|
|
|
[17, 26], |
|
[18, 25], |
|
[19, 24], |
|
[20, 23], |
|
[21, 22], |
|
|
|
[36, 45], |
|
[37, 44], |
|
[38, 43], |
|
[39, 42], |
|
[40, 47], |
|
[41, 46], |
|
|
|
[31, 35], |
|
[32, 34], |
|
|
|
[50, 52], |
|
[49, 53], |
|
[48, 54], |
|
[59, 55], |
|
[58, 56], |
|
[67, 65], |
|
[60, 64], |
|
[61, 63]] |
|
|
|
|
|
def mirror_landmarks(landmarks, val): |
|
result = landmarks.copy() |
|
|
|
for idx in mirror_idxs: |
|
result[idx] = result[idx[::-1]] |
|
|
|
result[:, 0] = val - result[:, 0] - 1 |
|
return result |
|
|
|
|
|
def get_face_struct_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,)): |
|
mask = np.zeros(image_shape[0:2] + (len(color),), dtype=np.float32) |
|
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) |
|
draw_landmarks(mask, image_landmarks, color=color, draw_circles=False, thickness=2) |
|
return mask |
|
|
|
|
|
def draw_landmarks(image, image_landmarks, color=(0, 255, 0), draw_circles=True, thickness=1, transparent_mask=False): |
|
if len(image_landmarks) != 68: |
|
raise Exception('get_image_eye_mask works only with 68 landmarks') |
|
|
|
int_lmrks = np.array(image_landmarks, dtype=np.int) |
|
|
|
jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])] |
|
right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])] |
|
left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])] |
|
mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])] |
|
right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])] |
|
left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])] |
|
nose = int_lmrks[slice(*landmarks_68_pt["nose"])] |
|
|
|
|
|
cv2.polylines(image, |
|
tuple(np.array([v]) for v in (right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])))), |
|
False, color, thickness=thickness, lineType=cv2.LINE_AA) |
|
|
|
cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)), |
|
True, color, thickness=thickness, lineType=cv2.LINE_AA) |
|
|
|
if draw_circles: |
|
|
|
for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0): |
|
cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA) |
|
|
|
for x, y in jaw: |
|
cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA) |
|
|
|
if transparent_mask: |
|
mask = get_image_hull_mask(image.shape, image_landmarks) |
|
image[...] = (image * (1 - mask) + image * mask / 2)[...] |
|
|
|
|
|
def draw_rect_landmarks(image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False, |
|
landmarks_color=(0, 255, 0)): |
|
draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask) |
|
imagelib.draw_rect(image, rect, (255, 0, 0), 2) |
|
|
|
image_to_face_mat = get_transform_mat(image_landmarks, face_size, face_type) |
|
points = transform_points([(0, 0), (0, face_size - 1), (face_size - 1, face_size - 1), (face_size - 1, 0)], |
|
image_to_face_mat, True) |
|
imagelib.draw_polygon(image, points, (0, 0, 255), 2) |
|
|
|
points = transform_points( |
|
[(int(face_size * 0.05), 0), (int(face_size * 0.1), int(face_size * 0.1)), (0, int(face_size * 0.1))], |
|
image_to_face_mat, True) |
|
imagelib.draw_polygon(image, points, (0, 0, 255), 2) |
|
|
|
|
|
def calc_face_pitch(landmarks): |
|
if not isinstance(landmarks, np.ndarray): |
|
landmarks = np.array(landmarks) |
|
t = ((landmarks[6][1] - landmarks[8][1]) + (landmarks[10][1] - landmarks[8][1])) / 2.0 |
|
b = landmarks[8][1] |
|
return float(b - t) |
|
|
|
|
|
def estimate_averaged_yaw(landmarks): |
|
|
|
if not isinstance(landmarks, np.ndarray): |
|
landmarks = np.array(landmarks) |
|
l = ((landmarks[27][0] - landmarks[0][0]) + (landmarks[28][0] - landmarks[1][0]) + ( |
|
landmarks[29][0] - landmarks[2][0])) / 3.0 |
|
r = ((landmarks[16][0] - landmarks[27][0]) + (landmarks[15][0] - landmarks[28][0]) + ( |
|
landmarks[14][0] - landmarks[29][0])) / 3.0 |
|
return float(r - l) |
|
|
|
|
|
def estimate_pitch_yaw_roll(aligned_landmarks, size=256): |
|
""" |
|
returns pitch,yaw,roll [-pi/2...+pi/2] |
|
""" |
|
shape = (size, size) |
|
focal_length = shape[1] |
|
camera_center = (shape[1] / 2, shape[0] / 2) |
|
camera_matrix = np.array( |
|
[[focal_length, 0, camera_center[0]], |
|
[0, focal_length, camera_center[1]], |
|
[0, 0, 1]], dtype=np.float32) |
|
|
|
(_, rotation_vector, _) = cv2.solvePnP( |
|
np.concatenate((landmarks_68_3D[:27], landmarks_68_3D[30:36]), axis=0), |
|
np.concatenate((aligned_landmarks[:27], aligned_landmarks[30:36]), axis=0).astype(np.float32), |
|
camera_matrix, |
|
np.zeros((4, 1))) |
|
|
|
pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles(cv2.Rodrigues(rotation_vector)[0]) |
|
|
|
half_pi = math.pi / 2.0 |
|
pitch = np.clip(pitch, -half_pi, half_pi) |
|
yaw = np.clip(yaw, -half_pi, half_pi) |
|
roll = np.clip(roll, -half_pi, half_pi) |
|
|
|
return -pitch, yaw, roll |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
""" |
|
def get_averaged_transform_mat (img_landmarks, |
|
img_landmarks_prev, |
|
img_landmarks_next, |
|
average_frame_count, |
|
average_center_frame_count, |
|
output_size, face_type, scale=1.0): |
|
|
|
l_c_list = [] |
|
tb_diag_vec_list = [] |
|
bt_diag_vec_list = [] |
|
mod_list = [] |
|
|
|
count = max(average_frame_count,average_center_frame_count) |
|
for i in range ( -count, count+1, 1 ): |
|
if i < 0: |
|
lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None |
|
elif i > 0: |
|
lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None |
|
else: |
|
lmrks = img_landmarks |
|
|
|
if lmrks is None: |
|
continue |
|
|
|
l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale) |
|
|
|
if i >= -average_frame_count and i <= average_frame_count: |
|
tb_diag_vec_list.append(tb_diag_vec) |
|
bt_diag_vec_list.append(bt_diag_vec) |
|
mod_list.append(mod) |
|
|
|
if i >= -average_center_frame_count and i <= average_center_frame_count: |
|
l_c_list.append(l_c) |
|
|
|
tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 ) |
|
bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 ) |
|
mod = np.mean( np.array(mod_list), axis=0 ) |
|
l_c = np.mean( np.array(l_c_list), axis=0 ) |
|
|
|
return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type) |
|
|
|
|
|
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0): |
|
if not isinstance(image_landmarks, np.ndarray): |
|
image_landmarks = np.array (image_landmarks) |
|
|
|
# get face padding value for FaceType |
|
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) |
|
|
|
# estimate landmarks transform from global space to local aligned space with bounds [0..1] |
|
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] |
|
|
|
# get corner points in global space |
|
l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True) |
|
l_c = l_p[4] |
|
|
|
# calc diagonal vectors between corners in global space |
|
tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32) |
|
tb_diag_vec /= npla.norm(tb_diag_vec) |
|
bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32) |
|
bt_diag_vec /= npla.norm(bt_diag_vec) |
|
|
|
# calc modifier of diagonal vectors for scale and padding value |
|
mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) ) |
|
|
|
# calc 3 points in global space to estimate 2d affine transform |
|
if not remove_align: |
|
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), |
|
np.round( l_c + bt_diag_vec*mod ), |
|
np.round( l_c + tb_diag_vec*mod ) ] ) |
|
else: |
|
# remove_align - face will be centered in the frame but not aligned |
|
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), |
|
np.round( l_c + bt_diag_vec*mod ), |
|
np.round( l_c + tb_diag_vec*mod ), |
|
np.round( l_c - bt_diag_vec*mod ), |
|
] ) |
|
|
|
# get area of face square in global space |
|
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) |
|
|
|
# calc side of square |
|
side = np.float32(math.sqrt(area) / 2) |
|
|
|
# calc 3 points with unrotated square |
|
l_t = np.array( [ np.round( l_c + [-side,-side] ), |
|
np.round( l_c + [ side,-side] ), |
|
np.round( l_c + [ side, side] ) ] ) |
|
|
|
# calc affine transform from 3 global space points to 3 local space points size of 'output_size' |
|
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) |
|
mat = cv2.getAffineTransform(l_t,pts2) |
|
|
|
return mat |
|
""" |
|
|