import colorsys import math from enum import IntEnum import cv2 import numpy as np import numpy.linalg as npla from face_detect.core import imagelib from face_detect.core import mathlib from face_detect.core.mathlib.umeyama import umeyama from face_detect.FaceType import FaceType mesh_33=[70,63,105,66,107,336,296,334,293,300,168,197,5,4,240,99,2,328,460,33,160,158,133,153,144,362,385,387,263,373,380,57,287] landmarks_2D_4=np.array([ [0.224152 , 0.2119465], #left iris mean 37 38 40 41 [0.75610125, 0.2119465],#right iris mean 43 44 46 47 [0.490127, 0.515625], # nose 30 [0.4901265, 0.780233] #mouth mean 48 54 ]) landmarks_2D_4_bottom=np.array([ [0.2218305, 0.244588 ], #left iris mean 40 41 [0.7584225, 0.244588],#right iris mean 46 47 [0.490127, 0.515625], # nose 30 [0.4901265, 0.780233] #mouth mean 48 54 ]) landmarks_2D = np.array([ [0.000213256, 0.106454], # 17 [0.0752622, 0.038915], # 18 [0.18113, 0.0187482], # 19 [0.29077, 0.0344891], # 20 [0.393397, 0.0773906], # 21 [0.586856, 0.0773906], # 22 [0.689483, 0.0344891], # 23 [0.799124, 0.0187482], # 24 [0.904991, 0.038915], # 25 [0.98004, 0.106454], # 26 [0.490127, 0.203352], # 27 [0.490127, 0.307009], # 28 [0.490127, 0.409805], # 29 [0.490127, 0.515625], # 30 [0.36688, 0.587326], # 31 [0.426036, 0.609345], # 32 [0.490127, 0.628106], # 33 [0.554217, 0.609345], # 34 [0.613373, 0.587326], # 35 [0.121737, 0.216423], # 36 [0.187122, 0.178758], # 37 [0.265825, 0.179852], # 38 [0.334606, 0.231733], # 39 [0.260918, 0.245099], # 40 [0.182743, 0.244077], # 41 [0.645647, 0.231733], # 42 [0.714428, 0.179852], # 43 [0.793132, 0.178758], # 44 [0.858516, 0.216423], # 45 [0.79751, 0.244077], # 46 [0.719335, 0.245099], # 47 [0.254149, 0.780233], # 48 [0.340985, 0.745405], # 49 [0.428858, 0.727388], # 50 [0.490127, 0.742578], # 51 [0.551395, 0.727388], # 52 [0.639268, 0.745405], # 53 [0.726104, 0.780233], # 54 [0.642159, 0.864805], # 55 [0.556721, 0.902192], # 56 [0.490127, 0.909281], # 57 [0.423532, 0.902192], # 58 [0.338094, 0.864805], # 59 [0.290379, 0.784792], # 60 [0.428096, 0.778746], # 61 [0.490127, 0.785343], # 62 [0.552157, 0.778746], # 63 [0.689874, 0.784792], # 64 [0.553364, 0.824182], # 65 [0.490127, 0.831803], # 66 [0.42689, 0.824182] # 67 ], dtype=np.float32) landmarks_2D_new = np.array([ [0.000213256, 0.106454], # 17 [0.0752622, 0.038915], # 18 [0.18113, 0.0187482], # 19 [0.29077, 0.0344891], # 20 [0.393397, 0.0773906], # 21 [0.586856, 0.0773906], # 22 [0.689483, 0.0344891], # 23 [0.799124, 0.0187482], # 24 [0.904991, 0.038915], # 25 [0.98004, 0.106454], # 26 [0.490127, 0.203352], # 27 [0.490127, 0.307009], # 28 [0.490127, 0.409805], # 29 [0.490127, 0.515625], # 30 [0.36688, 0.587326], # 31 [0.426036, 0.609345], # 32 [0.490127, 0.628106], # 33 [0.554217, 0.609345], # 34 [0.613373, 0.587326], # 35 [0.121737, 0.216423], # 36 [0.187122, 0.178758], # 37 [0.265825, 0.179852], # 38 [0.334606, 0.231733], # 39 [0.260918, 0.245099], # 40 [0.182743, 0.244077], # 41 [0.645647, 0.231733], # 42 [0.714428, 0.179852], # 43 [0.793132, 0.178758], # 44 [0.858516, 0.216423], # 45 [0.79751, 0.244077], # 46 [0.719335, 0.245099], # 47 [0.254149, 0.780233], # 48 [0.726104, 0.780233], # 54 ], dtype=np.float32) landmarks_2D_new_mesh = np.array([ [ 0.000213256, 0.106454 ], #17 [ 0.0752622, 0.038915 ], #18 [0.1281961, 0.0288316], #19[ 0.18113, 0.0187482 ] [ 0.29077, 0.0144891 ], #20 [ 0.393397, 0.0773906 ], #21 [ 0.586856, 0.0773906 ], #22 [ 0.689483, 0.0144891 ], #23 [0.8520575, 0.0288316], #24[ 0.799124, 0.0187482 ] [ 0.904991, 0.038915 ], #25 [ 0.98004, 0.106454 ], #26 [ 0.490127, 0.203352 ], #27 [ 0.490127, 0.307009 ], #28 [ 0.490127, 0.409805 ], #29 [ 0.490127, 0.515625 ], #30 [0.396458 , 0.5983355], #31 [ 0.36688, 0.587326 ] [ 0.426036, 0.609345 ], #32 [ 0.490127, 0.628106 ], #33 [ 0.554217, 0.609345 ], #34 [ 0.613373, 0.587326 ], #35 [ 0.071737, 0.136423 ], #36 [ 0.137122, 0.118758 ], #37 [ 0.215825, 0.119852 ], #38 [ 0.334606, 0.151733 ], #39 [ 0.210918, 0.165099 ], #40 [ 0.132743, 0.164077 ], #41 [ 0.645647, 0.151733 ], #42 [ 0.764428, 0.119852 ], #43 [ 0.743132, 0.118758 ], #44 [ 0.908516, 0.136423 ], #45 [ 0.84751, 0.164077 ], #46 [ 0.769335, 0.165099 ], #47 [ 0.254149, 0.780233 ], #48 [ 0.726104, 0.780233 ], #54 ], dtype=np.float32) # landmarks_468_moving_parts_indexes = [0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42, 43, 46, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 95, 96, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 117, 118, 124, 130, 132, 133, 135, 136, 138, 139, 140, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 168, 169, 170, 171, 172, 173, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 189, 190, 191, 192, 193, 194, 199, 200, 201, 202, 204, 208, 210, 211, 212, 213, 214, 215, 221, 222, 223, 224, 225, 226, 228, 229, 230, 231, 232, 233, 243, 244, 245, 246, 247, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 267, 268, 269, 270, 271, 272, 273, 276, 282, 283, 284, 285, 286, 287, 288, 291, 292, 293, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 306, 307, 308, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 324, 325, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 346, 347, 353, 359, 361, 362, 364, 365, 367, 368, 369, 372, 373, 374, 375, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 413, 414, 415, 416, 417, 418, 421, 422, 424, 428, 430, 431, 432, 433, 434, 435, 441, 442, 443, 444, 445, 446, 448, 449, 450, 451, 452, 453, 463, 464, 465, 466, 467] # uni_landmarks_468 = np.array( # [[ 0.49066195, 0.7133885 ], # [ 0.49042386, 0.52723485], # [ 0.49050152, 0.6244965 ], # [ 0.45844677, 0.39348277], # [ 0.4905825 , 0.49120593], # [ 0.49006602, 0.43998772], # [ 0.48907965, 0.26775706], # [ 0.11721139, 0.23243594], # [ 0.48957095, 0.11063451], # [ 0.48949632, 0.03535742], # [ 0.48905632, -0.25326234], # [ 0.4907858 , 0.73766613], # [ 0.49081355, 0.7606857 ], # [ 0.4908666 , 0.7839426 ], # [ 0.49079415, 0.78913504], # [ 0.4908271 , 0.80801845], # [ 0.49086872, 0.831855 ], # [ 0.49092326, 0.8631041 ], # [ 0.49104446, 0.94170016], # [ 0.49009967, 0.5546924 ], # [ 0.44398275, 0.5741402 ], # [-0.2106727 , 0.00861922], # [ 0.2523662 , 0.2832579 ], # [ 0.2042254 , 0.28945392], # [ 0.1552372 , 0.28322184], # [ 0.09056008, 0.24730967], # [ 0.30096018, 0.27277085], # [ 0.21548809, 0.16713436], # [ 0.2595488 , 0.17071684], # [ 0.16957955, 0.17298089], # [ 0.13164258, 0.18425746], # [ 0.043018 , 0.28581 ], # [ 0.30856833, 1.0507976 ], # [ 0.10015843, 0.22331452], # [-0.20773543, 0.26701325], # [-0.02414621, 0.25144747], # [ 0.23481508, 0.5045001 ], # [ 0.44063616, 0.7097012 ], # [ 0.4449884 , 0.762481 ], # [ 0.3840104 , 0.7218947 ], # [ 0.33943903, 0.73847425], # [ 0.40284824, 0.76374006], # [ 0.36457124, 0.76704985], # [ 0.26937196, 0.84716266], # [ 0.46683946, 0.5275276 ], # [ 0.4642676 , 0.49167544], # [ 0.06039319, 0.11509081], # [ 0.31504983, 0.36394927], # [ 0.3660137 , 0.52945083], # [ 0.3509634 , 0.50311893], # [ 0.09496811, 0.5005815 ], # [ 0.46075967, 0.4424029 ], # [ 0.20108324, 0.05883435], # [ 0.12877828, 0.07731954], # [-0.09675749, -0.09848522], # [ 0.39672711, 0.09345116], # [ 0.29908365, 0.18449144], # [ 0.23298171, 0.7922538 ], # [-0.27583498, 0.85219014], # [ 0.38898414, 0.5723152 ], # [ 0.41446668, 0.59347576], # [ 0.28167963, 0.7884952 ], # [ 0.30013445, 0.7875627 ], # [ 0.09448256, 0.03961415], # [ 0.3531811 , 0.5553779 ], # [ 0.2873921 , 0.05599196], # [ 0.28232294, 0.01076962], # [ 0.1903341 , -0.23029903], # [ 0.0108011 , -0.03099815], # [ 0.24915197, -0.10741784], # [ 0.01047484, 0.08868673], # [-0.08942058, 0.05201372], # [ 0.44268388, 0.7376863 ], # [ 0.39652622, 0.741894 ], # [ 0.35389552, 0.7514722 ], # [ 0.393559 , 0.5851372 ], # [ 0.2925385 , 0.7871472 ], # [ 0.31904542, 0.80939215], # [ 0.32005206, 0.787085 ], # [ 0.4195982 , 0.5444628 ], # [ 0.3688312 , 0.78418756], # [ 0.40608776, 0.7841225 ], # [ 0.4472093 , 0.78405076], # [ 0.43053833, 0.9379409 ], # [ 0.44192585, 0.8617842 ], # [ 0.44321233, 0.82923037], # [ 0.4432334 , 0.80578357], # [ 0.44304678, 0.78921837], # [ 0.36314115, 0.7893578 ], # [ 0.36057413, 0.8040033 ], # [ 0.35472178, 0.8187327 ], # [ 0.34614718, 0.83330894], # [ 0.2959003 , 0.69076014], # [-0.37090415, 0.5509728 ], # [ 0.4903264 , 0.5851119 ], # [ 0.3370172 , 0.78961957], # [ 0.33070365, 0.8010128 ], # [ 0.43397966, 0.6231119 ], # [ 0.35356513, 0.59569615], # [ 0.42509514, 0.6093918 ], # [ 0.2635329 , 0.39636588], # [ 0.19704658, 0.43663597], # [ 0.33384863, 0.52658314], # [ 0.03225203, -0.18047164], # [ 0.11854403, -0.08533629], # [ 0.18350407, 0.01215954], # [ 0.31292278, 0.8845064 ], # [ 0.3862302 , 0.02093028], # [ 0.36480215, -0.1098879 ], # [ 0.33342764, -0.2497105 ], # [ 0.11592615, 0.2646692 ], # [-0.00803981, 0.3294946 ], # [ 0.33535972, 0.26431814], # [ 0.05940344, 0.18766014], # [ 0.36188984, 0.33336782], # [ 0.39879864, 0.50869733], # [-0.07952328, 0.36885905], # [ 0.04230375, 0.36800843], # [ 0.11137532, 0.3864613 ], # [ 0.19386435, 0.37397826], # [ 0.25749052, 0.34993485], # [ 0.310977 , 0.3240539 ], # [ 0.44813582, 0.2762354 ], # [-0.06039021, 0.4864401 ], # [ 0.00945808, 0.17624807], # [ 0.4739895 , 0.55369264], # [ 0.32125092, 0.4170324 ], # [-0.36162117, 0.27013144], # [ 0.3592803 , 0.3023075 ], # [ 0.30784345, 0.529875 ], # [ 0.07601253, 0.22579695], # [ 0.3824061 , 0.47686696], # [-0.33810768, 0.70034444], # [ 0.34643772, 0.24336138], # [ 0.42429656, 0.45338264], # [ 0.02854156, 0.939626 ], # [-0.04352415, 1.0322431 ], # [-0.20510256, 0.51651907], # [-0.06969981, 0.8698207 ], # [-0.1581445 , 0.14948419], # [ 0.2889787 , 1.1224228 ], # [ 0.47446907, 0.58377683], # [ 0.2818322 , 0.4586393 ], # [-0.08708218, 0.2627534 ], # [ 0.16877942, 0.25976214], # [ 0.21234928, 0.267416 ], # [ 0.30676025, 0.81592965], # [-0.06259334, 0.6009466 ], # [ 0.36930662, 1.2302231 ], # [ 0.17070079, 1.149443 ], # [ 0.07714309, 1.0989524 ], # [ 0.48931465, -0.1052461 ], # [ 0.49159575, 1.2484183 ], # [ 0.2527582 , 0.26420003], # [ 0.30066028, 0.25829503], # [ 0.3310663 , 0.25034374], # [-0.05075949, 0.16421606], # [ 0.29250854, 0.19938153], # [ 0.2522571 , 0.18826446], # [ 0.21220936, 0.18724632], # [ 0.16866222, 0.19260857], # [ 0.13789575, 0.2011967 ], # [-0.29335994, 0.12383505], # [ 0.1379709 , 0.24424627], # [ 0.49057597, 0.65296 ], # [ 0.34147182, 0.663431 ], # [ 0.3941785 , 0.5603462 ], # [ 0.43007633, 0.6569765 ], # [ 0.48963526, 0.17996965], # [ 0.11681002, 1.0107123 ], # [ 0.19942053, 1.068824 ], # [ 0.38605705, 1.1563928 ], # [-0.16756529, 0.9615808 ], # [ 0.32817602, 0.21989337], # [ 0.41141313, 0.3578073 ], # [ 0.49127796, 1.1678538 ], # [ 0.27080515, 1.195178 ], # [-0.19307071, 0.6481067 ], # [ 0.399859 , 0.7892937 ], # [ 0.39875022, 0.80587196], # [ 0.39717573, 0.8256797 ], # [ 0.3931817 , 0.85224336], # [ 0.3670306 , 0.9161113 ], # [ 0.3256227 , 0.7724022 ], # [ 0.31488904, 0.76426226], # [ 0.3001029 , 0.7583232 ], # [ 0.2565659 , 0.73397243], # [ 0.0438394 , 0.6234349 ], # [ 0.40628996, 0.30296788], # [ 0.37707803, 0.19498621], # [ 0.34125936, 0.21069102], # [ 0.33733743, 0.7842425 ], # [ 0.00882016, 0.769232 ], # [ 0.4335431 , 0.1821002 ], # [ 0.33409703, 0.9826546 ], # [ 0.49011812, 0.3896104 ], # [ 0.45311242, 0.34152514], # [ 0.4899982 , 0.33611432], # [ 0.369907 , 0.43193236], # [ 0.49116373, 1.0932964 ], # [ 0.49107185, 1.0132186 ], # [ 0.41421878, 1.008873 ], # [ 0.21551576, 0.8785059 ], # [ 0.27587482, 0.57461077], # [ 0.2683325 , 0.9399872 ], # [ 0.17091931, 0.56899554], # [ 0.23741819, 0.6283017 ], # [ 0.12783033, 0.65916985], # [ 0.39875996, 1.0855893 ], # [ 0.33251646, 0.45881665], # [ 0.16138549, 0.93153137], # [ 0.23269826, 0.99740875], # [ 0.17994387, 0.8051213 ], # [-0.06026869, 0.7033027 ], # [ 0.10063827, 0.8241594 ], # [-0.15810522, 0.7679798 ], # [ 0.2014156 , 0.7000692 ], # [ 0.365875 , 0.3839739 ], # [ 0.4115726 , 0.5293855 ], # [ 0.378973 , 0.5476473 ], # [ 0.43235463, 0.49621448], # [ 0.3385827 , 0.15134089], # [ 0.27179635, 0.12940899], # [ 0.21341887, 0.12485553], # [ 0.15807948, 0.12881717], # [ 0.10610204, 0.14814937], # [ 0.03133116, 0.236169 ], # [-0.21341309, 0.38895622], # [ 0.07818349, 0.3101151 ], # [ 0.1318462 , 0.32528982], # [ 0.19485526, 0.32642388], # [ 0.25329807, 0.31256682], # [ 0.30569646, 0.29578218], # [ 0.34839994, 0.2842457 ], # [-0.3824783 , 0.41054142], # [ 0.37162504, 0.5664833 ], # [ 0.41687053, 0.40615496], # [ 0.4433516 , 0.5242282 ], # [ 0.44805393, 0.5562703 ], # [ 0.43453053, 0.5407472 ], # [ 0.37351128, 0.58924097], # [ 0.46121803, 0.55474806], # [ 0.45942986, 0.5810936 ], # [ 0.35955238, 0.24802393], # [ 0.38181108, 0.25985107], # [ 0.40143687, 0.26679716], # [ 0.11717269, 0.2102652 ], # [ 0.0940459 , 0.2016577 ], # [ 0.5217974 , 0.39331725], # [ 0.8625129 , 0.23113514], # [ 0.5369363 , 0.57397795], # [ 1.1896138 , 0.00617525], # [ 0.7275363 , 0.28242856], # [ 0.7756985 , 0.2884565 ], # [ 0.82466465, 0.28205347], # [ 0.88921595, 0.24591576], # [ 0.6788919 , 0.27210945], # [ 0.7640089 , 0.166177 ], # [ 0.7199609 , 0.16991326], # [ 0.8099376 , 0.17186326], # [ 0.8479136 , 0.18300733], # [ 0.9368992 , 0.28424102], # [ 0.67367214, 1.0503516 ], # [ 0.8795338 , 0.22195426], # [ 1.1875838 , 0.26458502], # [ 1.0039485 , 0.24965489], # [ 0.74551606, 0.50375396], # [ 0.54075617, 0.7095265 ], # [ 0.5365969 , 0.76231945], # [ 0.59742403, 0.7215222 ], # [ 0.6420548 , 0.7379461 ], # [ 0.5787324 , 0.7634331 ], # [ 0.617019 , 0.766611 ], # [ 0.71218634, 0.8469107 ], # [ 0.513503 , 0.52683127], # [ 0.5170686 , 0.49132976], # [ 0.91894245, 0.11362247], # [ 0.66487545, 0.36299667], # [ 0.61502695, 0.52894545], # [ 0.6296784 , 0.50242335], # [ 0.88566196, 0.49919614], # [ 0.5193738 , 0.4423927 ], # [ 0.7780587 , 0.05788935], # [ 0.8504331 , 0.07610969], # [ 1.0753254 , -0.1005309 ], # [ 0.5824533 , 0.09305263], # [ 0.6804744 , 0.18382579], # [ 0.7485537 , 0.79121745], # [ 1.2577202 , 0.8495136 ], # [ 0.59192824, 0.57196105], # [ 0.5665197 , 0.59321034], # [ 0.6999867 , 0.7877651 ], # [ 0.6814933 , 0.7868972 ], # [ 0.8846023 , 0.03829005], # [ 0.62761134, 0.5547819 ], # [ 0.6917209 , 0.05532694], # [ 0.6966465 , 0.01012804], # [ 0.7876697 , -0.2309872 ], # [ 0.9680314 , -0.03263693], # [ 0.7294528 , -0.1080169 ], # [ 0.96877015, 0.08704082], # [ 1.0685298 , 0.05000517], # [ 0.538806 , 0.7375185 ], # [ 0.5849781 , 0.7415651 ], # [ 0.62764204, 0.7509944 ], # [ 0.58739805, 0.5847989 ], # [ 0.68912315, 0.78645504], # [ 0.6626941 , 0.8087924 ], # [ 0.6616096 , 0.7864889 ], # [ 0.5612171 , 0.5442156 ], # [ 0.61282057, 0.7837617 ], # [ 0.575564 , 0.7838267 ], # [ 0.5344426 , 0.7838985 ], # [ 0.551505 , 0.93764293], # [ 0.5399973 , 0.8616131 ], # [ 0.53859717, 0.8290639 ], # [ 0.5384943 , 0.8056173 ], # [ 0.53862303, 0.78905153], # [ 0.6185288 , 0.78891206], # [ 0.62114686, 0.8035485 ], # [ 0.62705064, 0.81825733], # [ 0.635676 , 0.8328036 ], # [ 0.6854969 , 0.69067734], # [ 1.3517375 , 0.54796624], # [ 0.64465326, 0.78908265], # [ 0.6510032 , 0.8004538 ], # [ 0.5471015 , 0.62291807], # [ 0.62742317, 0.59512955], # [ 0.55593795, 0.6091671 ], # [ 0.7161671 , 0.39546603], # [ 0.7836529 , 0.435396 ], # [ 0.64694774, 0.5258542 ], # [ 0.94603044, -0.1820665 ], # [ 0.86011904, -0.08652072], # [ 0.79549086, 0.01118712], # [ 0.66893554, 0.8840338 ], # [ 0.59274685, 0.02056277], # [ 0.613851 , -0.11025709], # [ 0.64526045, -0.25000137], # [ 0.8639107 , 0.26336375], # [ 0.9881146 , 0.3277454 ], # [ 0.6445285 , 0.26371115], # [ 0.92017305, 0.18616839], # [ 0.61790556, 0.3323734 ], # [ 0.58225924, 0.5077285 ], # [ 1.0597262 , 0.36687428], # [ 0.93791103, 0.36642405], # [ 0.86892897, 0.38505408], # [ 0.78624976, 0.37287512], # [ 0.7223912 , 0.34902957], # [ 0.6687594 , 0.32310694], # [ 0.5315497 , 0.2757726 ], # [ 1.0409807 , 0.48452145], # [ 0.9700836 , 0.17458573], # [ 0.5065989 , 0.55419755], # [ 0.6590531 , 0.41624966], # [ 1.3414742 , 0.26715896], # [ 0.62023264, 0.30108824], # [ 0.67289865, 0.5290446 ], # [ 0.9036883 , 0.22435239], # [ 0.59769833, 0.47659585], # [ 1.3194624 , 0.6974514 ], # [ 0.63339525, 0.24286939], # [ 0.5571053 , 0.45250946], # [ 0.9535533 , 0.9380257 ], # [ 1.0260391 , 1.0303764 ], # [ 1.1858007 , 0.51410204], # [ 1.0515786 , 0.867869 ], # [ 1.1375865 , 0.14722979], # [ 0.6935665 , 1.1218798 ], # [ 0.5063422 , 0.58382744], # [ 0.69926125, 0.45745537], # [ 1.0669235 , 0.26074636], # [ 0.8110406 , 0.25864118], # [ 0.7674977 , 0.26644707], # [ 0.67500204, 0.81528693], # [ 1.0435516 , 0.5990178 ], # [ 0.6121316 , 1.2306852 ], # [ 0.81222653, 1.1483234 ], # [ 0.9056057 , 1.0975065 ], # [ 0.7270778 , 0.26337218], # [ 0.6791554 , 0.25763443], # [ 0.6487802 , 0.24975733], # [ 1.0302606 , 0.16233999], # [ 0.68710136, 0.19869283], # [ 0.72731376, 0.18743533], # [ 0.7673578 , 0.1862774 ], # [ 0.81092334, 0.1914876 ], # [ 0.84171957, 0.1999683 ], # [ 1.2727026 , 0.12110176], # [ 0.8417947 , 0.24301787], # [ 0.63978463, 0.6627527 ], # [ 0.5866921 , 0.5600102 ], # [ 0.5511283 , 0.6567636 ], # [ 0.8655194 , 1.009457 ], # [ 0.78306264, 1.0678959 ], # [ 0.59620714, 1.1564037 ], # [ 1.149833 , 0.9592815 ], # [ 0.65151644, 0.21932903], # [ 0.56865776, 0.3571483 ], # [ 0.71228063, 1.1944076 ], # [ 1.1742088 , 0.6457327 ], # [ 0.5818109 , 0.78897613], # [ 0.5829775 , 0.80555046], # [ 0.5846211 , 0.82535255], # [ 0.5887078 , 0.8519021 ], # [ 0.6150045 , 0.916079 ], # [ 0.65597004, 0.771831 ], # [ 0.66669285, 0.7636482 ], # [ 0.6814582 , 0.7576576 ], # [ 0.7245435 , 0.73241323], # [ 0.9371713 , 0.62184393], # [ 0.5736738 , 0.30186948], # [ 0.60240346, 0.19448838], # [ 0.6383993 , 0.21017241], # [ 0.64431435, 0.7837067 ], # [ 0.9726586 , 0.7675604 ], # [ 0.54576766, 0.18157108], # [ 0.6477745 , 0.98230904], # [ 0.5269076 , 0.34123868], # [ 0.61068684, 0.43131724], # [ 0.56792 , 1.0087004 ], # [ 0.7662271 , 0.8776794 ], # [ 0.7048996 , 0.57387614], # [ 0.7136024 , 0.9394351 ], # [ 0.8097781 , 0.56784695], # [ 0.7435453 , 0.62753886], # [ 0.85328954, 0.6578133 ], # [ 0.5835228 , 1.0854707 ], # [ 0.64810187, 0.45811343], # [ 0.82059515, 0.9304676 ], # [ 0.7494546 , 0.9966611 ], # [ 0.8015866 , 0.80400985], # [ 1.0415541 , 0.70138854], # [ 0.8809724 , 0.8228132 ], # [ 1.1396528 , 0.7657218 ], # [ 0.7798614 , 0.69881856], # [ 0.6143189 , 0.383193 ], # [ 0.56934875, 0.52867246], # [ 0.60162777, 0.54706186], # [ 0.5470082 , 0.4963955 ], # [ 0.6408297 , 0.15073723], # [ 0.7075675 , 0.12865019], # [ 0.76593757, 0.12391254], # [ 0.8212976 , 0.12768434], # [ 0.87334216, 0.14682971], # [ 0.948411 , 0.23457018], # [ 1.1936799 , 0.38651106], # [ 0.90181875, 0.30865455], # [ 0.84818983, 0.3240165 ], # [ 0.7851249 , 0.32537246], # [ 0.72658616, 0.3116911 ], # [ 0.6740513 , 0.2949461 ], # [ 0.63111407, 0.28325075], # [ 1.362823 , 0.4074953 ], # [ 0.60951644, 0.5658945 ], # [ 0.5634702 , 0.4055624 ], # [ 0.5374476 , 0.5247268 ], # [ 0.53280455, 0.5561224 ], # [ 0.5462737 , 0.5405522 ], # [ 0.6075077 , 0.58877414], # [ 0.51933056, 0.55477065], # [ 0.52143395, 0.58103496], # [ 0.62030756, 0.24758299], # [ 0.59746987, 0.2574137 ], # [ 0.5780933 , 0.2652785 ], # [ 0.8624742 , 0.2089644 ], # [ 0.8855709 , 0.20027623]], dtype=np.float32) # mesh_33 = np.arange(468) # mask = np.ones(len(mesh_33), dtype=bool) # mask[landmarks_468_moving_parts_indexes]=False # mesh_33=mesh_33[mask,...] # landmarks_2D_new_mesh=uni_landmarks_468[mask,...] # mouth_center_landmarks_2D = np.array([ # [-4.4202591e-07, 4.4916576e-01], # 48 # [1.8399176e-01, 3.7537053e-01], # 49 # [3.7018123e-01, 3.3719531e-01], # 50 # [5.0000089e-01, 3.6938059e-01], # 51 # [6.2981832e-01, 3.3719531e-01], # 52 # [8.1600773e-01, 3.7537053e-01], # 53 # [1.0000000e+00, 4.4916576e-01], # 54 # [8.2213330e-01, 6.2836081e-01], # 55 # [6.4110327e-01, 7.0757812e-01], # 56 # [5.0000089e-01, 7.2259867e-01], # 57 # [3.5889623e-01, 7.0757812e-01], # 58 # [1.7786618e-01, 6.2836081e-01], # 59 # [7.6765373e-02, 4.5882553e-01], # 60 # [3.6856663e-01, 4.4601500e-01], # 61 # [5.0000089e-01, 4.5999300e-01], # 62 # [6.3143289e-01, 4.4601500e-01], # 63 # [9.2323411e-01, 4.5882553e-01], # 64 # [6.3399029e-01, 5.4228687e-01], # 65 # [5.0000089e-01, 5.5843467e-01], # 66 # [3.6601129e-01, 5.4228687e-01] # 67 # ], dtype=np.float32) # 68 point landmark definitions landmarks_68_pt = {"mouth": (48, 68), "right_eyebrow": (17, 22), "left_eyebrow": (22, 27), "right_eye": (36, 42), "left_eye": (42, 48), "nose": (27, 36), # missed one point "jaw": (0, 17)} landmarks_68_3D = np.array([ [-73.393523, -29.801432, 47.667532], # 00 [-72.775014, -10.949766, 45.909403], # 01 [-70.533638, 7.929818, 44.842580], # 02 [-66.850058, 26.074280, 43.141114], # 03 [-59.790187, 42.564390, 38.635298], # 04 [-48.368973, 56.481080, 30.750622], # 05 [-34.121101, 67.246992, 18.456453], # 06 [-17.875411, 75.056892, 3.609035], # 07 [0.098749, 77.061286, -0.881698], # 08 [17.477031, 74.758448, 5.181201], # 09 [32.648966, 66.929021, 19.176563], # 10 [46.372358, 56.311389, 30.770570], # 11 [57.343480, 42.419126, 37.628629], # 12 [64.388482, 25.455880, 40.886309], # 13 [68.212038, 6.990805, 42.281449], # 14 [70.486405, -11.666193, 44.142567], # 15 [71.375822, -30.365191, 47.140426], # 16 [-61.119406, -49.361602, 14.254422], # 17 [-51.287588, -58.769795, 7.268147], # 18 [-37.804800, -61.996155, 0.442051], # 19 [-24.022754, -61.033399, -6.606501], # 20 [-11.635713, -56.686759, -11.967398], # 21 [12.056636, -57.391033, -12.051204], # 22 [25.106256, -61.902186, -7.315098], # 23 [38.338588, -62.777713, -1.022953], # 24 [51.191007, -59.302347, 5.349435], # 25 [60.053851, -50.190255, 11.615746], # 26 [0.653940, -42.193790, -13.380835], # 27 [0.804809, -30.993721, -21.150853], # 28 [0.992204, -19.944596, -29.284036], # 29 [1.226783, -8.414541, -36.948060], # 00 [-14.772472, 2.598255, -20.132003], # 01 [-7.180239, 4.751589, -23.536684], # 02 [0.555920, 6.562900, -25.944448], # 03 [8.272499, 4.661005, -23.695741], # 04 [15.214351, 2.643046, -20.858157], # 05 [-46.047290, -37.471411, 7.037989], # 06 [-37.674688, -42.730510, 3.021217], # 07 [-27.883856, -42.711517, 1.353629], # 08 [-19.648268, -36.754742, -0.111088], # 09 [-28.272965, -35.134493, -0.147273], # 10 [-38.082418, -34.919043, 1.476612], # 11 [19.265868, -37.032306, -0.665746], # 12 [27.894191, -43.342445, 0.247660], # 13 [37.437529, -43.110822, 1.696435], # 14 [45.170805, -38.086515, 4.894163], # 15 [38.196454, -35.532024, 0.282961], # 16 [28.764989, -35.484289, -1.172675], # 17 [-28.916267, 28.612716, -2.240310], # 18 [-17.533194, 22.172187, -15.934335], # 19 [-6.684590, 19.029051, -22.611355], # 20 [0.381001, 20.721118, -23.748437], # 21 [8.375443, 19.035460, -22.721995], # 22 [18.876618, 22.394109, -15.610679], # 23 [28.794412, 28.079924, -3.217393], # 24 [19.057574, 36.298248, -14.987997], # 25 [8.956375, 39.634575, -22.554245], # 26 [0.381549, 40.395647, -23.591626], # 27 [-7.428895, 39.836405, -22.406106], # 28 [-18.160634, 36.677899, -15.121907], # 29 [-24.377490, 28.677771, -4.785684], # 30 [-6.897633, 25.475976, -20.893742], # 31 [0.340663, 26.014269, -22.220479], # 32 [8.444722, 25.326198, -21.025520], # 33 [24.474473, 28.323008, -5.712776], # 34 [8.449166, 30.596216, -20.671489], # 35 [0.205322, 31.408738, -21.903670], # 36 [-7.198266, 30.844876, -20.328022] # 37 ], dtype=np.float32) FaceType_to_padding_remove_align = { FaceType.HALF: (0.0, False), FaceType.MID_FULL: (0.0675, False), FaceType.FULL: (0.2109375, False), FaceType.FULL_NO_ALIGN: (0.2109375, True), FaceType.WHOLE_FACE: (0.40, False), FaceType.HEAD: (0.70, False), FaceType.HEAD_NO_ALIGN: (0.70, True), } def convert_98_to_68(lmrks): # jaw result = [lmrks[0]] for i in range(2, 16, 2): result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2] result += [lmrks[16]] for i in range(18, 32, 2): result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2] result += [lmrks[32]] # eyebrows averaging result += [lmrks[33], (lmrks[34] + lmrks[41]) / 2, (lmrks[35] + lmrks[40]) / 2, (lmrks[36] + lmrks[39]) / 2, (lmrks[37] + lmrks[38]) / 2, ] result += [(lmrks[42] + lmrks[50]) / 2, (lmrks[43] + lmrks[49]) / 2, (lmrks[44] + lmrks[48]) / 2, (lmrks[45] + lmrks[47]) / 2, lmrks[46] ] # nose result += list(lmrks[51:60]) # left eye (from our view) result += [lmrks[60], lmrks[61], lmrks[63], lmrks[64], lmrks[65], lmrks[67]] # right eye result += [lmrks[68], lmrks[69], lmrks[71], lmrks[72], lmrks[73], lmrks[75]] # mouth result += list(lmrks[76:96]) return np.concatenate(result).reshape((68, 2)) def transform_points(points, mat, invert=False): if invert: mat = cv2.invertAffineTransform(mat) points = np.expand_dims(points, axis=1) points = cv2.transform(points, mat, points.shape) points = np.squeeze(points) return points def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array(image_landmarks) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama(np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]), landmarks_2D_new, True)[0:2] # get corner points in global space g_p = transform_points(np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True) g_c = g_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value # print(face_type) padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5)) if face_type == FaceType.WHOLE_FACE: # adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead vec = (g_p[0] - g_p[3]).astype(np.float32) vec_len = npla.norm(vec) vec /= vec_len g_c += vec * vec_len * 0.07 # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array([g_c - tb_diag_vec * mod, g_c + bt_diag_vec * mod, g_c + tb_diag_vec * mod]) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array([g_c - tb_diag_vec * mod, g_c + bt_diag_vec * mod, g_c + tb_diag_vec * mod, g_c - bt_diag_vec * mod, ]) # get area of face square in global space area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1]) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array([g_c + [-side, -side], g_c + [side, -side], g_c + [side, side]]) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size))) mat = cv2.getAffineTransform(l_t, pts2) return mat def get_rect_from_landmarks(image_landmarks): mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN) g_p = transform_points(np.float32([(0, 0), (255, 255)]), mat, True) (l, t, r, b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1] return (l, t, r, b) def get_transform_mat_all(image_landmarks,uni_landmarks,output_size,scale=1,gcx=-0.02,gcy=0.15,face_type=FaceType.WHOLE_FACE): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama(image_landmarks, uni_landmarks, True)[0:2] # get corner points in global space g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True) g_c = g_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5)) vec = (g_p[0]-g_p[3]).astype(np.float32) vec_len = npla.norm(vec) vec /= vec_len g_c += vec*vec_len*[gcx,gcy] # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array([g_c - tb_diag_vec * mod, g_c + bt_diag_vec * mod, g_c + tb_diag_vec * mod]) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array([g_c - tb_diag_vec * mod, g_c + bt_diag_vec * mod, g_c + tb_diag_vec * mod, g_c - bt_diag_vec * mod, ]) # get area of face square in global space area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1]) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array([g_c + [-side, -side], g_c + [side, -side], g_c + [side, side]]) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size))) mat = cv2.getAffineTransform(l_t, pts2) return mat def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0): if len(lmrks) != 68: raise Exception('works only with 68 landmarks') lmrks = np.array(lmrks.copy(), dtype=np.int) # #nose ml_pnt = (lmrks[36] + lmrks[0]) // 2 mr_pnt = (lmrks[16] + lmrks[45]) // 2 # mid points between the mid points and eye ql_pnt = (lmrks[36] + ml_pnt) // 2 qr_pnt = (lmrks[45] + mr_pnt) // 2 # Top of the eye arrays bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39])) bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt)) # Eyebrow arrays top_l = lmrks[17:22] top_r = lmrks[22:27] # Adjust eyebrow arrays lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l) lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r) return lmrks def get_image_hull_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0): hull_mask = np.zeros(image_shape[0:2] + (1,), dtype=np.float32) lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) r_jaw = (lmrks[0:9], lmrks[17:18]) l_jaw = (lmrks[8:17], lmrks[26:27]) r_cheek = (lmrks[17:20], lmrks[8:9]) l_cheek = (lmrks[24:27], lmrks[8:9]) nose_ridge = (lmrks[19:25], lmrks[8:9],) r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9]) l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9]) nose = (lmrks[27:31], lmrks[31:36]) parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose] for item in parts: merged = np.concatenate(item) cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,)) return hull_mask def get_image_eye_mask(image_shape, image_landmarks): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') h, w, c = image_shape hull_mask = np.zeros((h, w, 1), dtype=np.float32) image_landmarks = image_landmarks.astype(np.int) cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[36:42]), (1,)) cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[42:48]), (1,)) dilate = h // 32 hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1) blur = h // 16 blur = blur + (1 - blur % 2) hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0) hull_mask = hull_mask[..., None] return hull_mask def get_image_mouth_mask(image_shape, image_landmarks): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') h, w, c = image_shape hull_mask = np.zeros((h, w, 1), dtype=np.float32) image_landmarks = image_landmarks.astype(np.int) cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[60:]), (1,)) dilate = h // 32 hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1) blur = h // 16 blur = blur + (1 - blur % 2) hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0) hull_mask = hull_mask[..., None] return hull_mask def alpha_to_color(img_alpha, color): if len(img_alpha.shape) == 2: img_alpha = img_alpha[..., None] h, w, c = img_alpha.shape result = np.zeros((h, w, len(color)), dtype=np.float32) result[:, :] = color return result * img_alpha def get_cmask(image_shape, lmrks, eyebrows_expand_mod=1.0): h, w, c = image_shape hull = get_image_hull_mask(image_shape, lmrks, eyebrows_expand_mod) result = np.zeros((h, w, 3), dtype=np.float32) def process(w, h, data): d = {} cur_lc = 0 all_lines = [] for s, pts_loop_ar in data: lines = [] for pts, loop in pts_loop_ar: pts_len = len(pts) lines.append([[pts[i], pts[(i + 1) % pts_len]] for i in range(pts_len - (0 if loop else 1))]) lines = np.concatenate(lines) lc = lines.shape[0] all_lines.append(lines) d[s] = cur_lc, cur_lc + lc cur_lc += lc all_lines = np.concatenate(all_lines, 0) # calculate signed distance for all points and lines line_count = all_lines.shape[0] pts_count = w * h all_lines = np.repeat(all_lines[None, ...], pts_count, axis=0).reshape((pts_count * line_count, 2, 2)) pts = np.empty((h, w, line_count, 2), dtype=np.float32) pts[..., 1] = np.arange(h)[:, None, None] pts[..., 0] = np.arange(w)[:, None] pts = pts.reshape((h * w * line_count, -1)) a = all_lines[:, 0, :] b = all_lines[:, 1, :] pa = pts - a ba = b - a ph = np.clip(np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1) dists = npla.norm(pa - ba * ph[..., None], axis=1).reshape((h, w, line_count)) def get_dists(name, thickness=0): s, e = d[name] result = dists[..., s:e] if thickness != 0: result = np.abs(result) - thickness return np.min(result, axis=-1) return get_dists l_eye = lmrks[42:48] r_eye = lmrks[36:42] l_brow = lmrks[22:27] r_brow = lmrks[17:22] mouth = lmrks[48:60] up_nose = np.concatenate((lmrks[27:31], lmrks[33:34])) down_nose = lmrks[31:36] nose = np.concatenate((up_nose, down_nose)) gdf = process(w, h, ( ('eyes', ((l_eye, True), (r_eye, True))), ('brows', ((l_brow, False), (r_brow, False))), ('up_nose', ((up_nose, False),)), ('down_nose', ((down_nose, False),)), ('mouth', ((mouth, True),)), ) ) eyes_fall_dist = w // 32 eyes_thickness = max(w // 64, 1) brows_fall_dist = w // 32 brows_thickness = max(w // 256, 1) nose_fall_dist = w / 12 nose_thickness = max(w // 96, 1) mouth_fall_dist = w // 32 mouth_thickness = max(w // 64, 1) eyes_mask = gdf('eyes', eyes_thickness) eyes_mask = 1 - np.clip(eyes_mask / eyes_fall_dist, 0, 1) # eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1) # eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1) brows_mask = gdf('brows', brows_thickness) brows_mask = 1 - np.clip(brows_mask / brows_fall_dist, 0, 1) # brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1) mouth_mask = gdf('mouth', mouth_thickness) mouth_mask = 1 - np.clip(mouth_mask / mouth_fall_dist, 0, 1) # mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1) def blend(a, b, k): x = np.clip(0.5 + 0.5 * (b - a) / k, 0.0, 1.0) return (a - b) * x + b - k * x * (1.0 - x) # nose_mask = (a-b)*x+b - k*x*(1.0-x) # nose_mask = np.minimum (up_nose_mask , down_nose_mask ) # nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1) nose_mask = blend(gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness * 3) nose_mask = 1 - np.clip(nose_mask / nose_fall_dist, 0, 1) up_nose_mask = gdf('up_nose', nose_thickness) up_nose_mask = 1 - np.clip(up_nose_mask / nose_fall_dist, 0, 1) # up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1) down_nose_mask = gdf('down_nose', nose_thickness) down_nose_mask = 1 - np.clip(down_nose_mask / nose_fall_dist, 0, 1) # down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1) # nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 ) # nose_mask /= np.max(nose_mask) # nose_mask = np.maximum (up_nose_mask , down_nose_mask ) # nose_mask = down_nose_mask # nose_mask = np.zeros_like(nose_mask) eyes_mask = eyes_mask * (1 - mouth_mask) nose_mask = nose_mask * (1 - eyes_mask) hull_mask = hull[..., 0].copy() hull_mask = hull_mask * (1 - eyes_mask) * (1 - brows_mask) * (1 - nose_mask) * (1 - mouth_mask) # eyes_mask = eyes_mask * (1-nose_mask) mouth_mask = mouth_mask * (1 - nose_mask) brows_mask = brows_mask * (1 - nose_mask) * (1 - eyes_mask) hull_mask = alpha_to_color(hull_mask, (0, 1, 0)) eyes_mask = alpha_to_color(eyes_mask, (1, 0, 0)) brows_mask = alpha_to_color(brows_mask, (0, 0, 1)) nose_mask = alpha_to_color(nose_mask, (0, 1, 1)) mouth_mask = alpha_to_color(mouth_mask, (0, 0, 1)) # nose_mask = np.maximum( up_nose_mask, down_nose_mask ) result = hull_mask + mouth_mask + nose_mask + brows_mask + eyes_mask result *= hull # result = np.clip (result, 0, 1) return result def blur_image_hull_mask(hull_mask): maxregion = np.argwhere(hull_mask == 1.0) miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx; leny = maxy - miny; masky = int(minx + (lenx // 2)) maskx = int(miny + (leny // 2)) lowest_len = min(lenx, leny) ero = int(lowest_len * 0.085) blur = int(lowest_len * 0.10) hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) hull_mask = cv2.blur(hull_mask, (blur, blur)) hull_mask = np.expand_dims(hull_mask, -1) return hull_mask mirror_idxs = [ [0, 16], [1, 15], [2, 14], [3, 13], [4, 12], [5, 11], [6, 10], [7, 9], [17, 26], [18, 25], [19, 24], [20, 23], [21, 22], [36, 45], [37, 44], [38, 43], [39, 42], [40, 47], [41, 46], [31, 35], [32, 34], [50, 52], [49, 53], [48, 54], [59, 55], [58, 56], [67, 65], [60, 64], [61, 63]] def mirror_landmarks(landmarks, val): result = landmarks.copy() for idx in mirror_idxs: result[idx] = result[idx[::-1]] result[:, 0] = val - result[:, 0] - 1 return result def get_face_struct_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,)): mask = np.zeros(image_shape[0:2] + (len(color),), dtype=np.float32) lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) draw_landmarks(mask, image_landmarks, color=color, draw_circles=False, thickness=2) return mask def draw_landmarks(image, image_landmarks, color=(0, 255, 0), draw_circles=True, thickness=1, transparent_mask=False): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') int_lmrks = np.array(image_landmarks, dtype=np.int) jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])] right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])] left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])] mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])] right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])] left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])] nose = int_lmrks[slice(*landmarks_68_pt["nose"])] # open shapes cv2.polylines(image, tuple(np.array([v]) for v in (right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])))), False, color, thickness=thickness, lineType=cv2.LINE_AA) # closed shapes cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)), True, color, thickness=thickness, lineType=cv2.LINE_AA) if draw_circles: # the rest of the cicles for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0): cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA) # jaw big circles for x, y in jaw: cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA) if transparent_mask: mask = get_image_hull_mask(image.shape, image_landmarks) image[...] = (image * (1 - mask) + image * mask / 2)[...] def draw_rect_landmarks(image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False, landmarks_color=(0, 255, 0)): draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask) imagelib.draw_rect(image, rect, (255, 0, 0), 2) image_to_face_mat = get_transform_mat(image_landmarks, face_size, face_type) points = transform_points([(0, 0), (0, face_size - 1), (face_size - 1, face_size - 1), (face_size - 1, 0)], image_to_face_mat, True) imagelib.draw_polygon(image, points, (0, 0, 255), 2) points = transform_points( [(int(face_size * 0.05), 0), (int(face_size * 0.1), int(face_size * 0.1)), (0, int(face_size * 0.1))], image_to_face_mat, True) imagelib.draw_polygon(image, points, (0, 0, 255), 2) def calc_face_pitch(landmarks): if not isinstance(landmarks, np.ndarray): landmarks = np.array(landmarks) t = ((landmarks[6][1] - landmarks[8][1]) + (landmarks[10][1] - landmarks[8][1])) / 2.0 b = landmarks[8][1] return float(b - t) def estimate_averaged_yaw(landmarks): # Works much better than solvePnP if landmarks from "3DFAN" if not isinstance(landmarks, np.ndarray): landmarks = np.array(landmarks) l = ((landmarks[27][0] - landmarks[0][0]) + (landmarks[28][0] - landmarks[1][0]) + ( landmarks[29][0] - landmarks[2][0])) / 3.0 r = ((landmarks[16][0] - landmarks[27][0]) + (landmarks[15][0] - landmarks[28][0]) + ( landmarks[14][0] - landmarks[29][0])) / 3.0 return float(r - l) def estimate_pitch_yaw_roll(aligned_landmarks, size=256): """ returns pitch,yaw,roll [-pi/2...+pi/2] """ shape = (size, size) focal_length = shape[1] camera_center = (shape[1] / 2, shape[0] / 2) camera_matrix = np.array( [[focal_length, 0, camera_center[0]], [0, focal_length, camera_center[1]], [0, 0, 1]], dtype=np.float32) (_, rotation_vector, _) = cv2.solvePnP( np.concatenate((landmarks_68_3D[:27], landmarks_68_3D[30:36]), axis=0), np.concatenate((aligned_landmarks[:27], aligned_landmarks[30:36]), axis=0).astype(np.float32), camera_matrix, np.zeros((4, 1))) pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles(cv2.Rodrigues(rotation_vector)[0]) half_pi = math.pi / 2.0 pitch = np.clip(pitch, -half_pi, half_pi) yaw = np.clip(yaw, -half_pi, half_pi) roll = np.clip(roll, -half_pi, half_pi) return -pitch, yaw, roll # if remove_align: # bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True) # #import code # #code.interact(local=dict(globals(), **locals())) # area = mathlib.polygon_area(bbox[:,0], bbox[:,1] ) # side = math.sqrt(area) / 2 # center = transform_points ( [(output_size/2,output_size/2)], mat, True) # pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] )) # pts2 = np.float32([[0,0],[output_size,0],[0,output_size]]) # mat = cv2.getAffineTransform(pts1,pts2) # if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN): # #lmrks2 = expand_eyebrows(image_landmarks) # #lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False ) # #y_diff = np.float32( (0,np.min(lmrks2_[:,1])) ) # #y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True) # #y_diff = y_diff[1]-y_diff[0] # # x_diff = np.float32((0,0)) # # lmrks2_ = transform_points( [ image_landmarks[0], image_landmarks[16] ], mat, False ) # if lmrks2_[0,0] < 0: # x_diff = lmrks2_[0,0] # x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True) # x_diff = x_diff[1]-x_diff[0] # elif lmrks2_[1,0] >= output_size: # x_diff = lmrks2_[1,0]-(output_size-1) # x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True) # x_diff = x_diff[1]-x_diff[0] # # mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2) """ def get_averaged_transform_mat (img_landmarks, img_landmarks_prev, img_landmarks_next, average_frame_count, average_center_frame_count, output_size, face_type, scale=1.0): l_c_list = [] tb_diag_vec_list = [] bt_diag_vec_list = [] mod_list = [] count = max(average_frame_count,average_center_frame_count) for i in range ( -count, count+1, 1 ): if i < 0: lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None elif i > 0: lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None else: lmrks = img_landmarks if lmrks is None: continue l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale) if i >= -average_frame_count and i <= average_frame_count: tb_diag_vec_list.append(tb_diag_vec) bt_diag_vec_list.append(bt_diag_vec) mod_list.append(mod) if i >= -average_center_frame_count and i <= average_center_frame_count: l_c_list.append(l_c) tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 ) bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 ) mod = np.mean( np.array(mod_list), axis=0 ) l_c = np.mean( np.array(l_c_list), axis=0 ) return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type) def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) # get face padding value for FaceType padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] # get corner points in global space l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True) l_c = l_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) ) # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ) ] ) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ), np.round( l_c - bt_diag_vec*mod ), ] ) # get area of face square in global space area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array( [ np.round( l_c + [-side,-side] ), np.round( l_c + [ side,-side] ), np.round( l_c + [ side, side] ) ] ) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) return mat """