Upload 117 files

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/cam_demo1.gif filter=lfs diff=lfs merge=lfs -text
+assets/cam_demo2.gif filter=lfs diff=lfs merge=lfs -text
+assets/demo10.gif filter=lfs diff=lfs merge=lfs -text
+assets/demo20.gif filter=lfs diff=lfs merge=lfs -text
+data/source/demo.mp4 filter=lfs diff=lfs merge=lfs -text

HifiFaceAPI_parallel_base.py

@@ -0,0 +1,148 @@
+import os
+import time
+import numpy as np
+
+import numexpr as ne
+# ne.set_num_threads(10)
+
+from multiprocessing.dummy import Process, Queue
+from face_detect.face_align_68 import face_alignment_landmark
+from face_detect.face_detect import FaceDetect
+from face_lib.face_swap import HifiFace
+from face_restore.gfpgan_onnx_api import GFPGAN
+from face_restore.xseg_onnx_api import XSEG
+
+TRACKING_THRESHOLD = 0.15
+
+# def np_norm(x):
+#     return (x - np.average(x)) / np.std(x)
+
+def cosine_vectorized_v3(array1, array2):
+    sumyy = np.einsum('ij,ij->i', array2, array2)
+    sumxx = np.einsum('ij,ij->i', array1, array1)[:, None]
+    sumxy = array1.dot(array2.T)
+    sqrt_sumxx = ne.evaluate('sqrt(sumxx)')
+    sqrt_sumyy = ne.evaluate('sqrt(sumyy)')
+    return ne.evaluate('(sumxy/sqrt_sumxx)/sqrt_sumyy')
+
+
+class Consumer0Base(Process):
+    def __init__(self, opt, frame_queue_in, feature_dst_list=None, queue_list=None, block=True, fps_counter=False):
+        super().__init__()
+        self.queue_list = queue_list
+        self.fps_counter = fps_counter
+        self.block = block
+        self.pid = os.getpid()
+
+        self.opt = opt
+        self.frame_queue_in = frame_queue_in
+        self.feature_dst_list = feature_dst_list
+        self.crop_size = self.opt.input_size
+        self.scrfd_detector = FaceDetect(mode='scrfd_500m', tracking_thres=TRACKING_THRESHOLD)
+        self.face_alignment = face_alignment_landmark(lm_type=68)
+
+        print('init consumer {}, pid is {}.'.format(self.__class__.__name__, self.pid))
+
+
+class Consumer1BaseONNX(Process):
+    def __init__(self, opt, feature_list, queue_list: list, block=True, fps_counter=False,provider='gpu', load_xseg=True, xseg_flag=False):
+        super().__init__()
+        self.queue_list = queue_list
+        self.fps_counter = fps_counter
+        self.block = block
+        self.pid = os.getpid()
+        self.opt = opt
+        self.feature_list = feature_list
+        # self.index_list = index_list
+        # self.apply_gpen = apply_gpen
+        self.crop_size = self.opt.input_size
+        self.xseg_flag = xseg_flag
+
+        print("model_name:", self.opt.model_name)
+        self.hf = HifiFace(model_name='er8_bs1', provider=provider)
+        if load_xseg:
+            self.xseg = XSEG(model_type='xseg_0611', provider=provider)
+
+    def switch_xseg(self):
+        self.xseg_flag = not self.xseg_flag
+
+    def predict(self, src_face_image, dst_face_latent):
+        mask_out, swap_face_out = self.hf.forward(src_face_image, dst_face_latent)
+        if self.xseg_flag:
+            mask_out = self.xseg.forward(swap_face_out)[None,None]
+        return [mask_out, swap_face_out]
+
+
+class Consumer2Base(Process):
+    def __init__(self, queue_list: list, frame_queue_out, block=True, fps_counter=False):
+        super().__init__()
+        self.queue_list = queue_list
+        self.fps_counter = fps_counter
+        self.block = block
+        self.pid = os.getpid()
+        self.frame_queue_out = frame_queue_out
+
+        # from face_restore import FaceRestore
+        # self.fa = FaceRestore(use_gpu=True, mode='gfpgan')  # gfpgan gpen dfdnet
+
+        print('init consumer {}, pid is {}.'.format(self.__class__.__name__, self.pid))
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+
+        while True:
+            something_in = self.queue_list[0].get()
+
+            # exit condition
+            if something_in is None:
+                print('subprocess {} exit !'.format(self.pid))
+                break
+
+            self.forward_func(something_in)
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 4:
+                    print("Consumer2 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+        print('c2 stop')
+        # cv2.destroyAllWindows()
+
+class Consumer3Base(Process):
+    def __init__(self, queue_list, block=True, fps_counter=False, provider='gpu'):
+        super().__init__()
+        self.queue_list = queue_list
+        self.fps_counter = fps_counter
+        self.block = block
+        self.pid = os.getpid()
+
+        self.gfp = GFPGAN(model_type='GFPGANv1.4', provider=provider)
+
+        print('init consumer {}, pid is {}.'.format(self.__class__.__name__, self.pid))
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+
+        while True:
+            something_in = self.queue_list[0].get()
+
+            if something_in is None:
+                print('subprocess {} exit !'.format(self.pid))
+                self.queue_list[1].put(None)
+                break
+
+            self.forward_func(something_in)
+
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 4:
+                    print("Consumer3 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+
+        print('c3 stop')
+

HifiFaceAPI_parallel_trt_roi_realtime_api.py

@@ -0,0 +1,189 @@
+import os
+import cv2
+import time
+import numpy as np
+import numexpr as ne
+from multiprocessing.dummy import Process, Queue
+from options.hifi_test_options import HifiTestOptions
+from HifiFaceAPI_parallel_base import Consumer0Base, Consumer2Base, Consumer1BaseONNX
+
+
+def np_norm(x):
+    return (x - np.average(x)) / np.std(x)
+
+
+def reverse2wholeimage_hifi_trt_roi(swaped_img, mat_rev, img_mask, frame, roi_img, roi_box):
+    target_image = cv2.warpAffine(swaped_img, mat_rev, roi_img.shape[:2][::-1], borderMode=cv2.BORDER_REPLICATE)[
+                   ...,
+                   ::-1]
+
+    local_dict = {
+        'img_mask': img_mask,
+        'target_image': target_image,
+        'roi_img': roi_img,
+    }
+    img = ne.evaluate('img_mask * (target_image * 255)+(1 - img_mask) * roi_img', local_dict=local_dict,
+                      global_dict=None)
+    img = img.astype(np.uint8)
+    frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]] = img
+    return frame
+
+
+def get_max_face(np_rois):
+    roi_areas = []
+    for index in range(np_rois.shape[0]):
+        roi_areas.append((np_rois[index, 2] - np_rois[index, 0]) * (np_rois[index, 3] - np_rois[index, 1]))
+    return np.argmax(np.array(roi_areas))
+
+
+class Consumer0(Consumer0Base):
+    def __init__(self, opt, frame_queue_in, queue_list: list, block=True, fps_counter=False):
+        super().__init__(opt, frame_queue_in, None, queue_list, block, fps_counter)
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+        kpss_old = None
+        rois_old = faces_old = Ms_old = masks_old = None
+
+        while True:
+            frame = self.frame_queue_in.get()
+            if frame is None:
+                break
+            try:
+                _, bboxes, kpss = self.scrfd_detector.get_bboxes(frame, max_num=0)
+                rois, faces, Ms, masks = self.face_alignment.forward(
+                    frame, bboxes, kpss, limit=5, min_face_size=30,
+                    crop_size=(self.crop_size, self.crop_size), apply_roi=True
+                )
+
+            except (TypeError, IndexError, ValueError) as e:
+                self.queue_list[0].put([None, frame])
+                continue
+
+            if len(faces)==0:
+                self.queue_list[0].put([None, frame])
+                continue
+            elif len(faces)==1:
+                face = np.array(faces[0])
+                mat = Ms[0]
+                roi_box = rois[0]
+            else:
+                max_index = get_max_face(np.array(rois))
+                face = np.array(faces[max_index])
+                mat = Ms[max_index]
+                roi_box = rois[max_index]
+            roi_img = frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]]
+
+           # "The default normalization to the range of -1 to 1, where the model input is in RGB format
+            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
+
+            self.queue_list[0].put([face, mat, [], frame, roi_img, roi_box])
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 10:
+                    print("Consumer0 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+        self.queue_list[0].put(None)
+        print('co stop')
+
+
+class Consumer1(Consumer1BaseONNX):
+    def __init__(self, opt, feature_list, queue_list: list, block=True, fps_counter=False):
+        super().__init__(opt, feature_list, queue_list, block, fps_counter)
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+
+        while True:
+            something_in = self.queue_list[0].get()
+            if something_in is None:
+                break
+            elif len(something_in) == 2:
+                self.queue_list[1].put([None, something_in[1]])
+                continue
+
+
+            if len(self.feature_list) > 1:
+                self.feature_list.pop(0)
+
+            image_latent = self.feature_list[0][0]
+
+            mask_out, swap_face_out = self.predict(something_in[0], image_latent[0].reshape(1, -1))
+
+            mask = cv2.warpAffine(mask_out[0][0].astype(np.float32), something_in[1],
+                                  something_in[4].shape[:2][::-1])
+            mask[mask > 0.2] = 1
+            mask = mask[:, :, np.newaxis].astype(np.uint8)
+            swap_face = swap_face_out[0].transpose((1, 2, 0)).astype(np.float32)
+
+            self.queue_list[1].put(
+                [swap_face, something_in[1], mask, something_in[3], something_in[4], something_in[5]])
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 10:
+                    print("Consumer1 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+        self.queue_list[1].put(None)
+        print('c1 stop')
+
+
+class Consumer2(Consumer2Base):
+    def __init__(self, queue_list: list, frame_queue_out, block=True, fps_counter=False):
+        super().__init__(queue_list, frame_queue_out, block, fps_counter)
+        self.face_detect_flag = True
+
+    def forward_func(self, something_in):
+
+        # do your work here.
+        if len(something_in) == 2:
+            self.face_detect_flag = False
+            frame = something_in[1]
+            frame_out = frame.astype(np.uint8)
+        else:
+            self.face_detect_flag = True
+            # swap_face = something_in[0]
+            swap_face = ((something_in[0] + 1) / 2)
+            frame_out = reverse2wholeimage_hifi_trt_roi(
+                swap_face, something_in[1], something_in[2],
+                something_in[3], something_in[4], something_in[5]
+            )
+        self.frame_queue_out.put([frame_out, self.face_detect_flag])
+        # cv2.imshow('output', frame_out)
+        # cv2.waitKey(1)
+
+
+class HifiFaceRealTime:
+
+    def __init__(self, feature_dict_list_, frame_queue_in, frame_queue_out, gpu=True, model_name=''):
+        self.opt = HifiTestOptions().parse()
+        if model_name != '':
+            self.opt.model_name = model_name
+        self.opt.input_size = 256
+        self.feature_dict_list = feature_dict_list_
+        self.frame_queue_in = frame_queue_in
+        self.frame_queue_out = frame_queue_out
+
+        self.gpu = gpu
+
+    def forward(self):
+        self.q0 = Queue(2)
+        self.q1 = Queue(2)
+
+        self.c0 = Consumer0(self.opt, self.frame_queue_in, [self.q0], fps_counter=False)
+        self.c1 = Consumer1(self.opt, self.feature_dict_list, [self.q0, self.q1], fps_counter=False)
+        self.c2 = Consumer2([self.q1], self.frame_queue_out, fps_counter=False)
+
+        self.c0.start()
+        self.c1.start()
+        self.c2.start()
+
+        self.c0.join()
+        self.c1.join()
+        self.c2.join()
+        return

HifiFaceAPI_parallel_trt_roi_realtime_sr_api.py

@@ -0,0 +1,234 @@
+import os
+import cv2
+import time
+import numpy as np
+import numexpr as ne
+from multiprocessing.dummy import Process, Queue
+from options.hifi_test_options import HifiTestOptions
+from HifiFaceAPI_parallel_base import Consumer0Base, Consumer2Base, Consumer3Base,Consumer1BaseONNX
+from color_transfer import color_transfer
+
+
+def np_norm(x):
+    return (x - np.average(x)) / np.std(x)
+
+
+def reverse2wholeimage_hifi_trt_roi(swaped_img, mat_rev, img_mask, frame, roi_img, roi_box):
+    target_image = cv2.warpAffine(swaped_img, mat_rev, roi_img.shape[:2][::-1], borderMode=cv2.BORDER_REPLICATE)[
+                   ...,
+                   ::-1]
+    local_dict = {
+        'img_mask': img_mask,
+        'target_image': target_image,
+        'roi_img': roi_img,
+    }
+    img = ne.evaluate('img_mask * (target_image * 255)+(1 - img_mask) * roi_img', local_dict=local_dict,
+                      global_dict=None)
+    img = img.astype(np.uint8)
+    frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]] = img
+    return frame
+
+
+def get_max_face(np_rois):
+    roi_areas = []
+    for index in range(np_rois.shape[0]):
+        roi_areas.append((np_rois[index, 2] - np_rois[index, 0]) * (np_rois[index, 3] - np_rois[index, 1]))
+    return np.argmax(np.array(roi_areas))
+
+class Consumer0(Consumer0Base):
+    def __init__(self, opt, frame_queue_in, queue_list: list, block=True, fps_counter=False, align_method='68'):
+        super().__init__(opt, frame_queue_in, None, queue_list, block, fps_counter)
+        self.align_method = align_method
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+        kpss_old = None
+        rois_old = faces_old = Ms_old = masks_old = None
+
+        while True:
+            frame = self.frame_queue_in.get()
+            if frame is None:
+                break
+            try:
+                _, bboxes, kpss = self.scrfd_detector.get_bboxes(frame, max_num=0)
+                if self.align_method == '5class':
+                    rois, faces, Ms, masks = self.mtcnn_detector.align_multi_for_scrfd(
+                        frame, bboxes, kpss, limit=1, min_face_size=30,
+                        crop_size=(self.crop_size, self.crop_size), apply_roi=True, detector=None
+                    )
+                else:
+                    rois, faces, Ms, masks = self.face_alignment.forward(
+                        frame, bboxes, kpss, limit=5, min_face_size=30,
+                        crop_size=(self.crop_size, self.crop_size), apply_roi=True
+                    )
+
+            except (TypeError, IndexError, ValueError) as e:
+                self.queue_list[0].put([None, frame])
+                continue
+
+            if len(faces)==0:
+                self.queue_list[0].put([None, frame])
+                continue
+            elif len(faces)==1:
+                face = np.array(faces[0])
+                mat = Ms[0]
+                roi_box = rois[0]
+            else:
+                max_index = get_max_face(np.array(rois))
+                face = np.array(faces[max_index])
+                mat = Ms[max_index]
+                roi_box = rois[max_index]
+            roi_img = frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]]
+
+            #The default normalization to the range of -1 to 1, where the model input is in RGB format
+            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
+
+            self.queue_list[0].put([face, mat, [], frame, roi_img, roi_box])
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 10:
+                    print("Consumer0 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+        self.queue_list[0].put(None)
+        print('co stop')
+
+
+class Consumer1(Consumer1BaseONNX):
+    def __init__(self, opt, feature_list, queue_list: list, block=True, fps_counter=False):
+        super().__init__(opt, feature_list, queue_list, block, fps_counter)
+
+    def run(self):
+        counter = 0
+        start_time = time.time()
+
+        while True:
+            something_in = self.queue_list[0].get()
+            if something_in is None:
+                break
+            elif len(something_in) == 2:
+                self.queue_list[1].put([None, something_in[1]])
+                continue
+
+            if len(self.feature_list) > 1:
+                self.feature_list.pop(0)
+
+            image_latent = self.feature_list[0][0]
+
+            mask_out, swap_face_out = self.predict(something_in[0], image_latent[0].reshape(1, -1))
+
+            mask = cv2.warpAffine(mask_out[0][0].astype(np.float32), something_in[1],
+                                  something_in[4].shape[:2][::-1])
+            mask[mask > 0.2] = 1
+            mask = mask[:, :, np.newaxis].astype(np.uint8)
+            swap_face = swap_face_out[0].transpose((1, 2, 0)).astype(np.float32)
+
+            self.queue_list[1].put(
+                [swap_face, something_in[1], mask, something_in[3], something_in[4], something_in[5], something_in[0]])
+
+            if self.fps_counter:
+                counter += 1
+                if (time.time() - start_time) > 10:
+                    print("Consumer1 FPS: {}".format(counter / (time.time() - start_time)))
+                    counter = 0
+                    start_time = time.time()
+        self.queue_list[1].put(None)
+        print('c1 stop')
+
+
+class Consumer2(Consumer2Base):
+    def __init__(self, queue_list: list, frame_queue_out, block=True, fps_counter=False):
+        super().__init__(queue_list, frame_queue_out, block, fps_counter)
+
+    def forward_func(self, something_in):
+        if len(something_in) == 2:
+            frame = something_in[1]
+            frame_out = frame.astype(np.uint8)
+        else:
+            swap_face = ((something_in[0] + 1) / 2)
+            frame_out = reverse2wholeimage_hifi_trt_roi(
+                swap_face, something_in[1], something_in[2],
+                something_in[3], something_in[4], something_in[5]
+            )
+        self.frame_queue_out.put(frame_out)
+        # cv2.imshow('output', frame_out)
+        # cv2.waitKey(1)
+
+class Consumer3(Consumer3Base):
+    def __init__(self, queue_list, block=True, fps_counter=False, use_gfpgan=True, sr_weight=1.0,
+                 use_color_trans=False, color_trans_mode=''):
+        super().__init__(queue_list, block, fps_counter)
+        self.use_gfpgan = use_gfpgan
+        self.sr_weight = sr_weight
+        self.use_color_trans = use_color_trans
+        self.color_trans_mode = color_trans_mode
+
+    def forward_func(self, something_in):
+        if len(something_in) == 2:
+            self.queue_list[1].put([None, something_in[1]])
+        else:
+            swap_face = something_in[0]
+            target_face = (something_in[6] / 255).astype(np.float32)
+            if self.use_gfpgan:
+                sr_face = self.gfp.forward(swap_face)
+                if self.sr_weight != 1.0:
+                    sr_face = cv2.addWeighted(sr_face, alpha=self.sr_weight, src2=swap_face, beta=1.0 - self.sr_weight, gamma=0, dtype=cv2.CV_32F)
+                if self.use_color_trans:
+                    transed_face = color_transfer(self.color_trans_mode, (sr_face + 1) / 2, target_face)
+                    result_face = (transed_face * 2) - 1
+                else:
+                    result_face = sr_face
+            else:
+                if self.use_color_trans:
+                    transed_face = color_transfer(self.color_trans_mode, (swap_face + 1) / 2, target_face)
+                    result_face = (transed_face * 2) - 1
+                else:
+                    result_face = swap_face
+            self.queue_list[1].put([result_face, something_in[1], something_in[2], something_in[3],
+                                        something_in[4], something_in[5]])
+
+
+class HifiFaceRealTime:
+
+    def __init__(self, feature_dict_list_, frame_queue_in, frame_queue_out, gpu=True, model_name='er8_bs1', align_method='68',
+                 use_gfpgan=True, sr_weight=1.0, use_color_trans=False, color_trans_mode='rct'):
+        self.opt = HifiTestOptions().parse()
+        if model_name != '':
+            self.opt.model_name = model_name
+        self.opt.input_size = 256
+        self.feature_dict_list = feature_dict_list_
+        self.frame_queue_in = frame_queue_in
+        self.frame_queue_out = frame_queue_out
+
+        self.gpu = gpu
+        self.align_method = align_method
+        self.use_gfpgan = use_gfpgan
+        self.sr_weight = sr_weight
+        self.use_color_trans = use_color_trans
+        self.color_trans_mode = color_trans_mode
+
+
+    def forward(self):
+        self.q0 = Queue(2)
+        self.q1 = Queue(2)
+        self.q2 = Queue(2)
+
+        self.c0 = Consumer0(self.opt, self.frame_queue_in, [self.q0], fps_counter=False, align_method=self.align_method)
+        self.c1 = Consumer1(self.opt, self.feature_dict_list, [self.q0, self.q1], fps_counter=False)
+        self.c3 = Consumer3([self.q1, self.q2], fps_counter=False,
+                            use_gfpgan=self.use_gfpgan, sr_weight=self.sr_weight,
+                            use_color_trans=self.use_color_trans, color_trans_mode=self.color_trans_mode)
+        self.c2 = Consumer2([self.q2], self.frame_queue_out, fps_counter=False)
+
+        self.c0.start()
+        self.c1.start()
+        self.c3.start()
+        self.c2.start()
+
+        self.c0.join()
+        self.c1.join()
+        self.c3.join()
+        self.c2.join()
+        return

LICENSE

@@ -0,0 +1,32 @@
+Silicon Intelligence COMMUNITY LICENSE AGREEMENT
+
+“Agreement” means the terms and conditions for use, reproduction, distribution and modification of this product forth herein.
+
+“Documentation” means the specifications, manuals and documentation by Silicon Intelligence.
+
+“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
+
+“Silicon Intelligence Materials” means, collectively, Silicon Intelligence’s proprietary code and Documentation (and any portion thereof) made available under this Agreement.
+
+By clicking “I Accept” below or by using or distributing any portion or element of the Silicon Intelligence Materials, you agree to be bound by this Agreement.
+
+1. License Rights and Redistribution.
+
+	a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under ’s intellectual property or other rights owned by Silicon Intelligence embodied in the SILICON INTELLIGENCE Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Silicon Intelligence Materials.
+	b. Redistribution and Use.
+		i. If you distribute or make available the Silicon Intelligence Materials (or any derivative works thereof), or a product or service that uses any of them, you shall (A) provide a copy of this Agreement with any such Silicon Intelligence Materials; and (B) prominently display “Built with Silicon Intelligence” on a related website, user interface, blogpost, about page, or product documentation. If you use the Silicon Intelligence Materials to create, train, fine tune, or otherwise improve an AI model, which is distributed or made available, you shall also include “Silicon Intelligence” at the beginning of any such AI model name.
+		ii. If you receive Silicon Intelligence Materials, or any derivative works thereof, from a Licensee as part of an integrated end user product, then Section 2 of this Agreement will not apply to you.
+		iii. You must retain in all copies of the Silicon Intelligence Materials that you distribute the following attribution notice within a “Notice” text file distributed as a part of such copies: “Silicon Intelligence is licensed under the Silicon Intelligence Community License, Copyright © Silicon Intelligence Platforms, Inc. All Rights Reserved.”
+		iv. Your use of the Silicon Intelligence Materials must comply with applicable laws and regulations (including trade compliance laws and regulations) .
+
+2. Additional Commercial Terms. If, on the Silicon Intelligence duix.ai version release date, the monthly active users of the products or services made available by or for Licensee, or Licensee’s affiliates, is greater than 1 thousand monthly active users in the preceding calendar month,  or your product based Silicon Intelligence material your active users greater 1 thousand, you must request a license from Silicon Intelligence, which Silicon Intelligence may grant to you in its sole discretion, and you are not authorized to exercise any of the rights under this Agreement unless or until Silicon Intelligence otherwise expressly grants you such rights.
+
+3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SILICON INTELLIGENCE MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND SILICON INTELLIGENCE DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SILICON INTELLIGENCE MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SILICON INTELLIGENCE MATERIALS AND ANY OUTPUT AND RESULTS.
+
+4. Limitation of Liability. IN NO EVENT WILL SILICON INTELLIGENCE OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF SILICON INTELLIGENCE OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+
+5. Intellectual Property.
+	a. No trademark licenses are granted under this Agreement, and in connection with the Silicon Intelligence Materials, neither Silicon Intelligence nor Licensee may use any name or mark owned by or associated with the other or any of its affiliates, except as required for reasonable and customary use in describing and redistributing the Silicon Intelligence Materials or as set forth in this Section 5(a). Silicon Intelligence hereby grants you a license to use “Silicon Intelligence”  solely as required to comply with the last sentence of Section 1.b.i. You will comply with Silicon Intelligence’s brand guidelines . All goodwill arising out of your use of the Mark will inure to the benefit of Silicon Intelligence.
+	b. If you institute litigation or other proceedings against Silicon Intelligenceor any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Silicon Intelligence Materials or  outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Silicon Intelligence from and against any claim by any third party arising out of or related to your use or distribution of the Silicon Intelligence Materials.
+
+6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Silicon Intelligence Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Silicon Intelligence may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the Silicon Intelligence Materials. Sections 3, 4  shall survive the termination of this Agreement.

app.py

@@ -0,0 +1,175 @@
+import gradio as gr
+import cv2
+import os
+import numpy as np
+import numexpr as ne
+from concurrent.futures import ThreadPoolExecutor
+
+from face_feature.hifi_image_api import HifiImage
+from HifiFaceAPI_parallel_trt_roi_realtime_sr_api import HifiFaceRealTime
+from face_lib.face_swap import HifiFace
+from face_restore.gfpgan_onnx_api import GFPGAN
+from face_restore.xseg_onnx_api import XSEG
+from face_detect.face_align_68 import face_alignment_landmark
+from face_detect.face_detect import FaceDetect
+from options.hifi_test_options import HifiTestOptions
+from color_transfer import color_transfer
+
+opt = HifiTestOptions().parse()
+processor = None
+
+def initialize_processor():
+    global processor
+    if processor is None:
+        processor = FaceSwapProcessor(crop_size=opt.input_size)
+
+class FaceSwapProcessor:
+    def __init__(self, crop_size=256):
+        self.hi = HifiImage(crop_size=crop_size)
+        self.xseg = XSEG(model_type='xseg_0611', provider='gpu')
+        self.hf = HifiFace(model_name='er8_bs1', provider='gpu')
+        self.scrfd_detector = FaceDetect(mode='scrfd_500m', tracking_thres=0.15)
+        self.face_alignment = face_alignment_landmark(lm_type=68)
+        self.gfp = GFPGAN(model_type='GFPGANv1.4', provider='gpu')
+        self.crop_size = crop_size
+
+    def reverse2wholeimage_hifi_trt_roi(self, swaped_img, mat_rev, img_mask, frame, roi_img, roi_box):
+        target_image = cv2.warpAffine(swaped_img, mat_rev, roi_img.shape[:2][::-1], borderMode=cv2.BORDER_REPLICATE)[
+                       ...,
+                       ::-1]
+        local_dict = {
+            'img_mask': img_mask,
+            'target_image': target_image,
+            'roi_img': roi_img,
+        }
+        img = ne.evaluate('img_mask * (target_image * 255)+(1 - img_mask) * roi_img', local_dict=local_dict,
+                          global_dict=None)
+        img = img.astype(np.uint8)
+        frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]] = img
+        return frame
+
+    def process_frame(self, frame, image_latent, use_gfpgan, sr_weight, use_color_trans, color_trans_mode):
+        _, bboxes, kpss = self.scrfd_detector.get_bboxes(frame, max_num=0)
+        rois, faces, Ms, masks = self.face_alignment.forward(
+            frame, bboxes, kpss, limit=5, min_face_size=30,
+            crop_size=(self.crop_size, self.crop_size), apply_roi=True
+        )
+
+        if len(faces) == 0:
+            return frame
+        elif len(faces) == 1:
+            face = np.array(faces[0])
+            mat = Ms[0]
+            roi_box = rois[0]
+        else:
+            max_index = np.argmax([roi[2] * roi[3] for roi in rois])  # Get the largest face
+            face = np.array(faces[max_index])
+            mat = Ms[max_index]
+            roi_box = rois[max_index]
+
+        roi_img = frame[roi_box[1]:roi_box[3], roi_box[0]:roi_box[2]]
+        face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
+
+        mask_out, swap_face_out = self.hf.forward(face, image_latent[0].reshape(1, -1))
+        mask_out = self.xseg.forward(swap_face_out)[None, None]
+
+        mask = cv2.warpAffine(mask_out[0][0].astype(np.float32), mat, roi_img.shape[:2][::-1])
+        mask[mask > 0.2] = 1
+        mask = mask[:, :, np.newaxis].astype(np.uint8)
+        swap_face = swap_face_out[0].transpose((1, 2, 0)).astype(np.float32)
+        target_face = (face.copy() / 255).astype(np.float32)
+
+        if use_gfpgan:
+            sr_face = self.gfp.forward(swap_face)
+            if sr_weight != 1.0:
+                sr_face = cv2.addWeighted(sr_face, sr_weight, swap_face, 1.0 - sr_weight, 0)
+            if use_color_trans:
+                transed_face = color_transfer(color_trans_mode, (sr_face + 1) / 2, target_face)
+                swap_face = (transed_face * 2) - 1
+            else:
+                swap_face = sr_face
+        elif use_color_trans:
+            transed_face = color_transfer(color_trans_mode, (swap_face + 1) / 2, target_face)
+            swap_face = (transed_face * 2) - 1
+
+        swap_face = ((swap_face + 1) / 2)
+
+        frame_out = self.reverse2wholeimage_hifi_trt_roi(
+            swap_face, mat, mask,
+            frame, roi_img, roi_box
+        )
+
+        return frame_out
+
+def process_image_video(image, video_path, use_gfpgan, sr_weight, use_color_trans, color_trans_mode):
+    global processor
+    initialize_processor()
+
+    src_latent, _ = processor.hi.get_face_feature(image)
+    image_latent = [src_latent]
+
+    video = cv2.VideoCapture(video_path)
+    video_fps = video.get(cv2.CAP_PROP_FPS)
+    video_size = (int(video.get(cv2.CAP_PROP_FRAME_WIDTH)),
+                  int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)))
+    output_dir = 'data/output/'
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+    swap_video_path = output_dir + 'temp.mp4'
+    videoWriter = cv2.VideoWriter(swap_video_path, cv2.VideoWriter_fourcc(*'mp4v'), video_fps, video_size)
+
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+        futures = []
+        while True:
+            ret, frame = video.read()
+            if not ret:
+                break
+            future = executor.submit(processor.process_frame, frame, image_latent, use_gfpgan, sr_weight,
+                                     use_color_trans, color_trans_mode)
+            futures.append(future)
+
+        for future in futures:
+            processed_frame = future.result()
+            if processed_frame is not None:
+                videoWriter.write(processed_frame)
+
+    video.release()
+    videoWriter.release()
+
+    add_audio_to_video(video_path, swap_video_path)
+
+    return swap_video_path
+
+
+def add_audio_to_video(original_video_path, swapped_video_path):
+    audio_file_path = original_video_path.split('.')[0] + '.wav'
+    if not os.path.exists(audio_file_path):
+        os.system(f'ffmpeg -y -hide_banner -loglevel error -i "{original_video_path}" -f wav -vn "{audio_file_path}"')
+
+    temp_output_path = swapped_video_path.replace('.mp4', '_with_audio.mp4')
+    os.system(
+        f'ffmpeg -y -hide_banner -loglevel error -i "{swapped_video_path}" -i "{audio_file_path}" -c:v copy -c:a aac "{temp_output_path}"')
+
+    os.remove(swapped_video_path)
+    os.rename(temp_output_path, swapped_video_path)
+
+
+# Gradio interface setup
+iface = gr.Interface(
+    fn=process_image_video,
+    inputs=[
+        gr.Image(type="pil", label="Source Image"),
+        gr.Video(label="Input Video"),
+        gr.Checkbox(label="Use GFPGAN [Super-Resolution]"),
+        gr.Slider(minimum=0.1, maximum=1.0, step=0.1, label="SR Weight [only support GFPGAN enabled]", value=1.0),
+        gr.Checkbox(label="Use Color Transfer"),
+        gr.Dropdown(choices=["rct", "lct", "mkl", "idt", "sot"],
+                    label="Color Transfer Mode [only support Color-Transfer enabled]", value="rct")
+    ],
+    outputs=gr.Video(label="Output Video"),
+    title="Video Generation",
+    description="Upload an image and a video, and the system will generate a new video based on the input."
+)
+
+if __name__ == "__main__":
+    iface.launch()

color_transfer.py

@@ -0,0 +1,337 @@
+import cv2
+import numexpr as ne
+import numpy as np
+import scipy as sp
+from numpy import linalg as npla
+
+
+def color_transfer_sot(src,trg, steps=10, batch_size=5, reg_sigmaXY=16.0, reg_sigmaV=5.0):
+    """
+    Color Transform via Sliced Optimal Transfer
+    ported by @iperov from https://github.com/dcoeurjo/OTColorTransfer
+
+    src         - any float range any channel image
+    dst         - any float range any channel image, same shape as src
+    steps       - number of solver steps
+    batch_size  - solver batch size
+    reg_sigmaXY - apply regularization and sigmaXY of filter, otherwise set to 0.0
+    reg_sigmaV  - sigmaV of filter
+
+    return value - clip it manually
+    """
+    if not np.issubdtype(src.dtype, np.floating):
+        raise ValueError("src value must be float")
+    if not np.issubdtype(trg.dtype, np.floating):
+        raise ValueError("trg value must be float")
+
+    if len(src.shape) != 3:
+        raise ValueError("src shape must have rank 3 (h,w,c)")
+
+    if src.shape != trg.shape:
+        raise ValueError("src and trg shapes must be equal")
+
+    src_dtype = src.dtype
+    h,w,c = src.shape
+    new_src = src.copy()
+
+    advect = np.empty ( (h*w,c), dtype=src_dtype )
+    for step in range (steps):
+        advect.fill(0)
+        for batch in range (batch_size):
+            dir = np.random.normal(size=c).astype(src_dtype)
+            dir /= npla.norm(dir)
+
+            projsource = np.sum( new_src*dir, axis=-1).reshape ((h*w))
+            projtarget = np.sum( trg*dir, axis=-1).reshape ((h*w))
+
+            idSource = np.argsort (projsource)
+            idTarget = np.argsort (projtarget)
+
+            a = projtarget[idTarget]-projsource[idSource]
+            for i_c in range(c):
+                advect[idSource,i_c] += a * dir[i_c]
+        new_src += advect.reshape( (h,w,c) ) / batch_size
+
+    if reg_sigmaXY != 0.0:
+        src_diff = new_src-src
+        src_diff_filt = cv2.bilateralFilter (src_diff, 0, reg_sigmaV, reg_sigmaXY )
+        if len(src_diff_filt.shape) == 2:
+            src_diff_filt = src_diff_filt[...,None]
+        new_src = src + src_diff_filt
+    return new_src
+
+def color_transfer_mkl(x0, x1):
+    eps = np.finfo(float).eps
+
+    h,w,c = x0.shape
+    h1,w1,c1 = x1.shape
+
+    x0 = x0.reshape ( (h*w,c) )
+    x1 = x1.reshape ( (h1*w1,c1) )
+
+    a = np.cov(x0.T)
+    b = np.cov(x1.T)
+
+    Da2, Ua = np.linalg.eig(a)
+    Da = np.diag(np.sqrt(Da2.clip(eps, None)))
+
+    C = np.dot(np.dot(np.dot(np.dot(Da, Ua.T), b), Ua), Da)
+
+    Dc2, Uc = np.linalg.eig(C)
+    Dc = np.diag(np.sqrt(Dc2.clip(eps, None)))
+
+    Da_inv = np.diag(1./(np.diag(Da)))
+
+    t = np.dot(np.dot(np.dot(np.dot(np.dot(np.dot(Ua, Da_inv), Uc), Dc), Uc.T), Da_inv), Ua.T)
+
+    mx0 = np.mean(x0, axis=0)
+    mx1 = np.mean(x1, axis=0)
+
+    result = np.dot(x0-mx0, t) + mx1
+    return np.clip ( result.reshape ( (h,w,c) ).astype(x0.dtype), 0, 1)
+
+def color_transfer_idt(i0, i1, bins=256, n_rot=20):
+    import scipy.stats
+    
+    relaxation = 1 / n_rot
+    h,w,c = i0.shape
+    h1,w1,c1 = i1.shape
+
+    i0 = i0.reshape ( (h*w,c) )
+    i1 = i1.reshape ( (h1*w1,c1) )
+
+    n_dims = c
+
+    d0 = i0.T
+    d1 = i1.T
+
+    for i in range(n_rot):
+
+        r = sp.stats.special_ortho_group.rvs(n_dims).astype(np.float32)
+
+        d0r = np.dot(r, d0)
+        d1r = np.dot(r, d1)
+        d_r = np.empty_like(d0)
+
+        for j in range(n_dims):
+
+            lo = min(d0r[j].min(), d1r[j].min())
+            hi = max(d0r[j].max(), d1r[j].max())
+
+            p0r, edges = np.histogram(d0r[j], bins=bins, range=[lo, hi])
+            p1r, _     = np.histogram(d1r[j], bins=bins, range=[lo, hi])
+
+            cp0r = p0r.cumsum().astype(np.float32)
+            cp0r /= cp0r[-1]
+
+            cp1r = p1r.cumsum().astype(np.float32)
+            cp1r /= cp1r[-1]
+
+            f = np.interp(cp0r, cp1r, edges[1:])
+
+            d_r[j] = np.interp(d0r[j], edges[1:], f, left=0, right=bins)
+
+        d0 = relaxation * np.linalg.solve(r, (d_r - d0r)) + d0
+
+    return np.clip ( d0.T.reshape ( (h,w,c) ).astype(i0.dtype) , 0, 1)
+
+def reinhard_color_transfer(target : np.ndarray, source : np.ndarray, target_mask : np.ndarray = None, source_mask : np.ndarray = None, mask_cutoff=0.5) -> np.ndarray:
+    """
+    Transfer color using rct method.
+
+        target      np.ndarray H W 3C   (BGR)   np.float32
+        source      np.ndarray H W 3C   (BGR)   np.float32
+
+        target_mask(None)   np.ndarray H W 1C  np.float32
+        source_mask(None)   np.ndarray H W 1C  np.float32
+        
+        mask_cutoff(0.5)    float
+
+    masks are used to limit the space where color statistics will be computed to adjust the target
+
+    reference: Color Transfer between Images https://www.cs.tau.ac.il/~turkel/imagepapers/ColorTransfer.pdf
+    """
+    source = cv2.cvtColor(source, cv2.COLOR_BGR2LAB)
+    target = cv2.cvtColor(target, cv2.COLOR_BGR2LAB)
+
+    source_input = source
+    if source_mask is not None:
+        source_input = source_input.copy()
+        source_input[source_mask[...,0] < mask_cutoff] = [0,0,0]
+    
+    target_input = target
+    if target_mask is not None:
+        target_input = target_input.copy()
+        target_input[target_mask[...,0] < mask_cutoff] = [0,0,0]
+
+    target_l_mean, target_l_std, target_a_mean, target_a_std, target_b_mean, target_b_std, \
+        = target_input[...,0].mean(), target_input[...,0].std(), target_input[...,1].mean(), target_input[...,1].std(), target_input[...,2].mean(), target_input[...,2].std()
+    
+    source_l_mean, source_l_std, source_a_mean, source_a_std, source_b_mean, source_b_std, \
+        = source_input[...,0].mean(), source_input[...,0].std(), source_input[...,1].mean(), source_input[...,1].std(), source_input[...,2].mean(), source_input[...,2].std()
+    
+    # not as in the paper: scale by the standard deviations using reciprocal of paper proposed factor
+    target_l = target[...,0]
+    target_l = ne.evaluate('(target_l - target_l_mean) * source_l_std / target_l_std + source_l_mean')
+
+    target_a = target[...,1]
+    target_a = ne.evaluate('(target_a - target_a_mean) * source_a_std / target_a_std + source_a_mean')
+    
+    target_b = target[...,2]
+    target_b = ne.evaluate('(target_b - target_b_mean) * source_b_std / target_b_std + source_b_mean')
+
+    np.clip(target_l,    0, 100, out=target_l)
+    np.clip(target_a, -127, 127, out=target_a)
+    np.clip(target_b, -127, 127, out=target_b)
+
+    return cv2.cvtColor(np.stack([target_l,target_a,target_b], -1), cv2.COLOR_LAB2BGR)
+
+
+def linear_color_transfer(target_img, source_img, mode='pca', eps=1e-5):
+    '''
+    Matches the colour distribution of the target image to that of the source image
+    using a linear transform.
+    Images are expected to be of form (w,h,c) and float in [0,1].
+    Modes are chol, pca or sym for different choices of basis.
+    '''
+    mu_t = target_img.mean(0).mean(0)
+    t = target_img - mu_t
+    t = t.transpose(2,0,1).reshape( t.shape[-1],-1)
+    t = t.reshape( t.shape[-1],-1)
+    Ct = t.dot(t.T) / t.shape[1] + eps * np.eye(t.shape[0])
+    mu_s = source_img.mean(0).mean(0)
+    s = source_img - mu_s
+    s = s.transpose(2,0,1).reshape( s.shape[-1],-1)
+    Cs = s.dot(s.T) / s.shape[1] + eps * np.eye(s.shape[0])
+    if mode == 'chol':
+        chol_t = np.linalg.cholesky(Ct)
+        chol_s = np.linalg.cholesky(Cs)
+        ts = chol_s.dot(np.linalg.inv(chol_t)).dot(t)
+    if mode == 'pca':
+        eva_t, eve_t = np.linalg.eigh(Ct)
+        Qt = eve_t.dot(np.sqrt(np.diag(eva_t))).dot(eve_t.T)
+        eva_s, eve_s = np.linalg.eigh(Cs)
+        Qs = eve_s.dot(np.sqrt(np.diag(eva_s))).dot(eve_s.T)
+        ts = Qs.dot(np.linalg.inv(Qt)).dot(t)
+    if mode == 'sym':
+        eva_t, eve_t = np.linalg.eigh(Ct)
+        Qt = eve_t.dot(np.sqrt(np.diag(eva_t))).dot(eve_t.T)
+        Qt_Cs_Qt = Qt.dot(Cs).dot(Qt)
+        eva_QtCsQt, eve_QtCsQt = np.linalg.eigh(Qt_Cs_Qt)
+        QtCsQt = eve_QtCsQt.dot(np.sqrt(np.diag(eva_QtCsQt))).dot(eve_QtCsQt.T)
+        ts = np.linalg.inv(Qt).dot(QtCsQt).dot(np.linalg.inv(Qt)).dot(t)
+    matched_img = ts.reshape(*target_img.transpose(2,0,1).shape).transpose(1,2,0)
+    matched_img += mu_s
+    matched_img[matched_img>1] = 1
+    matched_img[matched_img<0] = 0
+    return np.clip(matched_img.astype(source_img.dtype), 0, 1)
+
+def lab_image_stats(image):
+    # compute the mean and standard deviation of each channel
+    (l, a, b) = cv2.split(image)
+    (lMean, lStd) = (l.mean(), l.std())
+    (aMean, aStd) = (a.mean(), a.std())
+    (bMean, bStd) = (b.mean(), b.std())
+
+    # return the color statistics
+    return (lMean, lStd, aMean, aStd, bMean, bStd)
+
+def _scale_array(arr, clip=True):
+    if clip:
+        return np.clip(arr, 0, 255)
+
+    mn = arr.min()
+    mx = arr.max()
+    scale_range = (max([mn, 0]), min([mx, 255]))
+
+    if mn < scale_range[0] or mx > scale_range[1]:
+        return (scale_range[1] - scale_range[0]) * (arr - mn) / (mx - mn) + scale_range[0]
+
+    return arr
+
+def channel_hist_match(source, template, hist_match_threshold=255, mask=None):
+    # Code borrowed from:
+    # https://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x
+    masked_source = source
+    masked_template = template
+
+    if mask is not None:
+        masked_source = source * mask
+        masked_template = template * mask
+
+    oldshape = source.shape
+    source = source.ravel()
+    template = template.ravel()
+    masked_source = masked_source.ravel()
+    masked_template = masked_template.ravel()
+    s_values, bin_idx, s_counts = np.unique(source, return_inverse=True,
+                                            return_counts=True)
+    t_values, t_counts = np.unique(template, return_counts=True)
+
+    s_quantiles = np.cumsum(s_counts).astype(np.float64)
+    s_quantiles = hist_match_threshold * s_quantiles / s_quantiles[-1]
+    t_quantiles = np.cumsum(t_counts).astype(np.float64)
+    t_quantiles = 255 * t_quantiles / t_quantiles[-1]
+    interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)
+
+    return interp_t_values[bin_idx].reshape(oldshape)
+
+def color_hist_match(src_im, tar_im, hist_match_threshold=255):
+    h,w,c = src_im.shape
+    matched_R = channel_hist_match(src_im[:,:,0], tar_im[:,:,0], hist_match_threshold, None)
+    matched_G = channel_hist_match(src_im[:,:,1], tar_im[:,:,1], hist_match_threshold, None)
+    matched_B = channel_hist_match(src_im[:,:,2], tar_im[:,:,2], hist_match_threshold, None)
+
+    to_stack = (matched_R, matched_G, matched_B)
+    for i in range(3, c):
+        to_stack += ( src_im[:,:,i],)
+
+
+    matched = np.stack(to_stack, axis=-1).astype(src_im.dtype)
+    return matched
+
+def color_transfer_mix(img_src,img_trg):
+    img_src = np.clip(img_src*255.0, 0, 255).astype(np.uint8)
+    img_trg = np.clip(img_trg*255.0, 0, 255).astype(np.uint8)
+
+    img_src_lab = cv2.cvtColor(img_src, cv2.COLOR_BGR2LAB)
+    img_trg_lab = cv2.cvtColor(img_trg, cv2.COLOR_BGR2LAB)
+
+    rct_light = np.clip ( linear_color_transfer(img_src_lab[...,0:1].astype(np.float32)/255.0,
+                                                img_trg_lab[...,0:1].astype(np.float32)/255.0 )[...,0]*255.0,
+                          0, 255).astype(np.uint8)
+
+    img_src_lab[...,0] = (np.ones_like (rct_light)*100).astype(np.uint8)
+    img_src_lab = cv2.cvtColor(img_src_lab, cv2.COLOR_LAB2BGR)
+
+    img_trg_lab[...,0] = (np.ones_like (rct_light)*100).astype(np.uint8)
+    img_trg_lab = cv2.cvtColor(img_trg_lab, cv2.COLOR_LAB2BGR)
+
+    img_rct = color_transfer_sot( img_src_lab.astype(np.float32), img_trg_lab.astype(np.float32) )
+    img_rct = np.clip(img_rct, 0, 255).astype(np.uint8)
+
+    img_rct = cv2.cvtColor(img_rct, cv2.COLOR_BGR2LAB)
+    img_rct[...,0] = rct_light
+    img_rct = cv2.cvtColor(img_rct, cv2.COLOR_LAB2BGR)
+
+
+    return (img_rct / 255.0).astype(np.float32)
+
+def color_transfer(ct_mode, img_src, img_trg):
+    """
+    color transfer for [0,1] float32 inputs
+    """
+    if ct_mode == 'lct':
+        out = linear_color_transfer(img_src, img_trg)
+    elif ct_mode == 'rct':
+        out = reinhard_color_transfer(img_src, img_trg)
+    elif ct_mode == 'mkl':
+        out = color_transfer_mkl(img_src, img_trg)
+    elif ct_mode == 'idt':
+        out = color_transfer_idt(img_src, img_trg)
+    elif ct_mode == 'sot':
+        out = color_transfer_sot(img_src, img_trg)
+        out = np.clip( out, 0.0, 1.0)
+    else:
+        raise ValueError(f"unknown ct_mode {ct_mode}")
+    return out

data/image_feature_dict.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32c5871c89e526e5c088cbee5db03b87135c27be4d985cfcd78c8ce02a4af482
+size 3975088

data/source/demo.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754d104f06af6d80da356c8311bbcabf1f3fd467cebd98a8b658d9e94f7507b8
+size 2402911

face_detect/FaceType.py

@@ -0,0 +1,37 @@
+from enum import IntEnum
+
+class FaceType(IntEnum):
+    #enumerating in order "next contains prev"
+    HALF = 0
+    MID_FULL = 1
+    FULL = 2
+    FULL_NO_ALIGN = 3
+    WHOLE_FACE = 4
+    HEAD = 10
+    HEAD_NO_ALIGN = 20
+
+    MARK_ONLY = 100, #no align at all, just embedded faceinfo
+
+    @staticmethod
+    def fromString (s):
+        r = from_string_dict.get (s.lower())
+        if r is None:
+            raise Exception ('FaceType.fromString value error')
+        return r
+
+    @staticmethod
+    def toString (face_type):
+        return to_string_dict[face_type]
+
+to_string_dict = { FaceType.HALF : 'half_face',
+                   FaceType.MID_FULL : 'midfull_face',
+                   FaceType.FULL : 'full_face',
+                   FaceType.FULL_NO_ALIGN : 'full_face_no_align',
+                   FaceType.WHOLE_FACE : 'whole_face',
+                   FaceType.HEAD : 'head',
+                   FaceType.HEAD_NO_ALIGN : 'head_no_align',
+                   
+                   FaceType.MARK_ONLY :'mark_only',  
+                 }
+
+from_string_dict = { to_string_dict[x] : x for x in to_string_dict.keys() }  

face_detect/LandmarksProcessor.py

@@ -0,0 +1,1482 @@
+import colorsys
+import math
+from enum import IntEnum
+
+import cv2
+import numpy as np
+import numpy.linalg as npla
+
+from face_detect.core import imagelib
+from face_detect.core import mathlib
+from face_detect.core.mathlib.umeyama import umeyama
+from face_detect.FaceType import FaceType
+
+mesh_33=[70,63,105,66,107,336,296,334,293,300,168,197,5,4,240,99,2,328,460,33,160,158,133,153,144,362,385,387,263,373,380,57,287]
+landmarks_2D_4=np.array([
+[0.224152 , 0.2119465], #left iris  mean 37 38 40 41
+[0.75610125, 0.2119465],#right iris mean 43 44 46 47
+[0.490127, 0.515625], # nose 30
+[0.4901265, 0.780233]  #mouth mean 48 54
+])
+landmarks_2D_4_bottom=np.array([
+[0.2218305, 0.244588 ], #left iris  mean 40 41
+[0.7584225, 0.244588],#right iris mean 46 47
+[0.490127, 0.515625], # nose 30
+[0.4901265, 0.780233]  #mouth mean 48 54
+])
+landmarks_2D = np.array([
+    [0.000213256, 0.106454],  # 17
+    [0.0752622, 0.038915],  # 18
+    [0.18113, 0.0187482],  # 19
+    [0.29077, 0.0344891],  # 20
+    [0.393397, 0.0773906],  # 21
+    [0.586856, 0.0773906],  # 22
+    [0.689483, 0.0344891],  # 23
+    [0.799124, 0.0187482],  # 24
+    [0.904991, 0.038915],  # 25
+    [0.98004, 0.106454],  # 26
+    [0.490127, 0.203352],  # 27
+    [0.490127, 0.307009],  # 28
+    [0.490127, 0.409805],  # 29
+    [0.490127, 0.515625],  # 30
+    [0.36688, 0.587326],  # 31
+    [0.426036, 0.609345],  # 32
+    [0.490127, 0.628106],  # 33
+    [0.554217, 0.609345],  # 34
+    [0.613373, 0.587326],  # 35
+    [0.121737, 0.216423],  # 36
+    [0.187122, 0.178758],  # 37
+    [0.265825, 0.179852],  # 38
+    [0.334606, 0.231733],  # 39
+    [0.260918, 0.245099],  # 40
+    [0.182743, 0.244077],  # 41
+    [0.645647, 0.231733],  # 42
+    [0.714428, 0.179852],  # 43
+    [0.793132, 0.178758],  # 44
+    [0.858516, 0.216423],  # 45
+    [0.79751, 0.244077],  # 46
+    [0.719335, 0.245099],  # 47
+    [0.254149, 0.780233],  # 48
+    [0.340985, 0.745405],  # 49
+    [0.428858, 0.727388],  # 50
+    [0.490127, 0.742578],  # 51
+    [0.551395, 0.727388],  # 52
+    [0.639268, 0.745405],  # 53
+    [0.726104, 0.780233],  # 54
+    [0.642159, 0.864805],  # 55
+    [0.556721, 0.902192],  # 56
+    [0.490127, 0.909281],  # 57
+    [0.423532, 0.902192],  # 58
+    [0.338094, 0.864805],  # 59
+    [0.290379, 0.784792],  # 60
+    [0.428096, 0.778746],  # 61
+    [0.490127, 0.785343],  # 62
+    [0.552157, 0.778746],  # 63
+    [0.689874, 0.784792],  # 64
+    [0.553364, 0.824182],  # 65
+    [0.490127, 0.831803],  # 66
+    [0.42689, 0.824182]  # 67
+], dtype=np.float32)
+
+landmarks_2D_new = np.array([
+    [0.000213256, 0.106454],  # 17
+    [0.0752622, 0.038915],  # 18
+    [0.18113, 0.0187482],  # 19
+    [0.29077, 0.0344891],  # 20
+    [0.393397, 0.0773906],  # 21
+    [0.586856, 0.0773906],  # 22
+    [0.689483, 0.0344891],  # 23
+    [0.799124, 0.0187482],  # 24
+    [0.904991, 0.038915],  # 25
+    [0.98004, 0.106454],  # 26
+    [0.490127, 0.203352],  # 27
+    [0.490127, 0.307009],  # 28
+    [0.490127, 0.409805],  # 29
+    [0.490127, 0.515625],  # 30
+    [0.36688, 0.587326],  # 31
+    [0.426036, 0.609345],  # 32
+    [0.490127, 0.628106],  # 33
+    [0.554217, 0.609345],  # 34
+    [0.613373, 0.587326],  # 35
+    [0.121737, 0.216423],  # 36
+    [0.187122, 0.178758],  # 37
+    [0.265825, 0.179852],  # 38
+    [0.334606, 0.231733],  # 39
+    [0.260918, 0.245099],  # 40
+    [0.182743, 0.244077],  # 41
+    [0.645647, 0.231733],  # 42
+    [0.714428, 0.179852],  # 43
+    [0.793132, 0.178758],  # 44
+    [0.858516, 0.216423],  # 45
+    [0.79751, 0.244077],  # 46
+    [0.719335, 0.245099],  # 47
+    [0.254149, 0.780233],  # 48
+    [0.726104, 0.780233],  # 54
+], dtype=np.float32)
+landmarks_2D_new_mesh = np.array([
+[ 0.000213256,  0.106454  ], #17
+[ 0.0752622,    0.038915  ], #18
+[0.1281961, 0.0288316], #19[ 0.18113,      0.0187482 ]
+[ 0.29077,      0.0144891 ], #20
+[ 0.393397,     0.0773906 ], #21
+[ 0.586856,     0.0773906 ], #22
+[ 0.689483,     0.0144891 ], #23
+[0.8520575, 0.0288316], #24[ 0.799124,     0.0187482 ]
+[ 0.904991,     0.038915  ], #25
+[ 0.98004,      0.106454  ], #26
+[ 0.490127,     0.203352  ], #27
+[ 0.490127,     0.307009  ], #28
+[ 0.490127,     0.409805  ], #29
+[ 0.490127,     0.515625  ], #30
+[0.396458 , 0.5983355], #31 [ 0.36688,      0.587326  ]
+[ 0.426036,     0.609345  ], #32
+[ 0.490127,     0.628106  ], #33
+[ 0.554217,     0.609345  ], #34
+[ 0.613373,     0.587326  ], #35
+[ 0.071737,     0.136423  ], #36
+[ 0.137122,     0.118758  ], #37
+[ 0.215825,     0.119852  ], #38
+[ 0.334606,     0.151733  ], #39
+[ 0.210918,     0.165099  ], #40
+[ 0.132743,     0.164077  ], #41
+[ 0.645647,     0.151733  ], #42
+[ 0.764428,     0.119852  ], #43
+[ 0.743132,     0.118758  ], #44
+[ 0.908516,     0.136423  ], #45
+[ 0.84751,      0.164077  ], #46
+[ 0.769335,     0.165099  ], #47
+[ 0.254149,     0.780233  ], #48
+[ 0.726104,     0.780233  ], #54
+], dtype=np.float32)
+
+# landmarks_468_moving_parts_indexes = [0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42, 43, 46, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 95, 96, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 117, 118, 124, 130, 132, 133, 135, 136, 138, 139, 140, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 168, 169, 170, 171, 172, 173, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 189, 190, 191, 192, 193, 194, 199, 200, 201, 202, 204, 208, 210, 211, 212, 213, 214, 215, 221, 222, 223, 224, 225, 226, 228, 229, 230, 231, 232, 233, 243, 244, 245, 246, 247, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 267, 268, 269, 270, 271, 272, 273, 276, 282, 283, 284, 285, 286, 287, 288, 291, 292, 293, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 306, 307, 308, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 324, 325, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 346, 347, 353, 359, 361, 362, 364, 365, 367, 368, 369, 372, 373, 374, 375, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 413, 414, 415, 416, 417, 418, 421, 422, 424, 428, 430, 431, 432, 433, 434, 435, 441, 442, 443, 444, 445, 446, 448, 449, 450, 451, 452, 453, 463, 464, 465, 466, 467]
+# uni_landmarks_468 = np.array(
+# [[ 0.49066195,  0.7133885 ],
+#        [ 0.49042386,  0.52723485],
+#        [ 0.49050152,  0.6244965 ],
+#        [ 0.45844677,  0.39348277],
+#        [ 0.4905825 ,  0.49120593],
+#        [ 0.49006602,  0.43998772],
+#        [ 0.48907965,  0.26775706],
+#        [ 0.11721139,  0.23243594],
+#        [ 0.48957095,  0.11063451],
+#        [ 0.48949632,  0.03535742],
+#        [ 0.48905632, -0.25326234],
+#        [ 0.4907858 ,  0.73766613],
+#        [ 0.49081355,  0.7606857 ],
+#        [ 0.4908666 ,  0.7839426 ],
+#        [ 0.49079415,  0.78913504],
+#        [ 0.4908271 ,  0.80801845],
+#        [ 0.49086872,  0.831855  ],
+#        [ 0.49092326,  0.8631041 ],
+#        [ 0.49104446,  0.94170016],
+#        [ 0.49009967,  0.5546924 ],
+#        [ 0.44398275,  0.5741402 ],
+#        [-0.2106727 ,  0.00861922],
+#        [ 0.2523662 ,  0.2832579 ],
+#        [ 0.2042254 ,  0.28945392],
+#        [ 0.1552372 ,  0.28322184],
+#        [ 0.09056008,  0.24730967],
+#        [ 0.30096018,  0.27277085],
+#        [ 0.21548809,  0.16713436],
+#        [ 0.2595488 ,  0.17071684],
+#        [ 0.16957955,  0.17298089],
+#        [ 0.13164258,  0.18425746],
+#        [ 0.043018  ,  0.28581   ],
+#        [ 0.30856833,  1.0507976 ],
+#        [ 0.10015843,  0.22331452],
+#        [-0.20773543,  0.26701325],
+#        [-0.02414621,  0.25144747],
+#        [ 0.23481508,  0.5045001 ],
+#        [ 0.44063616,  0.7097012 ],
+#        [ 0.4449884 ,  0.762481  ],
+#        [ 0.3840104 ,  0.7218947 ],
+#        [ 0.33943903,  0.73847425],
+#        [ 0.40284824,  0.76374006],
+#        [ 0.36457124,  0.76704985],
+#        [ 0.26937196,  0.84716266],
+#        [ 0.46683946,  0.5275276 ],
+#        [ 0.4642676 ,  0.49167544],
+#        [ 0.06039319,  0.11509081],
+#        [ 0.31504983,  0.36394927],
+#        [ 0.3660137 ,  0.52945083],
+#        [ 0.3509634 ,  0.50311893],
+#        [ 0.09496811,  0.5005815 ],
+#        [ 0.46075967,  0.4424029 ],
+#        [ 0.20108324,  0.05883435],
+#        [ 0.12877828,  0.07731954],
+#        [-0.09675749, -0.09848522],
+#        [ 0.39672711,  0.09345116],
+#        [ 0.29908365,  0.18449144],
+#        [ 0.23298171,  0.7922538 ],
+#        [-0.27583498,  0.85219014],
+#        [ 0.38898414,  0.5723152 ],
+#        [ 0.41446668,  0.59347576],
+#        [ 0.28167963,  0.7884952 ],
+#        [ 0.30013445,  0.7875627 ],
+#        [ 0.09448256,  0.03961415],
+#        [ 0.3531811 ,  0.5553779 ],
+#        [ 0.2873921 ,  0.05599196],
+#        [ 0.28232294,  0.01076962],
+#        [ 0.1903341 , -0.23029903],
+#        [ 0.0108011 , -0.03099815],
+#        [ 0.24915197, -0.10741784],
+#        [ 0.01047484,  0.08868673],
+#        [-0.08942058,  0.05201372],
+#        [ 0.44268388,  0.7376863 ],
+#        [ 0.39652622,  0.741894  ],
+#        [ 0.35389552,  0.7514722 ],
+#        [ 0.393559  ,  0.5851372 ],
+#        [ 0.2925385 ,  0.7871472 ],
+#        [ 0.31904542,  0.80939215],
+#        [ 0.32005206,  0.787085  ],
+#        [ 0.4195982 ,  0.5444628 ],
+#        [ 0.3688312 ,  0.78418756],
+#        [ 0.40608776,  0.7841225 ],
+#        [ 0.4472093 ,  0.78405076],
+#        [ 0.43053833,  0.9379409 ],
+#        [ 0.44192585,  0.8617842 ],
+#        [ 0.44321233,  0.82923037],
+#        [ 0.4432334 ,  0.80578357],
+#        [ 0.44304678,  0.78921837],
+#        [ 0.36314115,  0.7893578 ],
+#        [ 0.36057413,  0.8040033 ],
+#        [ 0.35472178,  0.8187327 ],
+#        [ 0.34614718,  0.83330894],
+#        [ 0.2959003 ,  0.69076014],
+#        [-0.37090415,  0.5509728 ],
+#        [ 0.4903264 ,  0.5851119 ],
+#        [ 0.3370172 ,  0.78961957],
+#        [ 0.33070365,  0.8010128 ],
+#        [ 0.43397966,  0.6231119 ],
+#        [ 0.35356513,  0.59569615],
+#        [ 0.42509514,  0.6093918 ],
+#        [ 0.2635329 ,  0.39636588],
+#        [ 0.19704658,  0.43663597],
+#        [ 0.33384863,  0.52658314],
+#        [ 0.03225203, -0.18047164],
+#        [ 0.11854403, -0.08533629],
+#        [ 0.18350407,  0.01215954],
+#        [ 0.31292278,  0.8845064 ],
+#        [ 0.3862302 ,  0.02093028],
+#        [ 0.36480215, -0.1098879 ],
+#        [ 0.33342764, -0.2497105 ],
+#        [ 0.11592615,  0.2646692 ],
+#        [-0.00803981,  0.3294946 ],
+#        [ 0.33535972,  0.26431814],
+#        [ 0.05940344,  0.18766014],
+#        [ 0.36188984,  0.33336782],
+#        [ 0.39879864,  0.50869733],
+#        [-0.07952328,  0.36885905],
+#        [ 0.04230375,  0.36800843],
+#        [ 0.11137532,  0.3864613 ],
+#        [ 0.19386435,  0.37397826],
+#        [ 0.25749052,  0.34993485],
+#        [ 0.310977  ,  0.3240539 ],
+#        [ 0.44813582,  0.2762354 ],
+#        [-0.06039021,  0.4864401 ],
+#        [ 0.00945808,  0.17624807],
+#        [ 0.4739895 ,  0.55369264],
+#        [ 0.32125092,  0.4170324 ],
+#        [-0.36162117,  0.27013144],
+#        [ 0.3592803 ,  0.3023075 ],
+#        [ 0.30784345,  0.529875  ],
+#        [ 0.07601253,  0.22579695],
+#        [ 0.3824061 ,  0.47686696],
+#        [-0.33810768,  0.70034444],
+#        [ 0.34643772,  0.24336138],
+#        [ 0.42429656,  0.45338264],
+#        [ 0.02854156,  0.939626  ],
+#        [-0.04352415,  1.0322431 ],
+#        [-0.20510256,  0.51651907],
+#        [-0.06969981,  0.8698207 ],
+#        [-0.1581445 ,  0.14948419],
+#        [ 0.2889787 ,  1.1224228 ],
+#        [ 0.47446907,  0.58377683],
+#        [ 0.2818322 ,  0.4586393 ],
+#        [-0.08708218,  0.2627534 ],
+#        [ 0.16877942,  0.25976214],
+#        [ 0.21234928,  0.267416  ],
+#        [ 0.30676025,  0.81592965],
+#        [-0.06259334,  0.6009466 ],
+#        [ 0.36930662,  1.2302231 ],
+#        [ 0.17070079,  1.149443  ],
+#        [ 0.07714309,  1.0989524 ],
+#        [ 0.48931465, -0.1052461 ],
+#        [ 0.49159575,  1.2484183 ],
+#        [ 0.2527582 ,  0.26420003],
+#        [ 0.30066028,  0.25829503],
+#        [ 0.3310663 ,  0.25034374],
+#        [-0.05075949,  0.16421606],
+#        [ 0.29250854,  0.19938153],
+#        [ 0.2522571 ,  0.18826446],
+#        [ 0.21220936,  0.18724632],
+#        [ 0.16866222,  0.19260857],
+#        [ 0.13789575,  0.2011967 ],
+#        [-0.29335994,  0.12383505],
+#        [ 0.1379709 ,  0.24424627],
+#        [ 0.49057597,  0.65296   ],
+#        [ 0.34147182,  0.663431  ],
+#        [ 0.3941785 ,  0.5603462 ],
+#        [ 0.43007633,  0.6569765 ],
+#        [ 0.48963526,  0.17996965],
+#        [ 0.11681002,  1.0107123 ],
+#        [ 0.19942053,  1.068824  ],
+#        [ 0.38605705,  1.1563928 ],
+#        [-0.16756529,  0.9615808 ],
+#        [ 0.32817602,  0.21989337],
+#        [ 0.41141313,  0.3578073 ],
+#        [ 0.49127796,  1.1678538 ],
+#        [ 0.27080515,  1.195178  ],
+#        [-0.19307071,  0.6481067 ],
+#        [ 0.399859  ,  0.7892937 ],
+#        [ 0.39875022,  0.80587196],
+#        [ 0.39717573,  0.8256797 ],
+#        [ 0.3931817 ,  0.85224336],
+#        [ 0.3670306 ,  0.9161113 ],
+#        [ 0.3256227 ,  0.7724022 ],
+#        [ 0.31488904,  0.76426226],
+#        [ 0.3001029 ,  0.7583232 ],
+#        [ 0.2565659 ,  0.73397243],
+#        [ 0.0438394 ,  0.6234349 ],
+#        [ 0.40628996,  0.30296788],
+#        [ 0.37707803,  0.19498621],
+#        [ 0.34125936,  0.21069102],
+#        [ 0.33733743,  0.7842425 ],
+#        [ 0.00882016,  0.769232  ],
+#        [ 0.4335431 ,  0.1821002 ],
+#        [ 0.33409703,  0.9826546 ],
+#        [ 0.49011812,  0.3896104 ],
+#        [ 0.45311242,  0.34152514],
+#        [ 0.4899982 ,  0.33611432],
+#        [ 0.369907  ,  0.43193236],
+#        [ 0.49116373,  1.0932964 ],
+#        [ 0.49107185,  1.0132186 ],
+#        [ 0.41421878,  1.008873  ],
+#        [ 0.21551576,  0.8785059 ],
+#        [ 0.27587482,  0.57461077],
+#        [ 0.2683325 ,  0.9399872 ],
+#        [ 0.17091931,  0.56899554],
+#        [ 0.23741819,  0.6283017 ],
+#        [ 0.12783033,  0.65916985],
+#        [ 0.39875996,  1.0855893 ],
+#        [ 0.33251646,  0.45881665],
+#        [ 0.16138549,  0.93153137],
+#        [ 0.23269826,  0.99740875],
+#        [ 0.17994387,  0.8051213 ],
+#        [-0.06026869,  0.7033027 ],
+#        [ 0.10063827,  0.8241594 ],
+#        [-0.15810522,  0.7679798 ],
+#        [ 0.2014156 ,  0.7000692 ],
+#        [ 0.365875  ,  0.3839739 ],
+#        [ 0.4115726 ,  0.5293855 ],
+#        [ 0.378973  ,  0.5476473 ],
+#        [ 0.43235463,  0.49621448],
+#        [ 0.3385827 ,  0.15134089],
+#        [ 0.27179635,  0.12940899],
+#        [ 0.21341887,  0.12485553],
+#        [ 0.15807948,  0.12881717],
+#        [ 0.10610204,  0.14814937],
+#        [ 0.03133116,  0.236169  ],
+#        [-0.21341309,  0.38895622],
+#        [ 0.07818349,  0.3101151 ],
+#        [ 0.1318462 ,  0.32528982],
+#        [ 0.19485526,  0.32642388],
+#        [ 0.25329807,  0.31256682],
+#        [ 0.30569646,  0.29578218],
+#        [ 0.34839994,  0.2842457 ],
+#        [-0.3824783 ,  0.41054142],
+#        [ 0.37162504,  0.5664833 ],
+#        [ 0.41687053,  0.40615496],
+#        [ 0.4433516 ,  0.5242282 ],
+#        [ 0.44805393,  0.5562703 ],
+#        [ 0.43453053,  0.5407472 ],
+#        [ 0.37351128,  0.58924097],
+#        [ 0.46121803,  0.55474806],
+#        [ 0.45942986,  0.5810936 ],
+#        [ 0.35955238,  0.24802393],
+#        [ 0.38181108,  0.25985107],
+#        [ 0.40143687,  0.26679716],
+#        [ 0.11717269,  0.2102652 ],
+#        [ 0.0940459 ,  0.2016577 ],
+#        [ 0.5217974 ,  0.39331725],
+#        [ 0.8625129 ,  0.23113514],
+#        [ 0.5369363 ,  0.57397795],
+#        [ 1.1896138 ,  0.00617525],
+#        [ 0.7275363 ,  0.28242856],
+#        [ 0.7756985 ,  0.2884565 ],
+#        [ 0.82466465,  0.28205347],
+#        [ 0.88921595,  0.24591576],
+#        [ 0.6788919 ,  0.27210945],
+#        [ 0.7640089 ,  0.166177  ],
+#        [ 0.7199609 ,  0.16991326],
+#        [ 0.8099376 ,  0.17186326],
+#        [ 0.8479136 ,  0.18300733],
+#        [ 0.9368992 ,  0.28424102],
+#        [ 0.67367214,  1.0503516 ],
+#        [ 0.8795338 ,  0.22195426],
+#        [ 1.1875838 ,  0.26458502],
+#        [ 1.0039485 ,  0.24965489],
+#        [ 0.74551606,  0.50375396],
+#        [ 0.54075617,  0.7095265 ],
+#        [ 0.5365969 ,  0.76231945],
+#        [ 0.59742403,  0.7215222 ],
+#        [ 0.6420548 ,  0.7379461 ],
+#        [ 0.5787324 ,  0.7634331 ],
+#        [ 0.617019  ,  0.766611  ],
+#        [ 0.71218634,  0.8469107 ],
+#        [ 0.513503  ,  0.52683127],
+#        [ 0.5170686 ,  0.49132976],
+#        [ 0.91894245,  0.11362247],
+#        [ 0.66487545,  0.36299667],
+#        [ 0.61502695,  0.52894545],
+#        [ 0.6296784 ,  0.50242335],
+#        [ 0.88566196,  0.49919614],
+#        [ 0.5193738 ,  0.4423927 ],
+#        [ 0.7780587 ,  0.05788935],
+#        [ 0.8504331 ,  0.07610969],
+#        [ 1.0753254 , -0.1005309 ],
+#        [ 0.5824533 ,  0.09305263],
+#        [ 0.6804744 ,  0.18382579],
+#        [ 0.7485537 ,  0.79121745],
+#        [ 1.2577202 ,  0.8495136 ],
+#        [ 0.59192824,  0.57196105],
+#        [ 0.5665197 ,  0.59321034],
+#        [ 0.6999867 ,  0.7877651 ],
+#        [ 0.6814933 ,  0.7868972 ],
+#        [ 0.8846023 ,  0.03829005],
+#        [ 0.62761134,  0.5547819 ],
+#        [ 0.6917209 ,  0.05532694],
+#        [ 0.6966465 ,  0.01012804],
+#        [ 0.7876697 , -0.2309872 ],
+#        [ 0.9680314 , -0.03263693],
+#        [ 0.7294528 , -0.1080169 ],
+#        [ 0.96877015,  0.08704082],
+#        [ 1.0685298 ,  0.05000517],
+#        [ 0.538806  ,  0.7375185 ],
+#        [ 0.5849781 ,  0.7415651 ],
+#        [ 0.62764204,  0.7509944 ],
+#        [ 0.58739805,  0.5847989 ],
+#        [ 0.68912315,  0.78645504],
+#        [ 0.6626941 ,  0.8087924 ],
+#        [ 0.6616096 ,  0.7864889 ],
+#        [ 0.5612171 ,  0.5442156 ],
+#        [ 0.61282057,  0.7837617 ],
+#        [ 0.575564  ,  0.7838267 ],
+#        [ 0.5344426 ,  0.7838985 ],
+#        [ 0.551505  ,  0.93764293],
+#        [ 0.5399973 ,  0.8616131 ],
+#        [ 0.53859717,  0.8290639 ],
+#        [ 0.5384943 ,  0.8056173 ],
+#        [ 0.53862303,  0.78905153],
+#        [ 0.6185288 ,  0.78891206],
+#        [ 0.62114686,  0.8035485 ],
+#        [ 0.62705064,  0.81825733],
+#        [ 0.635676  ,  0.8328036 ],
+#        [ 0.6854969 ,  0.69067734],
+#        [ 1.3517375 ,  0.54796624],
+#        [ 0.64465326,  0.78908265],
+#        [ 0.6510032 ,  0.8004538 ],
+#        [ 0.5471015 ,  0.62291807],
+#        [ 0.62742317,  0.59512955],
+#        [ 0.55593795,  0.6091671 ],
+#        [ 0.7161671 ,  0.39546603],
+#        [ 0.7836529 ,  0.435396  ],
+#        [ 0.64694774,  0.5258542 ],
+#        [ 0.94603044, -0.1820665 ],
+#        [ 0.86011904, -0.08652072],
+#        [ 0.79549086,  0.01118712],
+#        [ 0.66893554,  0.8840338 ],
+#        [ 0.59274685,  0.02056277],
+#        [ 0.613851  , -0.11025709],
+#        [ 0.64526045, -0.25000137],
+#        [ 0.8639107 ,  0.26336375],
+#        [ 0.9881146 ,  0.3277454 ],
+#        [ 0.6445285 ,  0.26371115],
+#        [ 0.92017305,  0.18616839],
+#        [ 0.61790556,  0.3323734 ],
+#        [ 0.58225924,  0.5077285 ],
+#        [ 1.0597262 ,  0.36687428],
+#        [ 0.93791103,  0.36642405],
+#        [ 0.86892897,  0.38505408],
+#        [ 0.78624976,  0.37287512],
+#        [ 0.7223912 ,  0.34902957],
+#        [ 0.6687594 ,  0.32310694],
+#        [ 0.5315497 ,  0.2757726 ],
+#        [ 1.0409807 ,  0.48452145],
+#        [ 0.9700836 ,  0.17458573],
+#        [ 0.5065989 ,  0.55419755],
+#        [ 0.6590531 ,  0.41624966],
+#        [ 1.3414742 ,  0.26715896],
+#        [ 0.62023264,  0.30108824],
+#        [ 0.67289865,  0.5290446 ],
+#        [ 0.9036883 ,  0.22435239],
+#        [ 0.59769833,  0.47659585],
+#        [ 1.3194624 ,  0.6974514 ],
+#        [ 0.63339525,  0.24286939],
+#        [ 0.5571053 ,  0.45250946],
+#        [ 0.9535533 ,  0.9380257 ],
+#        [ 1.0260391 ,  1.0303764 ],
+#        [ 1.1858007 ,  0.51410204],
+#        [ 1.0515786 ,  0.867869  ],
+#        [ 1.1375865 ,  0.14722979],
+#        [ 0.6935665 ,  1.1218798 ],
+#        [ 0.5063422 ,  0.58382744],
+#        [ 0.69926125,  0.45745537],
+#        [ 1.0669235 ,  0.26074636],
+#        [ 0.8110406 ,  0.25864118],
+#        [ 0.7674977 ,  0.26644707],
+#        [ 0.67500204,  0.81528693],
+#        [ 1.0435516 ,  0.5990178 ],
+#        [ 0.6121316 ,  1.2306852 ],
+#        [ 0.81222653,  1.1483234 ],
+#        [ 0.9056057 ,  1.0975065 ],
+#        [ 0.7270778 ,  0.26337218],
+#        [ 0.6791554 ,  0.25763443],
+#        [ 0.6487802 ,  0.24975733],
+#        [ 1.0302606 ,  0.16233999],
+#        [ 0.68710136,  0.19869283],
+#        [ 0.72731376,  0.18743533],
+#        [ 0.7673578 ,  0.1862774 ],
+#        [ 0.81092334,  0.1914876 ],
+#        [ 0.84171957,  0.1999683 ],
+#        [ 1.2727026 ,  0.12110176],
+#        [ 0.8417947 ,  0.24301787],
+#        [ 0.63978463,  0.6627527 ],
+#        [ 0.5866921 ,  0.5600102 ],
+#        [ 0.5511283 ,  0.6567636 ],
+#        [ 0.8655194 ,  1.009457  ],
+#        [ 0.78306264,  1.0678959 ],
+#        [ 0.59620714,  1.1564037 ],
+#        [ 1.149833  ,  0.9592815 ],
+#        [ 0.65151644,  0.21932903],
+#        [ 0.56865776,  0.3571483 ],
+#        [ 0.71228063,  1.1944076 ],
+#        [ 1.1742088 ,  0.6457327 ],
+#        [ 0.5818109 ,  0.78897613],
+#        [ 0.5829775 ,  0.80555046],
+#        [ 0.5846211 ,  0.82535255],
+#        [ 0.5887078 ,  0.8519021 ],
+#        [ 0.6150045 ,  0.916079  ],
+#        [ 0.65597004,  0.771831  ],
+#        [ 0.66669285,  0.7636482 ],
+#        [ 0.6814582 ,  0.7576576 ],
+#        [ 0.7245435 ,  0.73241323],
+#        [ 0.9371713 ,  0.62184393],
+#        [ 0.5736738 ,  0.30186948],
+#        [ 0.60240346,  0.19448838],
+#        [ 0.6383993 ,  0.21017241],
+#        [ 0.64431435,  0.7837067 ],
+#        [ 0.9726586 ,  0.7675604 ],
+#        [ 0.54576766,  0.18157108],
+#        [ 0.6477745 ,  0.98230904],
+#        [ 0.5269076 ,  0.34123868],
+#        [ 0.61068684,  0.43131724],
+#        [ 0.56792   ,  1.0087004 ],
+#        [ 0.7662271 ,  0.8776794 ],
+#        [ 0.7048996 ,  0.57387614],
+#        [ 0.7136024 ,  0.9394351 ],
+#        [ 0.8097781 ,  0.56784695],
+#        [ 0.7435453 ,  0.62753886],
+#        [ 0.85328954,  0.6578133 ],
+#        [ 0.5835228 ,  1.0854707 ],
+#        [ 0.64810187,  0.45811343],
+#        [ 0.82059515,  0.9304676 ],
+#        [ 0.7494546 ,  0.9966611 ],
+#        [ 0.8015866 ,  0.80400985],
+#        [ 1.0415541 ,  0.70138854],
+#        [ 0.8809724 ,  0.8228132 ],
+#        [ 1.1396528 ,  0.7657218 ],
+#        [ 0.7798614 ,  0.69881856],
+#        [ 0.6143189 ,  0.383193  ],
+#        [ 0.56934875,  0.52867246],
+#        [ 0.60162777,  0.54706186],
+#        [ 0.5470082 ,  0.4963955 ],
+#        [ 0.6408297 ,  0.15073723],
+#        [ 0.7075675 ,  0.12865019],
+#        [ 0.76593757,  0.12391254],
+#        [ 0.8212976 ,  0.12768434],
+#        [ 0.87334216,  0.14682971],
+#        [ 0.948411  ,  0.23457018],
+#        [ 1.1936799 ,  0.38651106],
+#        [ 0.90181875,  0.30865455],
+#        [ 0.84818983,  0.3240165 ],
+#        [ 0.7851249 ,  0.32537246],
+#        [ 0.72658616,  0.3116911 ],
+#        [ 0.6740513 ,  0.2949461 ],
+#        [ 0.63111407,  0.28325075],
+#        [ 1.362823  ,  0.4074953 ],
+#        [ 0.60951644,  0.5658945 ],
+#        [ 0.5634702 ,  0.4055624 ],
+#        [ 0.5374476 ,  0.5247268 ],
+#        [ 0.53280455,  0.5561224 ],
+#        [ 0.5462737 ,  0.5405522 ],
+#        [ 0.6075077 ,  0.58877414],
+#        [ 0.51933056,  0.55477065],
+#        [ 0.52143395,  0.58103496],
+#        [ 0.62030756,  0.24758299],
+#        [ 0.59746987,  0.2574137 ],
+#        [ 0.5780933 ,  0.2652785 ],
+#        [ 0.8624742 ,  0.2089644 ],
+#        [ 0.8855709 ,  0.20027623]], dtype=np.float32)
+
+# mesh_33 = np.arange(468)
+# mask = np.ones(len(mesh_33), dtype=bool)
+# mask[landmarks_468_moving_parts_indexes]=False
+# mesh_33=mesh_33[mask,...]
+# landmarks_2D_new_mesh=uni_landmarks_468[mask,...]
+# mouth_center_landmarks_2D = np.array([
+#     [-4.4202591e-07, 4.4916576e-01],  # 48
+#     [1.8399176e-01, 3.7537053e-01],  # 49
+#     [3.7018123e-01, 3.3719531e-01],  # 50
+#     [5.0000089e-01, 3.6938059e-01],  # 51
+#     [6.2981832e-01, 3.3719531e-01],  # 52
+#     [8.1600773e-01, 3.7537053e-01],  # 53
+#     [1.0000000e+00, 4.4916576e-01],  # 54
+#     [8.2213330e-01, 6.2836081e-01],  # 55
+#     [6.4110327e-01, 7.0757812e-01],  # 56
+#     [5.0000089e-01, 7.2259867e-01],  # 57
+#     [3.5889623e-01, 7.0757812e-01],  # 58
+#     [1.7786618e-01, 6.2836081e-01],  # 59
+#     [7.6765373e-02, 4.5882553e-01],  # 60
+#     [3.6856663e-01, 4.4601500e-01],  # 61
+#     [5.0000089e-01, 4.5999300e-01],  # 62
+#     [6.3143289e-01, 4.4601500e-01],  # 63
+#     [9.2323411e-01, 4.5882553e-01],  # 64
+#     [6.3399029e-01, 5.4228687e-01],  # 65
+#     [5.0000089e-01, 5.5843467e-01],  # 66
+#     [3.6601129e-01, 5.4228687e-01]  # 67
+# ], dtype=np.float32)
+
+# 68 point landmark definitions
+landmarks_68_pt = {"mouth": (48, 68),
+                   "right_eyebrow": (17, 22),
+                   "left_eyebrow": (22, 27),
+                   "right_eye": (36, 42),
+                   "left_eye": (42, 48),
+                   "nose": (27, 36),  # missed one point
+                   "jaw": (0, 17)}
+
+landmarks_68_3D = np.array([
+    [-73.393523, -29.801432, 47.667532],  # 00
+    [-72.775014, -10.949766, 45.909403],  # 01
+    [-70.533638, 7.929818, 44.842580],  # 02
+    [-66.850058, 26.074280, 43.141114],  # 03
+    [-59.790187, 42.564390, 38.635298],  # 04
+    [-48.368973, 56.481080, 30.750622],  # 05
+    [-34.121101, 67.246992, 18.456453],  # 06
+    [-17.875411, 75.056892, 3.609035],  # 07
+    [0.098749, 77.061286, -0.881698],  # 08
+    [17.477031, 74.758448, 5.181201],  # 09
+    [32.648966, 66.929021, 19.176563],  # 10
+    [46.372358, 56.311389, 30.770570],  # 11
+    [57.343480, 42.419126, 37.628629],  # 12
+    [64.388482, 25.455880, 40.886309],  # 13
+    [68.212038, 6.990805, 42.281449],  # 14
+    [70.486405, -11.666193, 44.142567],  # 15
+    [71.375822, -30.365191, 47.140426],  # 16
+    [-61.119406, -49.361602, 14.254422],  # 17
+    [-51.287588, -58.769795, 7.268147],  # 18
+    [-37.804800, -61.996155, 0.442051],  # 19
+    [-24.022754, -61.033399, -6.606501],  # 20
+    [-11.635713, -56.686759, -11.967398],  # 21
+    [12.056636, -57.391033, -12.051204],  # 22
+    [25.106256, -61.902186, -7.315098],  # 23
+    [38.338588, -62.777713, -1.022953],  # 24
+    [51.191007, -59.302347, 5.349435],  # 25
+    [60.053851, -50.190255, 11.615746],  # 26
+    [0.653940, -42.193790, -13.380835],  # 27
+    [0.804809, -30.993721, -21.150853],  # 28
+    [0.992204, -19.944596, -29.284036],  # 29
+    [1.226783, -8.414541, -36.948060],  # 00
+    [-14.772472, 2.598255, -20.132003],  # 01
+    [-7.180239, 4.751589, -23.536684],  # 02
+    [0.555920, 6.562900, -25.944448],  # 03
+    [8.272499, 4.661005, -23.695741],  # 04
+    [15.214351, 2.643046, -20.858157],  # 05
+    [-46.047290, -37.471411, 7.037989],  # 06
+    [-37.674688, -42.730510, 3.021217],  # 07
+    [-27.883856, -42.711517, 1.353629],  # 08
+    [-19.648268, -36.754742, -0.111088],  # 09
+    [-28.272965, -35.134493, -0.147273],  # 10
+    [-38.082418, -34.919043, 1.476612],  # 11
+    [19.265868, -37.032306, -0.665746],  # 12
+    [27.894191, -43.342445, 0.247660],  # 13
+    [37.437529, -43.110822, 1.696435],  # 14
+    [45.170805, -38.086515, 4.894163],  # 15
+    [38.196454, -35.532024, 0.282961],  # 16
+    [28.764989, -35.484289, -1.172675],  # 17
+    [-28.916267, 28.612716, -2.240310],  # 18
+    [-17.533194, 22.172187, -15.934335],  # 19
+    [-6.684590, 19.029051, -22.611355],  # 20
+    [0.381001, 20.721118, -23.748437],  # 21
+    [8.375443, 19.035460, -22.721995],  # 22
+    [18.876618, 22.394109, -15.610679],  # 23
+    [28.794412, 28.079924, -3.217393],  # 24
+    [19.057574, 36.298248, -14.987997],  # 25
+    [8.956375, 39.634575, -22.554245],  # 26
+    [0.381549, 40.395647, -23.591626],  # 27
+    [-7.428895, 39.836405, -22.406106],  # 28
+    [-18.160634, 36.677899, -15.121907],  # 29
+    [-24.377490, 28.677771, -4.785684],  # 30
+    [-6.897633, 25.475976, -20.893742],  # 31
+    [0.340663, 26.014269, -22.220479],  # 32
+    [8.444722, 25.326198, -21.025520],  # 33
+    [24.474473, 28.323008, -5.712776],  # 34
+    [8.449166, 30.596216, -20.671489],  # 35
+    [0.205322, 31.408738, -21.903670],  # 36
+    [-7.198266, 30.844876, -20.328022]  # 37
+], dtype=np.float32)
+
+FaceType_to_padding_remove_align = {
+    FaceType.HALF: (0.0, False),
+    FaceType.MID_FULL: (0.0675, False),
+    FaceType.FULL: (0.2109375, False),
+    FaceType.FULL_NO_ALIGN: (0.2109375, True),
+    FaceType.WHOLE_FACE: (0.40, False),
+    FaceType.HEAD: (0.70, False),
+    FaceType.HEAD_NO_ALIGN: (0.70, True),
+}
+
+
+def convert_98_to_68(lmrks):
+    # jaw
+    result = [lmrks[0]]
+    for i in range(2, 16, 2):
+        result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2]
+    result += [lmrks[16]]
+    for i in range(18, 32, 2):
+        result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2]
+    result += [lmrks[32]]
+
+    # eyebrows averaging
+    result += [lmrks[33],
+               (lmrks[34] + lmrks[41]) / 2,
+               (lmrks[35] + lmrks[40]) / 2,
+               (lmrks[36] + lmrks[39]) / 2,
+               (lmrks[37] + lmrks[38]) / 2,
+               ]
+
+    result += [(lmrks[42] + lmrks[50]) / 2,
+               (lmrks[43] + lmrks[49]) / 2,
+               (lmrks[44] + lmrks[48]) / 2,
+               (lmrks[45] + lmrks[47]) / 2,
+               lmrks[46]
+               ]
+
+    # nose
+    result += list(lmrks[51:60])
+
+    # left eye (from our view)
+    result += [lmrks[60],
+               lmrks[61],
+               lmrks[63],
+               lmrks[64],
+               lmrks[65],
+               lmrks[67]]
+
+    # right eye
+    result += [lmrks[68],
+               lmrks[69],
+               lmrks[71],
+               lmrks[72],
+               lmrks[73],
+               lmrks[75]]
+
+    # mouth
+    result += list(lmrks[76:96])
+
+    return np.concatenate(result).reshape((68, 2))
+
+
+def transform_points(points, mat, invert=False):
+    if invert:
+        mat = cv2.invertAffineTransform(mat)
+    points = np.expand_dims(points, axis=1)
+    points = cv2.transform(points, mat, points.shape)
+    points = np.squeeze(points)
+    return points
+
+
+def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0):
+    if not isinstance(image_landmarks, np.ndarray):
+        image_landmarks = np.array(image_landmarks)
+
+    # estimate landmarks transform from global space to local aligned space with bounds [0..1]
+    mat = umeyama(np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]), landmarks_2D_new, True)[0:2]
+
+    # get corner points in global space
+    g_p = transform_points(np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True)
+    g_c = g_p[4]
+
+    # calc diagonal vectors between corners in global space
+    tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32)
+    tb_diag_vec /= npla.norm(tb_diag_vec)
+    bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32)
+    bt_diag_vec /= npla.norm(bt_diag_vec)
+
+    # calc modifier of diagonal vectors for scale and padding value
+    # print(face_type)
+    padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
+    mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5))
+
+    if face_type == FaceType.WHOLE_FACE:
+        # adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead
+        vec = (g_p[0] - g_p[3]).astype(np.float32)
+        vec_len = npla.norm(vec)
+        vec /= vec_len
+        g_c += vec * vec_len * 0.07
+
+
+    # calc 3 points in global space to estimate 2d affine transform
+    if not remove_align:
+        l_t = np.array([g_c - tb_diag_vec * mod,
+                        g_c + bt_diag_vec * mod,
+                        g_c + tb_diag_vec * mod])
+    else:
+        # remove_align - face will be centered in the frame but not aligned
+        l_t = np.array([g_c - tb_diag_vec * mod,
+                        g_c + bt_diag_vec * mod,
+                        g_c + tb_diag_vec * mod,
+                        g_c - bt_diag_vec * mod,
+                        ])
+
+        # get area of face square in global space
+        area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1])
+
+        # calc side of square
+        side = np.float32(math.sqrt(area) / 2)
+
+        # calc 3 points with unrotated square
+        l_t = np.array([g_c + [-side, -side],
+                        g_c + [side, -side],
+                        g_c + [side, side]])
+
+    # calc affine transform from 3 global space points to 3 local space points size of 'output_size'
+    pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size)))
+    mat = cv2.getAffineTransform(l_t, pts2)
+    return mat
+
+
+def get_rect_from_landmarks(image_landmarks):
+    mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN)
+
+    g_p = transform_points(np.float32([(0, 0), (255, 255)]), mat, True)
+
+    (l, t, r, b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1]
+
+    return (l, t, r, b)
+
+def get_transform_mat_all(image_landmarks,uni_landmarks,output_size,scale=1,gcx=-0.02,gcy=0.15,face_type=FaceType.WHOLE_FACE):
+    if not isinstance(image_landmarks, np.ndarray):
+        image_landmarks = np.array (image_landmarks)
+    # estimate landmarks transform from global space to local aligned space with bounds [0..1]
+
+    mat = umeyama(image_landmarks, uni_landmarks, True)[0:2]
+
+    # get corner points in global space
+    g_p = transform_points (  np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True)
+    g_c = g_p[4]
+    # calc diagonal vectors between corners in global space
+
+
+    tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32)
+    tb_diag_vec /= npla.norm(tb_diag_vec)
+    bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32)
+    bt_diag_vec /= npla.norm(bt_diag_vec)
+
+    # calc modifier of diagonal vectors for scale and padding value
+    padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
+    mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5))
+
+    vec = (g_p[0]-g_p[3]).astype(np.float32)
+    vec_len = npla.norm(vec)
+    vec /= vec_len
+    g_c += vec*vec_len*[gcx,gcy]
+
+
+    # calc 3 points in global space to estimate 2d affine transform
+    if not remove_align:
+        l_t = np.array([g_c - tb_diag_vec * mod,
+                        g_c + bt_diag_vec * mod,
+                        g_c + tb_diag_vec * mod])
+    else:
+        # remove_align - face will be centered in the frame but not aligned
+        l_t = np.array([g_c - tb_diag_vec * mod,
+                        g_c + bt_diag_vec * mod,
+                        g_c + tb_diag_vec * mod,
+                        g_c - bt_diag_vec * mod,
+                        ])
+
+        # get area of face square in global space
+        area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1])
+
+        # calc side of square
+        side = np.float32(math.sqrt(area) / 2)
+
+        # calc 3 points with unrotated square
+        l_t = np.array([g_c + [-side, -side],
+                        g_c + [side, -side],
+                        g_c + [side, side]])
+
+    # calc affine transform from 3 global space points to 3 local space points size of 'output_size'
+    pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size)))
+    mat = cv2.getAffineTransform(l_t, pts2)
+    return mat
+
+
+def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0):
+
+    if len(lmrks) != 68:
+        raise Exception('works only with 68 landmarks')
+    lmrks = np.array(lmrks.copy(), dtype=np.int)
+
+    # #nose
+    ml_pnt = (lmrks[36] + lmrks[0]) // 2
+    mr_pnt = (lmrks[16] + lmrks[45]) // 2
+
+    # mid points between the mid points and eye
+    ql_pnt = (lmrks[36] + ml_pnt) // 2
+    qr_pnt = (lmrks[45] + mr_pnt) // 2
+
+    # Top of the eye arrays
+    bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39]))
+    bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt))
+
+    # Eyebrow arrays
+    top_l = lmrks[17:22]
+    top_r = lmrks[22:27]
+
+    # Adjust eyebrow arrays
+    lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l)
+    lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r)
+    return lmrks
+
+
+def get_image_hull_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0):
+    hull_mask = np.zeros(image_shape[0:2] + (1,), dtype=np.float32)
+
+    lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
+
+    r_jaw = (lmrks[0:9], lmrks[17:18])
+    l_jaw = (lmrks[8:17], lmrks[26:27])
+    r_cheek = (lmrks[17:20], lmrks[8:9])
+    l_cheek = (lmrks[24:27], lmrks[8:9])
+    nose_ridge = (lmrks[19:25], lmrks[8:9],)
+    r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9])
+    l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9])
+    nose = (lmrks[27:31], lmrks[31:36])
+    parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
+
+    for item in parts:
+        merged = np.concatenate(item)
+        cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,))
+
+    return hull_mask
+
+
+def get_image_eye_mask(image_shape, image_landmarks):
+    if len(image_landmarks) != 68:
+        raise Exception('get_image_eye_mask works only with 68 landmarks')
+
+    h, w, c = image_shape
+
+    hull_mask = np.zeros((h, w, 1), dtype=np.float32)
+
+    image_landmarks = image_landmarks.astype(np.int)
+
+    cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[36:42]), (1,))
+    cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[42:48]), (1,))
+
+    dilate = h // 32
+    hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1)
+
+    blur = h // 16
+    blur = blur + (1 - blur % 2)
+    hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0)
+    hull_mask = hull_mask[..., None]
+
+    return hull_mask
+
+
+def get_image_mouth_mask(image_shape, image_landmarks):
+    if len(image_landmarks) != 68:
+        raise Exception('get_image_eye_mask works only with 68 landmarks')
+
+    h, w, c = image_shape
+
+    hull_mask = np.zeros((h, w, 1), dtype=np.float32)
+
+    image_landmarks = image_landmarks.astype(np.int)
+
+    cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[60:]), (1,))
+
+    dilate = h // 32
+    hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1)
+
+    blur = h // 16
+    blur = blur + (1 - blur % 2)
+    hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0)
+    hull_mask = hull_mask[..., None]
+
+    return hull_mask
+
+
+def alpha_to_color(img_alpha, color):
+    if len(img_alpha.shape) == 2:
+        img_alpha = img_alpha[..., None]
+    h, w, c = img_alpha.shape
+    result = np.zeros((h, w, len(color)), dtype=np.float32)
+    result[:, :] = color
+
+    return result * img_alpha
+
+
+def get_cmask(image_shape, lmrks, eyebrows_expand_mod=1.0):
+    h, w, c = image_shape
+
+    hull = get_image_hull_mask(image_shape, lmrks, eyebrows_expand_mod)
+
+    result = np.zeros((h, w, 3), dtype=np.float32)
+
+    def process(w, h, data):
+        d = {}
+        cur_lc = 0
+        all_lines = []
+        for s, pts_loop_ar in data:
+            lines = []
+            for pts, loop in pts_loop_ar:
+                pts_len = len(pts)
+                lines.append([[pts[i], pts[(i + 1) % pts_len]] for i in range(pts_len - (0 if loop else 1))])
+            lines = np.concatenate(lines)
+
+            lc = lines.shape[0]
+            all_lines.append(lines)
+            d[s] = cur_lc, cur_lc + lc
+            cur_lc += lc
+        all_lines = np.concatenate(all_lines, 0)
+
+        # calculate signed distance for all points and lines
+        line_count = all_lines.shape[0]
+        pts_count = w * h
+
+        all_lines = np.repeat(all_lines[None, ...], pts_count, axis=0).reshape((pts_count * line_count, 2, 2))
+
+        pts = np.empty((h, w, line_count, 2), dtype=np.float32)
+        pts[..., 1] = np.arange(h)[:, None, None]
+        pts[..., 0] = np.arange(w)[:, None]
+        pts = pts.reshape((h * w * line_count, -1))
+
+        a = all_lines[:, 0, :]
+        b = all_lines[:, 1, :]
+        pa = pts - a
+        ba = b - a
+        ph = np.clip(np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1)
+        dists = npla.norm(pa - ba * ph[..., None], axis=1).reshape((h, w, line_count))
+
+        def get_dists(name, thickness=0):
+            s, e = d[name]
+            result = dists[..., s:e]
+            if thickness != 0:
+                result = np.abs(result) - thickness
+            return np.min(result, axis=-1)
+
+        return get_dists
+
+    l_eye = lmrks[42:48]
+    r_eye = lmrks[36:42]
+    l_brow = lmrks[22:27]
+    r_brow = lmrks[17:22]
+    mouth = lmrks[48:60]
+
+    up_nose = np.concatenate((lmrks[27:31], lmrks[33:34]))
+    down_nose = lmrks[31:36]
+    nose = np.concatenate((up_nose, down_nose))
+
+    gdf = process(w, h,
+                  (
+                      ('eyes', ((l_eye, True), (r_eye, True))),
+                      ('brows', ((l_brow, False), (r_brow, False))),
+                      ('up_nose', ((up_nose, False),)),
+                      ('down_nose', ((down_nose, False),)),
+                      ('mouth', ((mouth, True),)),
+                  )
+                  )
+
+    eyes_fall_dist = w // 32
+    eyes_thickness = max(w // 64, 1)
+
+    brows_fall_dist = w // 32
+    brows_thickness = max(w // 256, 1)
+
+    nose_fall_dist = w / 12
+    nose_thickness = max(w // 96, 1)
+
+    mouth_fall_dist = w // 32
+    mouth_thickness = max(w // 64, 1)
+
+    eyes_mask = gdf('eyes', eyes_thickness)
+    eyes_mask = 1 - np.clip(eyes_mask / eyes_fall_dist, 0, 1)
+    # eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
+    # eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
+
+    brows_mask = gdf('brows', brows_thickness)
+    brows_mask = 1 - np.clip(brows_mask / brows_fall_dist, 0, 1)
+    # brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1)
+
+    mouth_mask = gdf('mouth', mouth_thickness)
+    mouth_mask = 1 - np.clip(mouth_mask / mouth_fall_dist, 0, 1)
+
+    # mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1)
+
+    def blend(a, b, k):
+        x = np.clip(0.5 + 0.5 * (b - a) / k, 0.0, 1.0)
+        return (a - b) * x + b - k * x * (1.0 - x)
+
+    # nose_mask = (a-b)*x+b - k*x*(1.0-x)
+
+    # nose_mask = np.minimum (up_nose_mask , down_nose_mask )
+    # nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
+
+    nose_mask = blend(gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness * 3)
+    nose_mask = 1 - np.clip(nose_mask / nose_fall_dist, 0, 1)
+
+    up_nose_mask = gdf('up_nose', nose_thickness)
+    up_nose_mask = 1 - np.clip(up_nose_mask / nose_fall_dist, 0, 1)
+    # up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1)
+
+    down_nose_mask = gdf('down_nose', nose_thickness)
+    down_nose_mask = 1 - np.clip(down_nose_mask / nose_fall_dist, 0, 1)
+    # down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1)
+
+    # nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 )
+    # nose_mask /= np.max(nose_mask)
+    # nose_mask = np.maximum (up_nose_mask , down_nose_mask )
+    # nose_mask = down_nose_mask
+
+    # nose_mask = np.zeros_like(nose_mask)
+
+    eyes_mask = eyes_mask * (1 - mouth_mask)
+    nose_mask = nose_mask * (1 - eyes_mask)
+
+    hull_mask = hull[..., 0].copy()
+    hull_mask = hull_mask * (1 - eyes_mask) * (1 - brows_mask) * (1 - nose_mask) * (1 - mouth_mask)
+
+    # eyes_mask = eyes_mask * (1-nose_mask)
+
+    mouth_mask = mouth_mask * (1 - nose_mask)
+
+    brows_mask = brows_mask * (1 - nose_mask) * (1 - eyes_mask)
+
+    hull_mask = alpha_to_color(hull_mask, (0, 1, 0))
+    eyes_mask = alpha_to_color(eyes_mask, (1, 0, 0))
+    brows_mask = alpha_to_color(brows_mask, (0, 0, 1))
+    nose_mask = alpha_to_color(nose_mask, (0, 1, 1))
+    mouth_mask = alpha_to_color(mouth_mask, (0, 0, 1))
+
+    # nose_mask = np.maximum( up_nose_mask, down_nose_mask )
+
+    result = hull_mask + mouth_mask + nose_mask + brows_mask + eyes_mask
+    result *= hull
+    # result = np.clip (result, 0, 1)
+    return result
+
+
+def blur_image_hull_mask(hull_mask):
+    maxregion = np.argwhere(hull_mask == 1.0)
+    miny, minx = maxregion.min(axis=0)[:2]
+    maxy, maxx = maxregion.max(axis=0)[:2]
+    lenx = maxx - minx;
+    leny = maxy - miny;
+    masky = int(minx + (lenx // 2))
+    maskx = int(miny + (leny // 2))
+    lowest_len = min(lenx, leny)
+    ero = int(lowest_len * 0.085)
+    blur = int(lowest_len * 0.10)
+
+    hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1)
+    hull_mask = cv2.blur(hull_mask, (blur, blur))
+    hull_mask = np.expand_dims(hull_mask, -1)
+
+    return hull_mask
+
+
+mirror_idxs = [
+    [0, 16],
+    [1, 15],
+    [2, 14],
+    [3, 13],
+    [4, 12],
+    [5, 11],
+    [6, 10],
+    [7, 9],
+
+    [17, 26],
+    [18, 25],
+    [19, 24],
+    [20, 23],
+    [21, 22],
+
+    [36, 45],
+    [37, 44],
+    [38, 43],
+    [39, 42],
+    [40, 47],
+    [41, 46],
+
+    [31, 35],
+    [32, 34],
+
+    [50, 52],
+    [49, 53],
+    [48, 54],
+    [59, 55],
+    [58, 56],
+    [67, 65],
+    [60, 64],
+    [61, 63]]
+
+
+def mirror_landmarks(landmarks, val):
+    result = landmarks.copy()
+
+    for idx in mirror_idxs:
+        result[idx] = result[idx[::-1]]
+
+    result[:, 0] = val - result[:, 0] - 1
+    return result
+
+
+def get_face_struct_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,)):
+    mask = np.zeros(image_shape[0:2] + (len(color),), dtype=np.float32)
+    lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
+    draw_landmarks(mask, image_landmarks, color=color, draw_circles=False, thickness=2)
+    return mask
+
+
+def draw_landmarks(image, image_landmarks, color=(0, 255, 0), draw_circles=True, thickness=1, transparent_mask=False):
+    if len(image_landmarks) != 68:
+        raise Exception('get_image_eye_mask works only with 68 landmarks')
+
+    int_lmrks = np.array(image_landmarks, dtype=np.int)
+
+    jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])]
+    right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])]
+    left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])]
+    mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])]
+    right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])]
+    left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])]
+    nose = int_lmrks[slice(*landmarks_68_pt["nose"])]
+
+    # open shapes
+    cv2.polylines(image,
+                  tuple(np.array([v]) for v in (right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])))),
+                  False, color, thickness=thickness, lineType=cv2.LINE_AA)
+    # closed shapes
+    cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)),
+                  True, color, thickness=thickness, lineType=cv2.LINE_AA)
+
+    if draw_circles:
+        # the rest of the cicles
+        for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0):
+            cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA)
+        # jaw big circles
+        for x, y in jaw:
+            cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA)
+
+    if transparent_mask:
+        mask = get_image_hull_mask(image.shape, image_landmarks)
+        image[...] = (image * (1 - mask) + image * mask / 2)[...]
+
+
+def draw_rect_landmarks(image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False,
+                        landmarks_color=(0, 255, 0)):
+    draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask)
+    imagelib.draw_rect(image, rect, (255, 0, 0), 2)
+
+    image_to_face_mat = get_transform_mat(image_landmarks, face_size, face_type)
+    points = transform_points([(0, 0), (0, face_size - 1), (face_size - 1, face_size - 1), (face_size - 1, 0)],
+                              image_to_face_mat, True)
+    imagelib.draw_polygon(image, points, (0, 0, 255), 2)
+
+    points = transform_points(
+        [(int(face_size * 0.05), 0), (int(face_size * 0.1), int(face_size * 0.1)), (0, int(face_size * 0.1))],
+        image_to_face_mat, True)
+    imagelib.draw_polygon(image, points, (0, 0, 255), 2)
+
+
+def calc_face_pitch(landmarks):
+    if not isinstance(landmarks, np.ndarray):
+        landmarks = np.array(landmarks)
+    t = ((landmarks[6][1] - landmarks[8][1]) + (landmarks[10][1] - landmarks[8][1])) / 2.0
+    b = landmarks[8][1]
+    return float(b - t)
+
+
+def estimate_averaged_yaw(landmarks):
+    # Works much better than solvePnP if landmarks from "3DFAN"
+    if not isinstance(landmarks, np.ndarray):
+        landmarks = np.array(landmarks)
+    l = ((landmarks[27][0] - landmarks[0][0]) + (landmarks[28][0] - landmarks[1][0]) + (
+                landmarks[29][0] - landmarks[2][0])) / 3.0
+    r = ((landmarks[16][0] - landmarks[27][0]) + (landmarks[15][0] - landmarks[28][0]) + (
+                landmarks[14][0] - landmarks[29][0])) / 3.0
+    return float(r - l)
+
+
+def estimate_pitch_yaw_roll(aligned_landmarks, size=256):
+    """
+    returns pitch,yaw,roll [-pi/2...+pi/2]
+    """
+    shape = (size, size)
+    focal_length = shape[1]
+    camera_center = (shape[1] / 2, shape[0] / 2)
+    camera_matrix = np.array(
+        [[focal_length, 0, camera_center[0]],
+         [0, focal_length, camera_center[1]],
+         [0, 0, 1]], dtype=np.float32)
+
+    (_, rotation_vector, _) = cv2.solvePnP(
+        np.concatenate((landmarks_68_3D[:27], landmarks_68_3D[30:36]), axis=0),
+        np.concatenate((aligned_landmarks[:27], aligned_landmarks[30:36]), axis=0).astype(np.float32),
+        camera_matrix,
+        np.zeros((4, 1)))
+
+    pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles(cv2.Rodrigues(rotation_vector)[0])
+
+    half_pi = math.pi / 2.0
+    pitch = np.clip(pitch, -half_pi, half_pi)
+    yaw = np.clip(yaw, -half_pi, half_pi)
+    roll = np.clip(roll, -half_pi, half_pi)
+
+    return -pitch, yaw, roll
+
+
+# if remove_align:
+#    bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True)
+#    #import code
+#    #code.interact(local=dict(globals(), **locals()))
+#    area = mathlib.polygon_area(bbox[:,0], bbox[:,1] )
+#    side = math.sqrt(area) / 2
+#    center = transform_points ( [(output_size/2,output_size/2)], mat, True)
+#    pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] ))
+#    pts2 = np.float32([[0,0],[output_size,0],[0,output_size]])
+#    mat = cv2.getAffineTransform(pts1,pts2)
+# if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN):
+#    #lmrks2 = expand_eyebrows(image_landmarks)
+#    #lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False )
+#    #y_diff = np.float32( (0,np.min(lmrks2_[:,1])) )
+#    #y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True)
+#    #y_diff = y_diff[1]-y_diff[0]
+#
+#    x_diff = np.float32((0,0))
+#
+#    lmrks2_ = transform_points( [ image_landmarks[0], image_landmarks[16] ], mat, False )
+#    if lmrks2_[0,0] < 0:
+#        x_diff = lmrks2_[0,0]
+#        x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
+#        x_diff = x_diff[1]-x_diff[0]
+#    elif lmrks2_[1,0] >= output_size:
+#        x_diff = lmrks2_[1,0]-(output_size-1)
+#        x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
+#        x_diff = x_diff[1]-x_diff[0]
+#
+#    mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2)
+
+
+"""
+def get_averaged_transform_mat (img_landmarks,
+                                img_landmarks_prev,
+                                img_landmarks_next,
+                                average_frame_count,
+                                average_center_frame_count,
+                                output_size, face_type, scale=1.0):
+
+    l_c_list = []
+    tb_diag_vec_list = []
+    bt_diag_vec_list = []
+    mod_list = []
+
+    count = max(average_frame_count,average_center_frame_count)
+    for i in range ( -count, count+1, 1 ):
+        if i < 0:
+            lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None
+        elif i > 0:
+            lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None
+        else:
+            lmrks = img_landmarks
+
+        if lmrks is None:
+            continue
+
+        l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale)
+
+        if i >= -average_frame_count and i <= average_frame_count:
+            tb_diag_vec_list.append(tb_diag_vec)
+            bt_diag_vec_list.append(bt_diag_vec)
+            mod_list.append(mod)
+
+        if i >= -average_center_frame_count and i <= average_center_frame_count:
+            l_c_list.append(l_c)
+
+    tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 )
+    bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 )
+    mod         = np.mean( np.array(mod_list), axis=0 )
+    l_c         = np.mean( np.array(l_c_list), axis=0 )
+
+    return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type)
+
+
+def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
+    if not isinstance(image_landmarks, np.ndarray):
+        image_landmarks = np.array (image_landmarks)
+
+    # get face padding value for FaceType
+    padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
+
+    # estimate landmarks transform from global space to local aligned space with bounds [0..1]
+    mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
+
+    # get corner points in global space
+    l_p = transform_points (  np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True)
+    l_c = l_p[4]
+
+    # calc diagonal vectors between corners in global space
+    tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32)
+    tb_diag_vec /= npla.norm(tb_diag_vec)
+    bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32)
+    bt_diag_vec /= npla.norm(bt_diag_vec)
+
+    # calc modifier of diagonal vectors for scale and padding value
+    mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) )
+
+    # calc 3 points in global space to estimate 2d affine transform
+    if not remove_align:
+        l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
+                          np.round( l_c + bt_diag_vec*mod ),
+                          np.round( l_c + tb_diag_vec*mod ) ] )
+    else:
+        # remove_align - face will be centered in the frame but not aligned
+        l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
+                          np.round( l_c + bt_diag_vec*mod ),
+                          np.round( l_c + tb_diag_vec*mod ),
+                          np.round( l_c - bt_diag_vec*mod ),
+                         ] )
+
+        # get area of face square in global space
+        area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
+
+        # calc side of square
+        side = np.float32(math.sqrt(area) / 2)
+
+        # calc 3 points with unrotated square
+        l_t = np.array( [ np.round( l_c + [-side,-side] ),
+                          np.round( l_c + [ side,-side] ),
+                          np.round( l_c + [ side, side] ) ] )
+
+    # calc affine transform from 3 global space points to 3 local space points size of 'output_size'
+    pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
+    mat = cv2.getAffineTransform(l_t,pts2)
+
+    return mat
+"""

face_detect/__init__.py

@@ -0,0 +1,3 @@
+from .face_align_5_landmarks import FaceDetect5Landmarks
+from .face_align_utils import estimate_norm
+

face_detect/core/imagelib/SegIEPolys.py

@@ -0,0 +1,158 @@
+import numpy as np
+import cv2
+from enum import IntEnum
+
+
+class SegIEPolyType(IntEnum):
+    EXCLUDE = 0
+    INCLUDE = 1
+    
+
+
+class SegIEPoly():
+    def __init__(self, type=None, pts=None, **kwargs):
+        self.type = type
+        
+        if pts is None:
+            pts = np.empty( (0,2), dtype=np.float32 )
+        else:
+            pts = np.float32(pts)
+        self.pts = pts
+        self.n_max = self.n = len(pts)
+
+    def dump(self):
+        return {'type': int(self.type),
+                'pts' : self.get_pts(),
+               }
+    
+    def identical(self, b):
+        if self.n != b.n:
+            return False            
+        return (self.pts[0:self.n] == b.pts[0:b.n]).all()        
+        
+    def get_type(self):
+        return self.type
+
+    def add_pt(self, x, y):
+        self.pts = np.append(self.pts[0:self.n], [ ( float(x), float(y) ) ], axis=0).astype(np.float32)
+        self.n_max = self.n = self.n + 1
+
+    def undo(self):
+        self.n = max(0, self.n-1)
+        return self.n
+
+    def redo(self):
+        self.n = min(len(self.pts), self.n+1)
+        return self.n
+
+    def redo_clip(self):
+        self.pts = self.pts[0:self.n]
+        self.n_max = self.n
+
+    def insert_pt(self, n, pt):
+        if n < 0 or n > self.n:
+            raise ValueError("insert_pt out of range")
+        self.pts = np.concatenate( (self.pts[0:n], pt[None,...].astype(np.float32), self.pts[n:]), axis=0)
+        self.n_max = self.n = self.n+1
+        
+    def remove_pt(self, n):
+        if n < 0 or n >= self.n:
+            raise ValueError("remove_pt out of range")
+        self.pts = np.concatenate( (self.pts[0:n], self.pts[n+1:]), axis=0)
+        self.n_max = self.n = self.n-1
+        
+    def get_last_point(self):
+        return self.pts[self.n-1].copy()
+
+    def get_pts(self):
+        return self.pts[0:self.n].copy()
+        
+    def get_pts_count(self):
+        return self.n
+
+    def set_point(self, id, pt):
+        self.pts[id] = pt
+        
+    def set_points(self, pts):
+        self.pts = np.array(pts)
+        self.n_max = self.n = len(pts)
+        
+    def mult_points(self, val):
+        self.pts *= val       
+    
+        
+
+class SegIEPolys():
+    def __init__(self):
+        self.polys = []
+
+    def identical(self, b):
+        polys_len = len(self.polys)
+        o_polys_len = len(b.polys)
+        if polys_len != o_polys_len:
+            return False
+        
+        return all ([ a_poly.identical(b_poly) for a_poly, b_poly in zip(self.polys, b.polys) ])
+        
+    def add_poly(self, ie_poly_type):       
+        poly = SegIEPoly(ie_poly_type)
+        self.polys.append (poly)
+        return poly
+
+    def remove_poly(self, poly):
+        if poly in self.polys:
+            self.polys.remove(poly)
+
+    def has_polys(self):
+        return len(self.polys) != 0
+        
+    def get_poly(self, id):
+        return self.polys[id]
+
+    def get_polys(self):
+        return self.polys
+        
+    def get_pts_count(self):
+        return sum([poly.get_pts_count() for poly in self.polys])
+        
+    def sort(self):
+        poly_by_type = { SegIEPolyType.EXCLUDE : [], SegIEPolyType.INCLUDE : [] }
+
+        for poly in self.polys:
+            poly_by_type[poly.type].append(poly)
+            
+        self.polys = poly_by_type[SegIEPolyType.INCLUDE] + poly_by_type[SegIEPolyType.EXCLUDE]
+
+    def __iter__(self):
+        for poly in self.polys:
+            yield poly
+
+    def overlay_mask(self, mask):
+        h,w,c = mask.shape
+        white = (1,)*c
+        black = (0,)*c
+        for poly in self.polys:
+            pts = poly.get_pts().astype(np.int32)
+            if len(pts) != 0:
+                cv2.fillPoly(mask, [pts], white if poly.type == SegIEPolyType.INCLUDE else black )
+
+    def dump(self):
+        return {'polys' : [ poly.dump() for poly in self.polys ] }
+    
+    def mult_points(self, val):
+        for poly in self.polys:
+            poly.mult_points(val)
+        
+    @staticmethod
+    def load(data=None):
+        ie_polys = SegIEPolys()
+        if data is not None:
+            if isinstance(data, list):
+                # Backward comp
+                ie_polys.polys = [ SegIEPoly(type=type, pts=pts) for (type, pts) in data ]
+            elif isinstance(data, dict):                
+                ie_polys.polys = [ SegIEPoly(**poly_cfg) for poly_cfg in data['polys'] ]
+                
+        ie_polys.sort()   
+                
+        return ie_polys

face_detect/core/imagelib/__init__.py

@@ -0,0 +1,32 @@
+from .estimate_sharpness import estimate_sharpness
+
+from .equalize_and_stack_square import equalize_and_stack_square
+
+# from .text import get_text_image, get_draw_text_lines
+
+from .draw import draw_polygon, draw_rect
+
+from .morph import morph_by_points
+
+from .warp import gen_warp_params, warp_by_params
+
+from .reduce_colors import reduce_colors
+
+from .color_transfer import color_transfer, color_transfer_mix, color_transfer_sot, color_transfer_mkl, color_transfer_idt, color_hist_match, reinhard_color_transfer, linear_color_transfer
+
+from .common import random_crop, normalize_channels, cut_odd_image, overlay_alpha_image
+
+from .SegIEPolys import *
+
+from .blursharpen import LinearMotionBlur, blursharpen
+
+from .filters import apply_random_rgb_levels, \
+                     apply_random_overlay_triangle, \
+                     apply_random_hsv_shift, \
+                     apply_random_sharpen, \
+                     apply_random_motion_blur, \
+                     apply_random_gaussian_blur, \
+                     apply_random_nearest_resize, \
+                     apply_random_bilinear_resize, \
+                     apply_random_jpeg_compress, \
+                     apply_random_relight

face_detect/core/imagelib/blursharpen.py

@@ -0,0 +1,38 @@
+import cv2
+import numpy as np
+
+def LinearMotionBlur(image, size, angle):
+    k = np.zeros((size, size), dtype=np.float32)
+    k[ (size-1)// 2 , :] = np.ones(size, dtype=np.float32)
+    k = cv2.warpAffine(k, cv2.getRotationMatrix2D( (size / 2 -0.5 , size / 2 -0.5 ) , angle, 1.0), (size, size) )
+    k = k * ( 1.0 / np.sum(k) )
+    return cv2.filter2D(image, -1, k)
+    
+def blursharpen (img, sharpen_mode=0, kernel_size=3, amount=100):
+    if kernel_size % 2 == 0:
+        kernel_size += 1
+    if amount > 0:
+        if sharpen_mode == 1: #box
+            kernel = np.zeros( (kernel_size, kernel_size), dtype=np.float32)
+            kernel[ kernel_size//2, kernel_size//2] = 1.0
+            box_filter = np.ones( (kernel_size, kernel_size), dtype=np.float32) / (kernel_size**2)
+            kernel = kernel + (kernel - box_filter) * amount
+            return cv2.filter2D(img, -1, kernel)
+        elif sharpen_mode == 2: #gaussian
+            blur = cv2.GaussianBlur(img, (kernel_size, kernel_size) , 0)
+            img = cv2.addWeighted(img, 1.0 + (0.5 * amount), blur, -(0.5 * amount), 0)
+            return img
+    elif amount < 0:
+        n = -amount
+        while n > 0:
+
+            img_blur = cv2.medianBlur(img, 5)
+            if int(n / 10) != 0:
+                img = img_blur
+            else:
+                pass_power = (n % 10) / 10.0
+                img = img*(1.0-pass_power)+img_blur*pass_power
+            n = max(n-10,0)
+
+        return img
+    return img

face_detect/core/imagelib/color_transfer.py

@@ -0,0 +1,340 @@
+import cv2
+import numexpr as ne
+import numpy as np
+from numpy import linalg as npla
+import scipy as sp
+
+
+def color_transfer_sot(src, trg, steps=10, batch_size=5, reg_sigmaXY=16.0, reg_sigmaV=5.0):
+    """
+    Color Transform via Sliced Optimal Transfer
+    ported by @iperov from https://github.com/dcoeurjo/OTColorTransfer
+
+    src         - any float range any channel image
+    dst         - any float range any channel image, same shape as src
+    steps       - number of solver steps
+    batch_size  - solver batch size
+    reg_sigmaXY - apply regularization and sigmaXY of filter, otherwise set to 0.0
+    reg_sigmaV  - sigmaV of filter
+
+    return value - clip it manually
+    """
+    if not np.issubdtype(src.dtype, np.floating):
+        raise ValueError("src value must be float")
+    if not np.issubdtype(trg.dtype, np.floating):
+        raise ValueError("trg value must be float")
+
+    if len(src.shape) != 3:
+        raise ValueError("src shape must have rank 3 (h,w,c)")
+
+    if src.shape != trg.shape:
+        raise ValueError("src and trg shapes must be equal")
+
+    src_dtype = src.dtype
+    h, w, c = src.shape
+    new_src = src.copy()
+
+    advect = np.empty((h * w, c), dtype=src_dtype)
+    for step in range(steps):
+        advect.fill(0)
+        for batch in range(batch_size):
+            dir = np.random.normal(size=c).astype(src_dtype)
+            dir /= npla.norm(dir)
+
+            projsource = np.sum(new_src * dir, axis=-1).reshape((h * w))
+            projtarget = np.sum(trg * dir, axis=-1).reshape((h * w))
+
+            idSource = np.argsort(projsource)
+            idTarget = np.argsort(projtarget)
+
+            a = projtarget[idTarget] - projsource[idSource]
+            for i_c in range(c):
+                advect[idSource, i_c] += a * dir[i_c]
+        new_src += advect.reshape((h, w, c)) / batch_size
+
+    if reg_sigmaXY != 0.0:
+        src_diff = new_src - src
+        src_diff_filt = cv2.bilateralFilter(src_diff, 0, reg_sigmaV, reg_sigmaXY)
+        if len(src_diff_filt.shape) == 2:
+            src_diff_filt = src_diff_filt[..., None]
+        new_src = src + src_diff_filt
+    return new_src
+
+
+def color_transfer_mkl(x0, x1):
+    eps = np.finfo(float).eps
+
+    h, w, c = x0.shape
+    h1, w1, c1 = x1.shape
+
+    x0 = x0.reshape((h * w, c))
+    x1 = x1.reshape((h1 * w1, c1))
+
+    a = np.cov(x0.T)
+    b = np.cov(x1.T)
+
+    Da2, Ua = np.linalg.eig(a)
+    Da = np.diag(np.sqrt(Da2.clip(eps, None)))
+
+    C = np.dot(np.dot(np.dot(np.dot(Da, Ua.T), b), Ua), Da)
+
+    Dc2, Uc = np.linalg.eig(C)
+    Dc = np.diag(np.sqrt(Dc2.clip(eps, None)))
+
+    Da_inv = np.diag(1. / (np.diag(Da)))
+
+    t = np.dot(np.dot(np.dot(np.dot(np.dot(np.dot(Ua, Da_inv), Uc), Dc), Uc.T), Da_inv), Ua.T)
+
+    mx0 = np.mean(x0, axis=0)
+    mx1 = np.mean(x1, axis=0)
+
+    result = np.dot(x0 - mx0, t) + mx1
+    return np.clip(result.reshape((h, w, c)).astype(x0.dtype), 0, 1)
+
+
+def color_transfer_idt(i0, i1, bins=256, n_rot=20):
+    import scipy.stats
+
+    relaxation = 1 / n_rot
+    h, w, c = i0.shape
+    h1, w1, c1 = i1.shape
+
+    i0 = i0.reshape((h * w, c))
+    i1 = i1.reshape((h1 * w1, c1))
+
+    n_dims = c
+
+    d0 = i0.T
+    d1 = i1.T
+
+    for i in range(n_rot):
+
+        r = sp.stats.special_ortho_group.rvs(n_dims).astype(np.float32)
+
+        d0r = np.dot(r, d0)
+        d1r = np.dot(r, d1)
+        d_r = np.empty_like(d0)
+
+        for j in range(n_dims):
+            lo = min(d0r[j].min(), d1r[j].min())
+            hi = max(d0r[j].max(), d1r[j].max())
+
+            p0r, edges = np.histogram(d0r[j], bins=bins, range=[lo, hi])
+            p1r, _ = np.histogram(d1r[j], bins=bins, range=[lo, hi])
+
+            cp0r = p0r.cumsum().astype(np.float32)
+            cp0r /= cp0r[-1]
+
+            cp1r = p1r.cumsum().astype(np.float32)
+            cp1r /= cp1r[-1]
+
+            f = np.interp(cp0r, cp1r, edges[1:])
+
+            d_r[j] = np.interp(d0r[j], edges[1:], f, left=0, right=bins)
+
+        d0 = relaxation * np.linalg.solve(r, (d_r - d0r)) + d0
+
+    return np.clip(d0.T.reshape((h, w, c)).astype(i0.dtype), 0, 1)
+
+
+def reinhard_color_transfer(target: np.ndarray, source: np.ndarray, target_mask: np.ndarray = None,
+                            source_mask: np.ndarray = None, mask_cutoff=0.5) -> np.ndarray:
+    """
+    Transfer color using rct method.
+        target      np.ndarray H W 3C   (BGR)   np.float32
+        source      np.ndarray H W 3C   (BGR)   np.float32
+        target_mask(None)   np.ndarray H W 1C  np.float32
+        source_mask(None)   np.ndarray H W 1C  np.float32
+
+        mask_cutoff(0.5)    float
+    masks are used to limit the space where color statistics will be computed to adjust the target
+    reference: Color Transfer between Images https://www.cs.tau.ac.il/~turkel/imagepapers/ColorTransfer.pdf
+    """
+    source = cv2.cvtColor(source, cv2.COLOR_BGR2LAB)
+    target = cv2.cvtColor(target, cv2.COLOR_BGR2LAB)
+
+    source_input = source
+    if source_mask is not None:
+        source_input = source_input.copy()
+        source_input[source_mask[..., 0] < mask_cutoff] = [0, 0, 0]
+
+    target_input = target
+    if target_mask is not None:
+        target_input = target_input.copy()
+        target_input[target_mask[..., 0] < mask_cutoff] = [0, 0, 0]
+
+    target_l_mean, target_l_std, target_a_mean, target_a_std, target_b_mean, target_b_std, \
+        = target_input[..., 0].mean(), target_input[..., 0].std(), target_input[..., 1].mean(), target_input[
+        ..., 1].std(), target_input[..., 2].mean(), target_input[..., 2].std()
+
+    source_l_mean, source_l_std, source_a_mean, source_a_std, source_b_mean, source_b_std, \
+        = source_input[..., 0].mean(), source_input[..., 0].std(), source_input[..., 1].mean(), source_input[
+        ..., 1].std(), source_input[..., 2].mean(), source_input[..., 2].std()
+
+    # not as in the paper: scale by the standard deviations using reciprocal of paper proposed factor
+    target_l = target[..., 0]
+    target_l = ne.evaluate('(target_l - target_l_mean) * source_l_std / target_l_std + source_l_mean')
+
+    target_a = target[..., 1]
+    target_a = ne.evaluate('(target_a - target_a_mean) * source_a_std / target_a_std + source_a_mean')
+
+    target_b = target[..., 2]
+    target_b = ne.evaluate('(target_b - target_b_mean) * source_b_std / target_b_std + source_b_mean')
+
+    np.clip(target_l, 0, 100, out=target_l)
+    np.clip(target_a, -127, 127, out=target_a)
+    np.clip(target_b, -127, 127, out=target_b)
+
+    return cv2.cvtColor(np.stack([target_l, target_a, target_b], -1), cv2.COLOR_LAB2BGR)
+
+def linear_color_transfer(target_img, source_img, mode='pca', eps=1e-5):
+    '''
+    Matches the colour distribution of the target image to that of the source image
+    using a linear transform.
+    Images are expected to be of form (w,h,c) and float in [0,1].
+    Modes are chol, pca or sym for different choices of basis.
+    '''
+    mu_t = target_img.mean(0).mean(0)
+    t = target_img - mu_t
+    t = t.transpose(2, 0, 1).reshape(t.shape[-1], -1)
+    Ct = t.dot(t.T) / t.shape[1] + eps * np.eye(t.shape[0])
+    mu_s = source_img.mean(0).mean(0)
+    s = source_img - mu_s
+    s = s.transpose(2, 0, 1).reshape(s.shape[-1], -1)
+    Cs = s.dot(s.T) / s.shape[1] + eps * np.eye(s.shape[0])
+    if mode == 'chol':
+        chol_t = np.linalg.cholesky(Ct)
+        chol_s = np.linalg.cholesky(Cs)
+        ts = chol_s.dot(np.linalg.inv(chol_t)).dot(t)
+    if mode == 'pca':
+        eva_t, eve_t = np.linalg.eigh(Ct)
+        Qt = eve_t.dot(np.sqrt(np.diag(eva_t))).dot(eve_t.T)
+        eva_s, eve_s = np.linalg.eigh(Cs)
+        Qs = eve_s.dot(np.sqrt(np.diag(eva_s))).dot(eve_s.T)
+        ts = Qs.dot(np.linalg.inv(Qt)).dot(t)
+    if mode == 'sym':
+        eva_t, eve_t = np.linalg.eigh(Ct)
+        Qt = eve_t.dot(np.sqrt(np.diag(eva_t))).dot(eve_t.T)
+        Qt_Cs_Qt = Qt.dot(Cs).dot(Qt)
+        eva_QtCsQt, eve_QtCsQt = np.linalg.eigh(Qt_Cs_Qt)
+        QtCsQt = eve_QtCsQt.dot(np.sqrt(np.diag(eva_QtCsQt))).dot(eve_QtCsQt.T)
+        ts = np.linalg.inv(Qt).dot(QtCsQt).dot(np.linalg.inv(Qt)).dot(t)
+    matched_img = ts.reshape(*target_img.transpose(2, 0, 1).shape).transpose(1, 2, 0)
+    matched_img += mu_s
+    matched_img[matched_img > 1] = 1
+    matched_img[matched_img < 0] = 0
+    return np.clip(matched_img.astype(source_img.dtype), 0, 1)
+
+
+def lab_image_stats(image):
+    # compute the mean and standard deviation of each channel
+    (l, a, b) = cv2.split(image)
+    (lMean, lStd) = (l.mean(), l.std())
+    (aMean, aStd) = (a.mean(), a.std())
+    (bMean, bStd) = (b.mean(), b.std())
+
+    # return the color statistics
+    return (lMean, lStd, aMean, aStd, bMean, bStd)
+
+
+def _scale_array(arr, clip=True):
+    if clip:
+        return np.clip(arr, 0, 255)
+
+    mn = arr.min()
+    mx = arr.max()
+    scale_range = (max([mn, 0]), min([mx, 255]))
+
+    if mn < scale_range[0] or mx > scale_range[1]:
+        return (scale_range[1] - scale_range[0]) * (arr - mn) / (mx - mn) + scale_range[0]
+
+    return arr
+
+
+def channel_hist_match(source, template, hist_match_threshold=255, mask=None):
+    # Code borrowed from:
+    # https://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x
+    masked_source = source
+    masked_template = template
+
+    if mask is not None:
+        masked_source = source * mask
+        masked_template = template * mask
+
+    oldshape = source.shape
+    source = source.ravel()
+    template = template.ravel()
+    masked_source = masked_source.ravel()
+    masked_template = masked_template.ravel()
+    s_values, bin_idx, s_counts = np.unique(source, return_inverse=True,
+                                            return_counts=True)
+    t_values, t_counts = np.unique(template, return_counts=True)
+
+    s_quantiles = np.cumsum(s_counts).astype(np.float64)
+    s_quantiles = hist_match_threshold * s_quantiles / s_quantiles[-1]
+    t_quantiles = np.cumsum(t_counts).astype(np.float64)
+    t_quantiles = 255 * t_quantiles / t_quantiles[-1]
+    interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)
+
+    return interp_t_values[bin_idx].reshape(oldshape)
+
+
+def color_hist_match(src_im, tar_im, hist_match_threshold=255):
+    h, w, c = src_im.shape
+    matched_R = channel_hist_match(src_im[:, :, 0], tar_im[:, :, 0], hist_match_threshold, None)
+    matched_G = channel_hist_match(src_im[:, :, 1], tar_im[:, :, 1], hist_match_threshold, None)
+    matched_B = channel_hist_match(src_im[:, :, 2], tar_im[:, :, 2], hist_match_threshold, None)
+
+    to_stack = (matched_R, matched_G, matched_B)
+    for i in range(3, c):
+        to_stack += (src_im[:, :, i],)
+
+    matched = np.stack(to_stack, axis=-1).astype(src_im.dtype)
+    return matched
+
+
+def color_transfer_mix(img_src, img_trg):
+    img_src = np.clip(img_src * 255.0, 0, 255).astype(np.uint8)
+    img_trg = np.clip(img_trg * 255.0, 0, 255).astype(np.uint8)
+
+    img_src_lab = cv2.cvtColor(img_src, cv2.COLOR_BGR2LAB)
+    img_trg_lab = cv2.cvtColor(img_trg, cv2.COLOR_BGR2LAB)
+
+    rct_light = np.clip(linear_color_transfer(img_src_lab[..., 0:1].astype(np.float32) / 255.0,
+                                              img_trg_lab[..., 0:1].astype(np.float32) / 255.0)[..., 0] * 255.0,
+                        0, 255).astype(np.uint8)
+
+    img_src_lab[..., 0] = (np.ones_like(rct_light) * 100).astype(np.uint8)
+    img_src_lab = cv2.cvtColor(img_src_lab, cv2.COLOR_LAB2BGR)
+
+    img_trg_lab[..., 0] = (np.ones_like(rct_light) * 100).astype(np.uint8)
+    img_trg_lab = cv2.cvtColor(img_trg_lab, cv2.COLOR_LAB2BGR)
+
+    img_rct = color_transfer_sot(img_src_lab.astype(np.float32), img_trg_lab.astype(np.float32))
+    img_rct = np.clip(img_rct, 0, 255).astype(np.uint8)
+
+    img_rct = cv2.cvtColor(img_rct, cv2.COLOR_BGR2LAB)
+    img_rct[..., 0] = rct_light
+    img_rct = cv2.cvtColor(img_rct, cv2.COLOR_LAB2BGR)
+
+    return (img_rct / 255.0).astype(np.float32)
+
+
+def color_transfer(ct_mode, img_src, img_trg):
+    """
+    color transfer for [0,1] float32 inputs
+    """
+    if ct_mode == 'lct':
+        out = linear_color_transfer(img_src, img_trg)
+    elif ct_mode == 'rct':
+        out = reinhard_color_transfer(img_src, img_trg)
+    elif ct_mode == 'mkl':
+        out = color_transfer_mkl(img_src, img_trg)
+    elif ct_mode == 'idt':
+        out = color_transfer_idt(img_src, img_trg)
+    elif ct_mode == 'sot':
+        out = color_transfer_sot(img_src, img_trg)
+        out = np.clip(out, 0.0, 1.0)
+    else:
+        raise ValueError(f"unknown ct_mode {ct_mode}")
+    return out

face_detect/core/imagelib/common.py

@@ -0,0 +1,62 @@
+import numpy as np
+
+
+def random_crop(img, w, h):
+    height, width = img.shape[:2]
+
+    h_rnd = height - h
+    w_rnd = width - w
+
+    y = np.random.randint(0, h_rnd) if h_rnd > 0 else 0
+    x = np.random.randint(0, w_rnd) if w_rnd > 0 else 0
+
+    return img[y:y + height, x:x + width]
+
+
+def normalize_channels(img, target_channels):
+    img_shape_len = len(img.shape)
+    if img_shape_len == 2:
+        h, w = img.shape
+        c = 0
+    elif img_shape_len == 3:
+        h, w, c = img.shape
+    else:
+        raise ValueError("normalize: incorrect image dimensions.")
+
+    if c == 0 and target_channels > 0:
+        img = img[..., np.newaxis]
+        c = 1
+
+    if c == 1 and target_channels > 1:
+        img = np.repeat(img, target_channels, -1)
+        c = target_channels
+
+    if c > target_channels:
+        img = img[..., 0:target_channels]
+        c = target_channels
+
+    return img
+
+
+def cut_odd_image(img):
+    h, w, c = img.shape
+    wm, hm = w % 2, h % 2
+    if wm + hm != 0:
+        img = img[0:h - hm, 0:w - wm, :]
+    return img
+
+
+def overlay_alpha_image(img_target, img_source, xy_offset=(0, 0)):
+    (h, w, c) = img_source.shape
+    if c != 4:
+        raise ValueError("overlay_alpha_image, img_source must have 4 channels")
+
+    x1, x2 = xy_offset[0], xy_offset[0] + w
+    y1, y2 = xy_offset[1], xy_offset[1] + h
+
+    alpha_s = img_source[:, :, 3] / 255.0
+    alpha_l = 1.0 - alpha_s
+
+    for c in range(0, 3):
+        img_target[y1:y2, x1:x2, c] = (alpha_s * img_source[:, :, c] +
+                                       alpha_l * img_target[y1:y2, x1:x2, c])

face_detect/core/imagelib/draw.py

@@ -0,0 +1,13 @@
+import numpy as np
+import cv2
+
+def draw_polygon (image, points, color, thickness = 1):
+    points_len = len(points)
+    for i in range (0, points_len):
+        p0 = tuple( points[i] )
+        p1 = tuple( points[ (i+1) % points_len] )
+        cv2.line (image, p0, p1, color, thickness=thickness)
+
+def draw_rect(image, rect, color, thickness=1):
+    l,t,r,b = rect
+    draw_polygon (image, [ (l,t), (r,t), (r,b), (l,b ) ], color, thickness)

face_detect/core/imagelib/equalize_and_stack_square.py

@@ -0,0 +1,45 @@
+import numpy as np
+import cv2
+
+def equalize_and_stack_square (images, axis=1):
+    max_c = max ([ 1 if len(image.shape) == 2 else image.shape[2]  for image in images ] )
+
+    target_wh = 99999
+    for i,image in enumerate(images):
+        if len(image.shape) == 2:
+            h,w = image.shape
+            c = 1
+        else:
+            h,w,c = image.shape
+
+        if h < target_wh:
+            target_wh = h
+
+        if w < target_wh:
+            target_wh = w
+
+    for i,image in enumerate(images):
+        if len(image.shape) == 2:
+            h,w = image.shape
+            c = 1
+        else:
+            h,w,c = image.shape
+
+        if c < max_c:
+            if c == 1:
+                if len(image.shape) == 2:
+                    image = np.expand_dims ( image, -1 )
+                image = np.concatenate ( (image,)*max_c, -1 )
+            elif c == 2: #GA
+                image = np.expand_dims ( image[...,0], -1 )
+                image = np.concatenate ( (image,)*max_c, -1 )
+            else:
+                image = np.concatenate ( (image, np.ones((h,w,max_c - c))), -1 )
+
+        if h != target_wh or w != target_wh:
+            image = cv2.resize ( image, (target_wh, target_wh) )
+            h,w,c = image.shape
+
+        images[i] = image
+
+    return np.concatenate ( images, axis = 1 )

face_detect/core/imagelib/estimate_sharpness.py

@@ -0,0 +1,278 @@
+"""
+Copyright (c) 2009-2010 Arizona Board of Regents.  All Rights Reserved.
+ Contact: Lina Karam (karam@asu.edu) and Niranjan Narvekar (nnarveka@asu.edu)
+ Image, Video, and Usabilty (IVU) Lab, http://ivulab.asu.edu , Arizona State University
+ This copyright statement may not be removed from any file containing it or from modifications to these files.
+ This copyright notice must also be included in any file or product that is derived from the source files.
+
+ Redistribution and use of this code in source and binary forms,  with or without modification, are permitted provided that the
+ following conditions are met:
+ - Redistribution's of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ - Redistribution's in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the distribution.
+ - The Image, Video, and Usability Laboratory (IVU Lab, http://ivulab.asu.edu) is acknowledged in any publication that
+ reports research results using this code, copies of this code, or modifications of this code.
+ The code and our papers are to be cited in the bibliography as:
+
+N. D. Narvekar and L. J. Karam, "CPBD Sharpness Metric Software", http://ivulab.asu.edu/Quality/CPBD
+
+N. D. Narvekar and L. J. Karam, "A No-Reference Image Blur Metric Based on the Cumulative
+Probability of Blur Detection (CPBD)," accepted and to appear in the IEEE Transactions on Image Processing,  2011.
+
+N. D. Narvekar and L. J. Karam, "An Improved No-Reference Sharpness Metric Based on the Probability of Blur Detection," International Workshop on Video Processing and Quality Metrics for Consumer Electronics (VPQM), January 2010, http://www.vpqm.org (pdf)
+
+N. D. Narvekar and L. J. Karam, "A No Reference Perceptual Quality Metric based on Cumulative Probability of Blur Detection," First International Workshop on the Quality of Multimedia Experience (QoMEX), pp. 87-91, July 2009.
+
+ DISCLAIMER:
+ This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the Arizona Board of Regents, Arizona State University, IVU Lab members, authors or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute
+goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage.
+"""
+
+import numpy as np
+import cv2
+from math import atan2, pi
+
+
+def sobel(image):
+    # type: (numpy.ndarray) -> numpy.ndarray
+    """
+    Find edges using the Sobel approximation to the derivatives.
+
+    Inspired by the [Octave implementation](https://sourceforge.net/p/octave/image/ci/default/tree/inst/edge.m#l196).
+    """
+    from skimage.filters.edges import HSOBEL_WEIGHTS
+    h1 = np.array(HSOBEL_WEIGHTS)
+    h1 /= np.sum(abs(h1))  # normalize h1
+    
+    from scipy.ndimage import convolve
+    strength2 = np.square(convolve(image, h1.T))
+
+    # Note: https://sourceforge.net/p/octave/image/ci/default/tree/inst/edge.m#l59
+    thresh2 = 2 * np.sqrt(np.mean(strength2))
+
+    strength2[strength2 <= thresh2] = 0
+    return _simple_thinning(strength2)
+
+
+def _simple_thinning(strength):
+    # type: (numpy.ndarray) -> numpy.ndarray
+    """
+    Perform a very simple thinning.
+
+    Inspired by the [Octave implementation](https://sourceforge.net/p/octave/image/ci/default/tree/inst/edge.m#l512).
+    """
+    num_rows, num_cols = strength.shape
+
+    zero_column = np.zeros((num_rows, 1))
+    zero_row = np.zeros((1, num_cols))
+
+    x = (
+        (strength > np.c_[zero_column, strength[:, :-1]]) &
+        (strength > np.c_[strength[:, 1:], zero_column])
+    )
+
+    y = (
+        (strength > np.r_[zero_row, strength[:-1, :]]) &
+        (strength > np.r_[strength[1:, :], zero_row])
+    )
+
+    return x | y
+
+
+
+
+
+# threshold to characterize blocks as edge/non-edge blocks
+THRESHOLD = 0.002
+# fitting parameter
+BETA = 3.6
+# block size
+BLOCK_HEIGHT, BLOCK_WIDTH = (64, 64)
+# just noticeable widths based on the perceptual experiments
+WIDTH_JNB = np.concatenate([5*np.ones(51), 3*np.ones(205)])
+
+
+def compute(image):
+    # type: (numpy.ndarray) -> float
+    """Compute the sharpness metric for the given data."""
+
+    # convert the image to double for further processing
+    image = image.astype(np.float64)
+
+    # edge detection using canny and sobel canny edge detection is done to
+    # classify the blocks as edge or non-edge blocks and sobel edge
+    # detection is done for the purpose of edge width measurement.
+    from skimage.feature import canny
+    canny_edges = canny(image)
+    sobel_edges = sobel(image)
+
+    # edge width calculation
+    marziliano_widths = marziliano_method(sobel_edges, image)
+
+    # sharpness metric calculation
+    return _calculate_sharpness_metric(image, canny_edges, marziliano_widths)
+
+
+def marziliano_method(edges, image):
+    # type: (numpy.ndarray, numpy.ndarray) -> numpy.ndarray
+    """
+    Calculate the widths of the given edges.
+
+    :return: A matrix with the same dimensions as the given image with 0's at
+        non-edge locations and edge-widths at the edge locations.
+    """
+
+    # `edge_widths` consists of zero and non-zero values. A zero value
+    # indicates that there is no edge at that position and a non-zero value
+    # indicates that there is an edge at that position and the value itself
+    # gives the edge width.
+    edge_widths = np.zeros(image.shape)
+
+    # find the gradient for the image
+    gradient_y, gradient_x = np.gradient(image)
+
+    # dimensions of the image
+    img_height, img_width = image.shape
+
+    # holds the angle information of the edges
+    edge_angles = np.zeros(image.shape)
+
+    # calculate the angle of the edges
+    for row in range(img_height):
+        for col in range(img_width):
+            if gradient_x[row, col] != 0:
+                edge_angles[row, col] = atan2(gradient_y[row, col], gradient_x[row, col]) * (180 / pi)
+            elif gradient_x[row, col] == 0 and gradient_y[row, col] == 0:
+                edge_angles[row,col] = 0
+            elif gradient_x[row, col] == 0 and gradient_y[row, col] == pi/2:
+                edge_angles[row, col] = 90
+
+
+    if np.any(edge_angles):
+
+        # quantize the angle
+        quantized_angles = 45 * np.round(edge_angles / 45)
+
+        for row in range(1, img_height - 1):
+            for col in range(1, img_width - 1):
+                if edges[row, col] == 1:
+
+                    # gradient angle = 180 or -180
+                    if quantized_angles[row, col] == 180 or quantized_angles[row, col] == -180:
+                        for margin in range(100 + 1):
+                            inner_border = (col - 1) - margin
+                            outer_border = (col - 2) - margin
+
+                            # outside image or intensity increasing from left to right
+                            if outer_border < 0 or (image[row, outer_border] - image[row, inner_border]) <= 0:
+                                break
+
+                        width_left = margin + 1
+
+                        for margin in range(100 + 1):
+                            inner_border = (col + 1) + margin
+                            outer_border = (col + 2) + margin
+
+                            # outside image or intensity increasing from left to right
+                            if outer_border >= img_width or (image[row, outer_border] - image[row, inner_border]) >= 0:
+                                break
+
+                        width_right = margin + 1
+
+                        edge_widths[row, col] = width_left + width_right
+
+
+                    # gradient angle = 0
+                    if quantized_angles[row, col] == 0:
+                        for margin in range(100 + 1):
+                            inner_border = (col - 1) - margin
+                            outer_border = (col - 2) - margin
+
+                            # outside image or intensity decreasing from left to right
+                            if outer_border < 0 or (image[row, outer_border] - image[row, inner_border]) >= 0:
+                                break
+
+                        width_left = margin + 1
+
+                        for margin in range(100 + 1):
+                            inner_border = (col + 1) + margin
+                            outer_border = (col + 2) + margin
+
+                            # outside image or intensity decreasing from left to right
+                            if outer_border >= img_width or (image[row, outer_border] - image[row, inner_border]) <= 0:
+                                break
+
+                        width_right = margin + 1
+
+                        edge_widths[row, col] = width_right + width_left
+
+    return edge_widths
+
+
+def _calculate_sharpness_metric(image, edges, edge_widths):
+    # type: (numpy.array, numpy.array, numpy.array) -> numpy.float64
+
+    # get the size of image
+    img_height, img_width = image.shape
+
+    total_num_edges = 0
+    hist_pblur = np.zeros(101)
+
+    # maximum block indices
+    num_blocks_vertically = int(img_height / BLOCK_HEIGHT)
+    num_blocks_horizontally = int(img_width / BLOCK_WIDTH)
+
+    #  loop over the blocks
+    for i in range(num_blocks_vertically):
+        for j in range(num_blocks_horizontally):
+
+            # get the row and col indices for the block pixel positions
+            rows = slice(BLOCK_HEIGHT * i, BLOCK_HEIGHT * (i + 1))
+            cols = slice(BLOCK_WIDTH * j, BLOCK_WIDTH * (j + 1))
+
+            if is_edge_block(edges[rows, cols], THRESHOLD):
+                block_widths = edge_widths[rows, cols]
+                # rotate block to simulate column-major boolean indexing
+                block_widths = np.rot90(np.flipud(block_widths), 3)
+                block_widths = block_widths[block_widths != 0]
+
+                block_contrast = get_block_contrast(image[rows, cols])
+                block_jnb = WIDTH_JNB[block_contrast]
+
+                # calculate the probability of blur detection at the edges
+                # detected in the block
+                prob_blur_detection = 1 - np.exp(-abs(block_widths/block_jnb) ** BETA)
+
+                # update the statistics using the block information
+                for probability in prob_blur_detection:
+                    bucket = int(round(probability * 100))
+                    hist_pblur[bucket] += 1
+                    total_num_edges += 1
+
+    # normalize the pdf
+    if total_num_edges > 0:
+        hist_pblur = hist_pblur / total_num_edges
+
+    # calculate the sharpness metric
+    return np.sum(hist_pblur[:64])
+
+
+def is_edge_block(block, threshold):
+    # type: (numpy.ndarray, float) -> bool
+    """Decide whether the given block is an edge block."""
+    return np.count_nonzero(block) > (block.size * threshold)
+
+
+def get_block_contrast(block):
+    # type: (numpy.ndarray) -> int
+    return int(np.max(block) - np.min(block))
+
+
+def estimate_sharpness(image):
+    if image.ndim == 3:
+        if image.shape[2] > 1:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        else:
+            image = image[...,0]
+        
+    return compute(image)

face_detect/core/imagelib/filters.py

@@ -0,0 +1,245 @@
+import numpy as np
+from .blursharpen import LinearMotionBlur, blursharpen
+import cv2
+
+def apply_random_rgb_levels(img, mask=None, rnd_state=None):
+    if rnd_state is None:
+        rnd_state = np.random
+    np_rnd = rnd_state.rand
+
+    inBlack  = np.array([np_rnd()*0.25    , np_rnd()*0.25    , np_rnd()*0.25], dtype=np.float32)
+    inWhite  = np.array([1.0-np_rnd()*0.25, 1.0-np_rnd()*0.25, 1.0-np_rnd()*0.25], dtype=np.float32)
+    inGamma  = np.array([0.5+np_rnd(), 0.5+np_rnd(), 0.5+np_rnd()], dtype=np.float32)
+
+    outBlack  = np.array([np_rnd()*0.25    , np_rnd()*0.25    , np_rnd()*0.25], dtype=np.float32)
+    outWhite  = np.array([1.0-np_rnd()*0.25, 1.0-np_rnd()*0.25, 1.0-np_rnd()*0.25], dtype=np.float32)
+
+    result = np.clip( (img - inBlack) / (inWhite - inBlack), 0, 1 )
+    result = ( result ** (1/inGamma) ) *  (outWhite - outBlack) + outBlack
+    result = np.clip(result, 0, 1)
+
+    if mask is not None:
+        result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_hsv_shift(img, mask=None, rnd_state=None):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    h, s, v = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))
+    h = ( h + rnd_state.randint(360) ) % 360
+    s = np.clip ( s + rnd_state.random()-0.5, 0, 1 )
+    v = np.clip ( v + rnd_state.random()-0.5, 0, 1 )
+
+    result = np.clip( cv2.cvtColor(cv2.merge([h, s, v]), cv2.COLOR_HSV2BGR) , 0, 1 )
+    if mask is not None:
+        result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_sharpen( img, chance, kernel_max_size, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    sharp_rnd_kernel = rnd_state.randint(kernel_max_size)+1
+
+    result = img
+    if rnd_state.randint(100) < np.clip(chance, 0, 100):
+        if rnd_state.randint(2) == 0:
+            result = blursharpen(result, 1, sharp_rnd_kernel, rnd_state.randint(10) )
+        else:
+            result = blursharpen(result, 2, sharp_rnd_kernel, rnd_state.randint(50) )
+                
+        if mask is not None:
+            result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_motion_blur( img, chance, mb_max_size, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    mblur_rnd_kernel = rnd_state.randint(mb_max_size)+1
+    mblur_rnd_deg    = rnd_state.randint(360)
+
+    result = img
+    if rnd_state.randint(100) < np.clip(chance, 0, 100):
+        result = LinearMotionBlur (result, mblur_rnd_kernel, mblur_rnd_deg )
+        if mask is not None:
+            result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_gaussian_blur( img, chance, kernel_max_size, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    result = img
+    if rnd_state.randint(100) < np.clip(chance, 0, 100):
+        gblur_rnd_kernel = rnd_state.randint(kernel_max_size)*2+1
+        result = cv2.GaussianBlur(result, (gblur_rnd_kernel,)*2 , 0)
+        if mask is not None:
+            result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_resize( img, chance, max_size_per, interpolation=cv2.INTER_LINEAR, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    result = img
+    if rnd_state.randint(100) < np.clip(chance, 0, 100):
+        h,w,c = result.shape
+
+        trg = rnd_state.rand()
+        rw = w - int( trg * int(w*(max_size_per/100.0)) )
+        rh = h - int( trg * int(h*(max_size_per/100.0)) )
+
+        result = cv2.resize (result, (rw,rh), interpolation=interpolation )
+        result = cv2.resize (result, (w,h), interpolation=interpolation )
+        if mask is not None:
+            result = img*(1-mask) + result*mask
+
+    return result
+
+def apply_random_nearest_resize( img, chance, max_size_per, mask=None, rnd_state=None ):
+    return apply_random_resize( img, chance, max_size_per, interpolation=cv2.INTER_NEAREST, mask=mask, rnd_state=rnd_state )
+
+def apply_random_bilinear_resize( img, chance, max_size_per, mask=None, rnd_state=None ):
+    return apply_random_resize( img, chance, max_size_per, interpolation=cv2.INTER_LINEAR, mask=mask, rnd_state=rnd_state )
+
+def apply_random_jpeg_compress( img, chance, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    result = img
+    if rnd_state.randint(100) < np.clip(chance, 0, 100):
+        h,w,c = result.shape
+
+        quality = rnd_state.randint(10,101)
+
+        ret, result = cv2.imencode('.jpg', np.clip(img*255, 0,255).astype(np.uint8), [int(cv2.IMWRITE_JPEG_QUALITY), quality] )
+        if ret == True:
+            result = cv2.imdecode(result, flags=cv2.IMREAD_UNCHANGED)
+            result = result.astype(np.float32) / 255.0
+            if mask is not None:
+                result = img*(1-mask) + result*mask
+
+    return result
+    
+def apply_random_overlay_triangle( img, max_alpha, mask=None, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    h,w,c = img.shape
+    pt1 = [rnd_state.randint(w), rnd_state.randint(h) ]
+    pt2 = [rnd_state.randint(w), rnd_state.randint(h) ]
+    pt3 = [rnd_state.randint(w), rnd_state.randint(h) ]
+    
+    alpha = rnd_state.uniform()*max_alpha
+    
+    tri_mask = cv2.fillPoly( np.zeros_like(img), [ np.array([pt1,pt2,pt3], np.int32) ], (alpha,)*c )
+    
+    if rnd_state.randint(2) == 0:
+        result = np.clip(img+tri_mask, 0, 1)
+    else:
+        result = np.clip(img-tri_mask, 0, 1)
+    
+    if mask is not None:
+        result = img*(1-mask) + result*mask
+
+    return result
+    
+def _min_resize(x, m):
+    if x.shape[0] < x.shape[1]:
+        s0 = m
+        s1 = int(float(m) / float(x.shape[0]) * float(x.shape[1]))
+    else:
+        s0 = int(float(m) / float(x.shape[1]) * float(x.shape[0]))
+        s1 = m
+    new_max = min(s1, s0)
+    raw_max = min(x.shape[0], x.shape[1])
+    return cv2.resize(x, (s1, s0), interpolation=cv2.INTER_LANCZOS4)
+    
+def _d_resize(x, d, fac=1.0):
+    new_min = min(int(d[1] * fac), int(d[0] * fac))
+    raw_min = min(x.shape[0], x.shape[1])
+    if new_min < raw_min:
+        interpolation = cv2.INTER_AREA
+    else:
+        interpolation = cv2.INTER_LANCZOS4
+    y = cv2.resize(x, (int(d[1] * fac), int(d[0] * fac)), interpolation=interpolation)
+    return y
+    
+def _get_image_gradient(dist):
+    cols = cv2.filter2D(dist, cv2.CV_32F, np.array([[-1, 0, +1], [-2, 0, +2], [-1, 0, +1]]))
+    rows = cv2.filter2D(dist, cv2.CV_32F, np.array([[-1, -2, -1], [0, 0, 0], [+1, +2, +1]]))
+    return cols, rows
+
+def _generate_lighting_effects(content):
+    h512 = content
+    h256 = cv2.pyrDown(h512)
+    h128 = cv2.pyrDown(h256)
+    h64 = cv2.pyrDown(h128)
+    h32 = cv2.pyrDown(h64)
+    h16 = cv2.pyrDown(h32)
+    c512, r512 = _get_image_gradient(h512)
+    c256, r256 = _get_image_gradient(h256)
+    c128, r128 = _get_image_gradient(h128)
+    c64, r64 = _get_image_gradient(h64)
+    c32, r32 = _get_image_gradient(h32)
+    c16, r16 = _get_image_gradient(h16)
+    c = c16
+    c = _d_resize(cv2.pyrUp(c), c32.shape) * 4.0 + c32
+    c = _d_resize(cv2.pyrUp(c), c64.shape) * 4.0 + c64
+    c = _d_resize(cv2.pyrUp(c), c128.shape) * 4.0 + c128
+    c = _d_resize(cv2.pyrUp(c), c256.shape) * 4.0 + c256
+    c = _d_resize(cv2.pyrUp(c), c512.shape) * 4.0 + c512
+    r = r16
+    r = _d_resize(cv2.pyrUp(r), r32.shape) * 4.0 + r32
+    r = _d_resize(cv2.pyrUp(r), r64.shape) * 4.0 + r64
+    r = _d_resize(cv2.pyrUp(r), r128.shape) * 4.0 + r128
+    r = _d_resize(cv2.pyrUp(r), r256.shape) * 4.0 + r256
+    r = _d_resize(cv2.pyrUp(r), r512.shape) * 4.0 + r512
+    coarse_effect_cols = c
+    coarse_effect_rows = r
+    EPS = 1e-10
+
+    max_effect = np.max((coarse_effect_cols**2 + coarse_effect_rows**2)**0.5, axis=0, keepdims=True, ).max(1, keepdims=True)
+    coarse_effect_cols = (coarse_effect_cols + EPS) / (max_effect + EPS)
+    coarse_effect_rows = (coarse_effect_rows + EPS) / (max_effect + EPS)
+
+    return np.stack([ np.zeros_like(coarse_effect_rows), coarse_effect_rows, coarse_effect_cols], axis=-1)
+    
+def apply_random_relight(img, mask=None, rnd_state=None):
+    if rnd_state is None:
+        rnd_state = np.random
+        
+    def_img = img
+        
+    if rnd_state.randint(2) == 0:
+        light_pos_y = 1.0 if rnd_state.randint(2) == 0 else -1.0
+        light_pos_x = rnd_state.uniform()*2-1.0
+    else:
+        light_pos_y = rnd_state.uniform()*2-1.0
+        light_pos_x = 1.0 if rnd_state.randint(2) == 0 else -1.0
+                    
+    light_source_height = 0.3*rnd_state.uniform()*0.7
+    light_intensity = 1.0+rnd_state.uniform()
+    ambient_intensity = 0.5
+    
+    light_source_location = np.array([[[light_source_height, light_pos_y, light_pos_x ]]], dtype=np.float32)
+    light_source_direction = light_source_location / np.sqrt(np.sum(np.square(light_source_location)))
+
+    lighting_effect = _generate_lighting_effects(img)
+    lighting_effect = np.sum(lighting_effect * light_source_direction, axis=-1).clip(0, 1)
+    lighting_effect = np.mean(lighting_effect, axis=-1, keepdims=True)
+
+    result = def_img * (ambient_intensity + lighting_effect * light_intensity) #light_source_color
+    result = np.clip(result, 0, 1)
+    
+    if mask is not None:
+        result = def_img*(1-mask) + result*mask
+    
+    return result

face_detect/core/imagelib/morph.py

@@ -0,0 +1,37 @@
+import numpy as np
+import cv2
+from scipy.spatial import Delaunay
+
+
+def applyAffineTransform(src, srcTri, dstTri, size) :
+    warpMat = cv2.getAffineTransform( np.float32(srcTri), np.float32(dstTri) )
+    return cv2.warpAffine( src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )
+
+def morphTriangle(dst_img, src_img, st, dt) :
+    (h,w,c) = dst_img.shape
+    sr = np.array( cv2.boundingRect(np.float32(st)) )
+    dr = np.array( cv2.boundingRect(np.float32(dt)) )
+    sRect = st - sr[0:2]
+    dRect = dt - dr[0:2]
+    d_mask = np.zeros((dr[3], dr[2], c), dtype = np.float32)
+    cv2.fillConvexPoly(d_mask, np.int32(dRect), (1.0,)*c, 8, 0);
+    imgRect = src_img[sr[1]:sr[1] + sr[3], sr[0]:sr[0] + sr[2]]
+    size = (dr[2], dr[3])
+    warpImage1 = applyAffineTransform(imgRect, sRect, dRect, size)
+
+    if c == 1:
+        warpImage1 = np.expand_dims( warpImage1, -1 )
+
+    dst_img[dr[1]:dr[1]+dr[3], dr[0]:dr[0]+dr[2]] = dst_img[dr[1]:dr[1]+dr[3], dr[0]:dr[0]+dr[2]]*(1-d_mask) + warpImage1 * d_mask
+
+def morph_by_points (image, sp, dp):
+    if sp.shape != dp.shape:
+        raise ValueError ('morph_by_points() sp.shape != dp.shape')
+    (h,w,c) = image.shape
+
+    result_image = np.zeros(image.shape, dtype = image.dtype)
+
+    for tri in Delaunay(dp).simplices:
+        morphTriangle(result_image, image, sp[tri], dp[tri])
+
+    return result_image

face_detect/core/imagelib/reduce_colors.py

@@ -0,0 +1,14 @@
+import numpy as np
+import cv2
+from PIL import Image
+
+#n_colors = [0..256]
+def reduce_colors (img_bgr, n_colors):
+    img_rgb = (img_bgr[...,::-1] * 255.0).astype(np.uint8)
+    img_rgb_pil = Image.fromarray(img_rgb)
+    img_rgb_pil_p = img_rgb_pil.convert('P', palette=Image.ADAPTIVE, colors=n_colors)
+
+    img_rgb_p = img_rgb_pil_p.convert('RGB')
+    img_bgr = cv2.cvtColor( np.array(img_rgb_p, dtype=np.float32) / 255.0, cv2.COLOR_RGB2BGR )
+
+    return img_bgr

face_detect/core/imagelib/sd/__init__.py

@@ -0,0 +1,2 @@
+from .draw import circle_faded, random_circle_faded, bezier, random_bezier_split_faded, random_faded
+from .calc import *

face_detect/core/imagelib/sd/calc.py

@@ -0,0 +1,25 @@
+import numpy as np
+import numpy.linalg as npla
+
+def dist_to_edges(pts, pt, is_closed=False):
+    """
+    returns array of dist from pt to edge and projection pt to edges
+    """
+    if is_closed:
+        a = pts
+        b = np.concatenate( (pts[1:,:], pts[0:1,:]), axis=0 )
+    else:
+        a = pts[:-1,:]
+        b = pts[1:,:]
+
+    pa = pt-a
+    ba = b-a
+    
+    div = np.einsum('ij,ij->i', ba, ba)
+    div[div==0]=1
+    h = np.clip( np.einsum('ij,ij->i', pa, ba) / div, 0, 1 )
+    
+    x = npla.norm ( pa - ba*h[...,None], axis=1 )
+    
+    return x, a+ba*h[...,None]
+    

face_detect/core/imagelib/sd/draw.py

@@ -0,0 +1,200 @@
+"""
+Signed distance drawing functions using numpy.
+"""
+import math
+
+import numpy as np
+from numpy import linalg as npla
+
+
+def vector2_dot(a,b):
+    return a[...,0]*b[...,0]+a[...,1]*b[...,1]
+    
+def vector2_dot2(a):
+    return a[...,0]*a[...,0]+a[...,1]*a[...,1]
+    
+def vector2_cross(a,b):
+    return a[...,0]*b[...,1]-a[...,1]*b[...,0]
+
+
+def circle_faded( wh, center, fade_dists ):
+    """
+    returns drawn circle in [h,w,1] output range [0..1.0] float32
+    
+    wh         = [w,h]                      resolution
+    center     = [x,y]                      center of circle
+    fade_dists = [fade_start, fade_end]     fade values
+    """
+    w,h = wh      
+    
+    pts = np.empty( (h,w,2), dtype=np.float32 )
+    pts[...,0] = np.arange(w)[:,None]
+    pts[...,1] = np.arange(h)[None,:]
+    
+    pts = pts.reshape ( (h*w, -1) )
+
+    pts_dists = np.abs ( npla.norm(pts-center, axis=-1) )
+    
+    if fade_dists[1] == 0:
+        fade_dists[1] = 1
+        
+    pts_dists = ( pts_dists - fade_dists[0] ) / fade_dists[1]
+        
+    pts_dists = np.clip( 1-pts_dists, 0, 1)
+    
+    return pts_dists.reshape ( (h,w,1) ).astype(np.float32)
+
+
+def bezier( wh, A, B, C ):
+    """
+    returns drawn bezier in [h,w,1] output range float32, 
+    every pixel contains signed distance to bezier line
+
+        wh      [w,h]       resolution
+        A,B,C   points [x,y]
+    """
+    
+    width,height = wh
+    
+    A = np.float32(A)
+    B = np.float32(B)
+    C = np.float32(C)
+    
+
+    pos = np.empty( (height,width,2), dtype=np.float32 )
+    pos[...,0] = np.arange(width)[:,None]
+    pos[...,1] = np.arange(height)[None,:]
+    
+
+    a = B-A
+    b = A - 2.0*B + C
+    c = a * 2.0
+    d = A - pos
+
+    b_dot = vector2_dot(b,b)
+    if b_dot == 0.0:
+        return np.zeros( (height,width), dtype=np.float32 )
+    
+    kk = 1.0 / b_dot
+
+    kx = kk * vector2_dot(a,b)
+    ky = kk * (2.0*vector2_dot(a,a)+vector2_dot(d,b))/3.0;
+    kz = kk * vector2_dot(d,a);
+
+    res = 0.0;
+    sgn = 0.0;
+
+    p = ky - kx*kx;
+    
+    p3 = p*p*p;
+    q = kx*(2.0*kx*kx - 3.0*ky) + kz;
+    h = q*q + 4.0*p3;
+    
+    hp_sel = h >= 0.0
+    
+    hp_p = h[hp_sel]
+    hp_p = np.sqrt(hp_p)
+    
+    hp_x = ( np.stack( (hp_p,-hp_p), -1) -q[hp_sel,None] ) / 2.0
+    hp_uv = np.sign(hp_x) * np.power( np.abs(hp_x), [1.0/3.0, 1.0/3.0] )
+    hp_t = np.clip( hp_uv[...,0] + hp_uv[...,1] - kx, 0.0, 1.0 )
+    
+    hp_t = hp_t[...,None]
+    hp_q = d[hp_sel]+(c+b*hp_t)*hp_t
+    hp_res = vector2_dot2(hp_q)
+    hp_sgn = vector2_cross(c+2.0*b*hp_t,hp_q)
+    
+    hl_sel = h < 0.0
+    
+    hl_q = q[hl_sel]
+    hl_p = p[hl_sel]
+    hl_z = np.sqrt(-hl_p)
+    hl_v = np.arccos( hl_q / (hl_p*hl_z*2.0)) / 3.0
+    
+    hl_m = np.cos(hl_v)
+    hl_n = np.sin(hl_v)*1.732050808;
+    
+    hl_t = np.clip( np.stack( (hl_m+hl_m,-hl_n-hl_m,hl_n-hl_m), -1)*hl_z[...,None]-kx, 0.0, 1.0 );
+    
+    hl_d = d[hl_sel]
+    
+    hl_qx = hl_d+(c+b*hl_t[...,0:1])*hl_t[...,0:1]
+    
+    hl_dx = vector2_dot2(hl_qx)
+    hl_sx = vector2_cross(c+2.0*b*hl_t[...,0:1], hl_qx)
+    
+    hl_qy = hl_d+(c+b*hl_t[...,1:2])*hl_t[...,1:2]
+    hl_dy = vector2_dot2(hl_qy)
+    hl_sy = vector2_cross(c+2.0*b*hl_t[...,1:2],hl_qy);
+    
+    hl_dx_l_dy = hl_dx<hl_dy
+    hl_dx_ge_dy = hl_dx>=hl_dy
+    
+    hl_res = np.empty_like(hl_dx)
+    hl_res[hl_dx_l_dy] = hl_dx[hl_dx_l_dy]
+    hl_res[hl_dx_ge_dy] = hl_dy[hl_dx_ge_dy]
+    
+    hl_sgn = np.empty_like(hl_sx)
+    hl_sgn[hl_dx_l_dy] = hl_sx[hl_dx_l_dy]
+    hl_sgn[hl_dx_ge_dy] = hl_sy[hl_dx_ge_dy]
+    
+    res = np.empty( (height, width), np.float32 )
+    res[hp_sel] = hp_res
+    res[hl_sel] = hl_res
+    
+    sgn = np.empty( (height, width), np.float32 )
+    sgn[hp_sel] = hp_sgn
+    sgn[hl_sel] = hl_sgn
+    
+    sgn = np.sign(sgn)
+    res = np.sqrt(res)*sgn
+    
+    return res[...,None]
+        
+def random_faded(wh):
+    """
+    apply one of them:
+     random_circle_faded
+     random_bezier_split_faded
+    """
+    rnd = np.random.randint(2)
+    if rnd == 0:
+        return random_circle_faded(wh)
+    elif rnd == 1:
+        return random_bezier_split_faded(wh)
+    
+def random_circle_faded ( wh, rnd_state=None ):
+    if rnd_state is None:
+        rnd_state = np.random
+        
+    w,h = wh
+    wh_max = max(w,h)
+    fade_start = rnd_state.randint(wh_max)
+    fade_end = fade_start + rnd_state.randint(wh_max- fade_start)
+    
+    return circle_faded (wh, [ rnd_state.randint(h), rnd_state.randint(w) ], 
+                             [fade_start, fade_end] ) 
+                             
+def random_bezier_split_faded( wh ):
+    width, height = wh
+    
+    degA = np.random.randint(360)
+    degB = np.random.randint(360)
+    degC = np.random.randint(360)
+    
+    deg_2_rad = math.pi / 180.0
+    
+    center = np.float32([width / 2.0, height / 2.0])
+    
+    radius = max(width, height)
+    
+    A = center + radius*np.float32([ math.sin( degA * deg_2_rad), math.cos( degA * deg_2_rad) ] ) 
+    B = center + np.random.randint(radius)*np.float32([ math.sin( degB * deg_2_rad), math.cos( degB * deg_2_rad) ] ) 
+    C = center + radius*np.float32([ math.sin( degC * deg_2_rad), math.cos( degC * deg_2_rad) ] ) 
+    
+    x = bezier( (width,height), A, B, C )
+    
+    x = x / (1+np.random.randint(radius)) + 0.5
+    
+    x = np.clip(x, 0, 1)
+    return x

face_detect/core/imagelib/warp.py

@@ -0,0 +1,72 @@
+import numpy as np
+import cv2
+from face_detect.core import randomex
+
+def gen_warp_params (w, flip=False, rotation_range=[-10,10], scale_range=[-0.5, 0.5], tx_range=[-0.05, 0.05], ty_range=[-0.05, 0.05], rnd_state=None  ):
+    if rnd_state is None:
+        rnd_state = np.random
+
+    rw = None
+    if w < 64:        
+        rw = w
+        w = 64
+        
+    rotation = rnd_state.uniform( rotation_range[0], rotation_range[1] )
+    scale = rnd_state.uniform(1 +scale_range[0], 1 +scale_range[1])
+    tx = rnd_state.uniform( tx_range[0], tx_range[1] )
+    ty = rnd_state.uniform( ty_range[0], ty_range[1] )
+    p_flip = flip and rnd_state.randint(10) < 4
+
+    #random warp by grid
+    cell_size = [ w // (2**i) for i in range(1,4) ] [ rnd_state.randint(3) ]
+    cell_count = w // cell_size + 1
+
+    grid_points = np.linspace( 0, w, cell_count)
+    mapx = np.broadcast_to(grid_points, (cell_count, cell_count)).copy()
+    mapy = mapx.T
+
+    mapx[1:-1,1:-1] = mapx[1:-1,1:-1] + randomex.random_normal( size=(cell_count-2, cell_count-2) )*(cell_size*0.24)
+    mapy[1:-1,1:-1] = mapy[1:-1,1:-1] + randomex.random_normal( size=(cell_count-2, cell_count-2) )*(cell_size*0.24)
+
+    half_cell_size = cell_size // 2
+
+    mapx = cv2.resize(mapx, (w+cell_size,)*2 )[half_cell_size:-half_cell_size,half_cell_size:-half_cell_size].astype(np.float32)
+    mapy = cv2.resize(mapy, (w+cell_size,)*2 )[half_cell_size:-half_cell_size,half_cell_size:-half_cell_size].astype(np.float32)
+
+    #random transform
+    random_transform_mat = cv2.getRotationMatrix2D((w // 2, w // 2), rotation, scale)
+    random_transform_mat[:, 2] += (tx*w, ty*w)
+
+    params = dict()
+    params['mapx'] = mapx
+    params['mapy'] = mapy
+    params['rmat'] = random_transform_mat
+    u_mat = random_transform_mat.copy()
+    u_mat[:,2] /= w
+    params['umat'] = u_mat
+    params['w'] = w
+    params['rw'] = rw
+    params['flip'] = p_flip
+
+    return params
+
+def warp_by_params (params, img, can_warp, can_transform, can_flip, border_replicate, cv2_inter=cv2.INTER_CUBIC):
+    rw = params['rw']
+    
+    if (can_warp or can_transform) and rw is not None:
+        img = cv2.resize(img, (64,64), interpolation=cv2_inter)
+        
+    if can_warp:
+        img = cv2.remap(img, params['mapx'], params['mapy'], cv2_inter )
+    if can_transform:
+        img = cv2.warpAffine( img, params['rmat'], (params['w'], params['w']), borderMode=(cv2.BORDER_REPLICATE if border_replicate else cv2.BORDER_CONSTANT), flags=cv2_inter )
+    
+    
+    if (can_warp or can_transform) and rw is not None:
+        img = cv2.resize(img, (rw,rw), interpolation=cv2_inter)
+    
+    if len(img.shape) == 2:
+        img = img[...,None]
+    if can_flip and params['flip']:
+        img = img[:,::-1,...]
+    return img

face_detect/core/leras/__init__.py

@@ -0,0 +1 @@
+from .nn import nn

face_detect/core/leras/archis/ArchiBase.py

@@ -0,0 +1,17 @@
+from face_feature.core.leras import nn
+
+class ArchiBase():
+    
+    def __init__(self, *args, name=None, **kwargs):
+        self.name=name
+        
+       
+    #overridable 
+    def flow(self, *args, **kwargs):
+        raise Exception("this archi does not support flow. Use model classes directly.")
+    
+    #overridable
+    def get_weights(self):
+        pass
+    
+nn.ArchiBase = ArchiBase

face_detect/core/leras/archis/DeepFakeArchi.py

@@ -0,0 +1,223 @@
+from face_feature.core.leras import nn
+
+tf = nn.tf
+
+
+class DeepFakeArchi(nn.ArchiBase):
+    """
+    resolution
+
+    mod     None - default
+            'quick'
+    """
+
+    def __init__(self, resolution, mod=None, opts=None):
+        super().__init__()
+
+        if opts is None:
+            opts = ''
+
+        if mod is None:
+            class Downscale(nn.ModelBase):
+                def __init__(self, in_ch, out_ch, kernel_size=5, *kwargs):
+                    self.in_ch = in_ch
+                    self.out_ch = out_ch
+                    self.kernel_size = kernel_size
+                    super().__init__(*kwargs)
+
+                def on_build(self, *args, **kwargs):
+                    self.conv1 = nn.Conv2D(self.in_ch, self.out_ch, kernel_size=self.kernel_size, strides=2,
+                                           padding='SAME')
+
+                def forward(self, x):
+                    x = self.conv1(x)
+                    x = tf.nn.leaky_relu(x, 0.1)
+                    return x
+
+                def get_out_ch(self):
+                    return self.out_ch
+
+            class DownscaleBlock(nn.ModelBase):
+                def on_build(self, in_ch, ch, n_downscales, kernel_size):
+                    self.downs = []
+
+                    last_ch = in_ch
+                    for i in range(n_downscales):
+                        cur_ch = ch * (min(2 ** i, 8))
+                        self.downs.append(Downscale(last_ch, cur_ch, kernel_size=kernel_size))
+                        last_ch = self.downs[-1].get_out_ch()
+
+                def forward(self, inp):
+                    x = inp
+                    for down in self.downs:
+                        x = down(x)
+                    return x
+
+            class Upscale(nn.ModelBase):
+                def on_build(self, in_ch, out_ch, kernel_size=3):
+                    self.conv1 = nn.Conv2D(in_ch, out_ch * 4, kernel_size=kernel_size, padding='SAME')
+
+                def forward(self, x):
+                    x = self.conv1(x)
+                    x = tf.nn.leaky_relu(x, 0.1)
+                    x = nn.depth_to_space(x, 2)
+                    return x
+
+            class ResidualBlock(nn.ModelBase):
+                def on_build(self, ch, kernel_size=3):
+                    self.conv1 = nn.Conv2D(ch, ch, kernel_size=kernel_size, padding='SAME')
+                    self.conv2 = nn.Conv2D(ch, ch, kernel_size=kernel_size, padding='SAME')
+
+                def forward(self, inp):
+                    x = self.conv1(inp)
+                    x = tf.nn.leaky_relu(x, 0.2)
+                    x = self.conv2(x)
+                    x = tf.nn.leaky_relu(inp + x, 0.2)
+                    return x
+
+            class Encoder(nn.ModelBase):
+                def __init__(self, in_ch, e_ch, **kwargs):
+                    self.in_ch = in_ch
+                    self.e_ch = e_ch
+                    super().__init__(**kwargs)
+
+                def on_build(self):
+                    self.down1 = DownscaleBlock(self.in_ch, self.e_ch, n_downscales=4, kernel_size=5)
+
+                def forward(self, inp):
+                    return nn.flatten(self.down1(inp))
+
+                def get_out_res(self, res):
+                    return res // (2 ** 4)
+
+                def get_out_ch(self):
+                    return self.e_ch * 8
+
+            lowest_dense_res = resolution // (32 if 'd' in opts else 16)
+
+            class Inter(nn.ModelBase):
+                def __init__(self, in_ch, ae_ch, ae_out_ch, **kwargs):
+                    self.in_ch, self.ae_ch, self.ae_out_ch = in_ch, ae_ch, ae_out_ch
+                    super().__init__(**kwargs)
+
+                def on_build(self):
+                    in_ch, ae_ch, ae_out_ch = self.in_ch, self.ae_ch, self.ae_out_ch
+                    if 'u' in opts:
+                        self.dense_norm = nn.DenseNorm()
+
+                    self.dense1 = nn.Dense(in_ch, ae_ch)
+                    self.dense2 = nn.Dense(ae_ch, lowest_dense_res * lowest_dense_res * ae_out_ch)
+                    self.upscale1 = Upscale(ae_out_ch, ae_out_ch)
+
+                def forward(self, inp):
+                    x = inp
+                    if 'u' in opts:
+                        x = self.dense_norm(x)
+                    x = self.dense1(x)
+                    x = self.dense2(x)
+                    x = nn.reshape_4D(x, lowest_dense_res, lowest_dense_res, self.ae_out_ch)
+                    x = self.upscale1(x)
+                    return x
+
+                def get_out_res(self):
+                    return lowest_dense_res * 2
+
+                def get_out_ch(self):
+                    return self.ae_out_ch
+
+            class Decoder(nn.ModelBase):
+                def on_build(self, in_ch, d_ch, d_mask_ch):
+                    self.upscale0 = Upscale(in_ch, d_ch * 8, kernel_size=3)
+                    self.upscale1 = Upscale(d_ch * 8, d_ch * 4, kernel_size=3)
+                    self.upscale2 = Upscale(d_ch * 4, d_ch * 2, kernel_size=3)
+
+                    self.res0 = ResidualBlock(d_ch * 8, kernel_size=3)
+                    self.res1 = ResidualBlock(d_ch * 4, kernel_size=3)
+                    self.res2 = ResidualBlock(d_ch * 2, kernel_size=3)
+
+                    self.out_conv = nn.Conv2D(d_ch * 2, 3, kernel_size=1, padding='SAME')
+
+                    # self.upscalem0 = Upscale(in_ch, d_mask_ch * 8, kernel_size=3)
+                    # self.upscalem1 = Upscale(d_mask_ch * 8, d_mask_ch * 4, kernel_size=3)
+                    # self.upscalem2 = Upscale(d_mask_ch * 4, d_mask_ch * 2, kernel_size=3)
+                    # self.out_convm = nn.Conv2D(d_mask_ch * 2, 1, kernel_size=1, padding='SAME')
+
+                    if 'd' in opts:
+                        self.out_conv1 = nn.Conv2D(d_ch * 2, 3, kernel_size=3, padding='SAME')
+                        self.out_conv2 = nn.Conv2D(d_ch * 2, 3, kernel_size=3, padding='SAME')
+                        self.out_conv3 = nn.Conv2D(d_ch * 2, 3, kernel_size=3, padding='SAME')
+                        # self.upscalem3 = Upscale(d_mask_ch * 2, d_mask_ch * 1, kernel_size=3)
+                        # self.out_convm = nn.Conv2D(d_mask_ch * 1, 1, kernel_size=1, padding='SAME')
+                    else:
+                        # self.out_convm = nn.Conv2D(d_mask_ch * 2, 1, kernel_size=1, padding='SAME')
+                        pass
+
+                def forward(self, inp):
+                    z = inp
+
+                    x = self.upscale0(z)
+                    x = self.res0(x)
+                    x = self.upscale1(x)
+                    x = self.res1(x)
+                    x = self.upscale2(x)
+                    x = self.res2(x)
+
+                    if 'd' in opts:
+                        x0 = tf.nn.sigmoid(self.out_conv(x))
+                        x0 = nn.upsample2d(x0)
+                        x1 = tf.nn.sigmoid(self.out_conv1(x))
+                        x1 = nn.upsample2d(x1)
+                        x2 = tf.nn.sigmoid(self.out_conv2(x))
+                        x2 = nn.upsample2d(x2)
+                        x3 = tf.nn.sigmoid(self.out_conv3(x))
+                        x3 = nn.upsample2d(x3)
+
+                        if nn.data_format == "NHWC":
+                            tile_cfg = (1, resolution // 2, resolution // 2, 1)
+                        else:
+                            tile_cfg = (1, 1, resolution // 2, resolution // 2)
+
+                        z0 = tf.concat(
+                            (tf.concat((tf.ones((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))), axis=nn.conv2d_spatial_axes[1]),
+                             tf.concat((tf.zeros((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))),
+                                       axis=nn.conv2d_spatial_axes[1])), axis=nn.conv2d_spatial_axes[0])
+
+                        z0 = tf.tile(z0, tile_cfg)
+
+                        z1 = tf.concat(
+                            (tf.concat((tf.zeros((1, 1, 1, 1)), tf.ones((1, 1, 1, 1))), axis=nn.conv2d_spatial_axes[1]),
+                             tf.concat((tf.zeros((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))),
+                                       axis=nn.conv2d_spatial_axes[1])), axis=nn.conv2d_spatial_axes[0])
+                        z1 = tf.tile(z1, tile_cfg)
+
+                        z2 = tf.concat((tf.concat((tf.zeros((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))),
+                                                  axis=nn.conv2d_spatial_axes[1]),
+                                        tf.concat((tf.ones((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))),
+                                                  axis=nn.conv2d_spatial_axes[1])), axis=nn.conv2d_spatial_axes[0])
+                        z2 = tf.tile(z2, tile_cfg)
+
+                        z3 = tf.concat((tf.concat((tf.zeros((1, 1, 1, 1)), tf.zeros((1, 1, 1, 1))),
+                                                  axis=nn.conv2d_spatial_axes[1]),
+                                        tf.concat((tf.zeros((1, 1, 1, 1)), tf.ones((1, 1, 1, 1))),
+                                                  axis=nn.conv2d_spatial_axes[1])), axis=nn.conv2d_spatial_axes[0])
+                        z3 = tf.tile(z3, tile_cfg)
+
+                        x = x0 * z0 + x1 * z1 + x2 * z2 + x3 * z3
+                    else:
+                        x = tf.nn.sigmoid(self.out_conv(x))
+
+                    # m = self.upscalem0(z)
+                    # m = self.upscalem1(m)
+                    # m = self.upscalem2(m)
+                    # if 'd' in opts:
+                    #     m = self.upscalem3(m)
+                    # m = tf.nn.sigmoid(self.out_convm(m))
+
+                    return x
+
+        self.Encoder = Encoder
+        self.Inter = Inter
+        self.Decoder = Decoder
+
+
+nn.DeepFakeArchi = DeepFakeArchi

face_detect/core/leras/archis/__init__.py

@@ -0,0 +1,2 @@
+from .ArchiBase import *
+from .DeepFakeArchi import *

face_detect/core/leras/device.py

@@ -0,0 +1,272 @@
+import sys
+import ctypes
+import os
+import multiprocessing
+import json
+import time
+from pathlib import Path
+# from face_feature.core.interact import interact as io
+
+
+class Device(object):
+    def __init__(self, index, tf_dev_type, name, total_mem, free_mem):
+        self.index = index
+        self.tf_dev_type = tf_dev_type
+        self.name = name
+        
+        self.total_mem = total_mem
+        self.total_mem_gb = total_mem / 1024**3
+        self.free_mem = free_mem
+        self.free_mem_gb = free_mem / 1024**3
+
+    def __str__(self):
+        return f"[{self.index}]:[{self.name}][{self.free_mem_gb:.3}/{self.total_mem_gb :.3}]"
+
+class Devices(object):
+    all_devices = None
+
+    def __init__(self, devices):
+        self.devices = devices
+
+    def __len__(self):
+        return len(self.devices)
+
+    def __getitem__(self, key):
+        result = self.devices[key]
+        if isinstance(key, slice):
+            return Devices(result)
+        return result
+
+    def __iter__(self):
+        for device in self.devices:
+            yield device
+
+    def get_best_device(self):
+        result = None
+        idx_mem = 0
+        for device in self.devices:
+            mem = device.total_mem
+            if mem > idx_mem:
+                result = device
+                idx_mem = mem
+        return result
+
+    def get_worst_device(self):
+        result = None
+        idx_mem = sys.maxsize
+        for device in self.devices:
+            mem = device.total_mem
+            if mem < idx_mem:
+                result = device
+                idx_mem = mem
+        return result
+
+    def get_device_by_index(self, idx):
+        for device in self.devices:
+            if device.index == idx:
+                return device
+        return None
+
+    def get_devices_from_index_list(self, idx_list):
+        result = []
+        for device in self.devices:
+            if device.index in idx_list:
+                result += [device]
+        return Devices(result)
+
+    def get_equal_devices(self, device):
+        device_name = device.name
+        result = []
+        for device in self.devices:
+            if device.name == device_name:
+                result.append (device)
+        return Devices(result)
+
+    def get_devices_at_least_mem(self, totalmemsize_gb):
+        result = []
+        for device in self.devices:
+            if device.total_mem >= totalmemsize_gb*(1024**3):
+                result.append (device)
+        return Devices(result)
+
+    @staticmethod
+    def _get_tf_devices_proc(q : multiprocessing.Queue):
+        
+        if sys.platform[0:3] == 'win':
+            compute_cache_path = Path(os.environ['APPDATA']) / 'NVIDIA' / ('ComputeCache_ALL')
+            os.environ['CUDA_CACHE_PATH'] = str(compute_cache_path)
+            if not compute_cache_path.exists():
+                # io.log_info("Caching GPU kernels...")
+                compute_cache_path.mkdir(parents=True, exist_ok=True)
+                
+        import tensorflow
+        
+        tf_version = tensorflow.version.VERSION
+        #if tf_version is None:
+        #    tf_version = tensorflow.version.GIT_VERSION
+        if tf_version[0] == 'v':
+            tf_version = tf_version[1:]
+        if tf_version[0] == '2':
+            tf = tensorflow.compat.v1
+        else:
+            tf = tensorflow
+        
+        import logging
+        # Disable tensorflow warnings
+        tf_logger = logging.getLogger('tensorflow')
+        tf_logger.setLevel(logging.ERROR)
+
+        from tensorflow.python.client import device_lib
+
+        devices = []
+        
+        physical_devices = device_lib.list_local_devices()
+        physical_devices_f = {}
+        for dev in physical_devices:
+            dev_type = dev.device_type
+            dev_tf_name = dev.name
+            dev_tf_name = dev_tf_name[ dev_tf_name.index(dev_type) : ]
+            
+            dev_idx = int(dev_tf_name.split(':')[-1])
+            
+            if dev_type in ['GPU','DML']:
+                dev_name = dev_tf_name
+                
+                dev_desc = dev.physical_device_desc
+                if len(dev_desc) != 0:
+                    if dev_desc[0] == '{':
+                        dev_desc_json = json.loads(dev_desc)
+                        dev_desc_json_name = dev_desc_json.get('name',None)
+                        if dev_desc_json_name is not None:
+                            dev_name = dev_desc_json_name
+                    else:
+                        for param, value in ( v.split(':') for v in dev_desc.split(',') ):
+                            param = param.strip()
+                            value = value.strip()
+                            if param == 'name':
+                                dev_name = value
+                                break
+                
+                physical_devices_f[dev_idx] = (dev_type, dev_name, dev.memory_limit)
+                        
+        q.put(physical_devices_f)
+        time.sleep(0.1)
+        
+        
+    @staticmethod
+    def initialize_main_env():
+        if int(os.environ.get("NN_DEVICES_INITIALIZED", 0)) != 0:
+            return
+
+        if 'CUDA_VISIBLE_DEVICES' in os.environ.keys():
+            os.environ.pop('CUDA_VISIBLE_DEVICES')
+
+        os.environ['CUDA_​CACHE_​MAXSIZE'] = '2147483647'
+        os.environ['TF_MIN_GPU_MULTIPROCESSOR_COUNT'] = '2'
+        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # tf log errors only
+        
+        q = multiprocessing.Queue()
+        p = multiprocessing.Process(target=Devices._get_tf_devices_proc, args=(q,), daemon=True)
+        p.start()
+        p.join()
+
+        visible_devices = q.get()
+
+        os.environ['NN_DEVICES_INITIALIZED'] = '1'
+        os.environ['NN_DEVICES_COUNT'] = str(len(visible_devices))
+        
+        for i in visible_devices:
+            dev_type, name, total_mem = visible_devices[i]
+
+            os.environ[f'NN_DEVICE_{i}_TF_DEV_TYPE'] = dev_type
+            os.environ[f'NN_DEVICE_{i}_NAME'] = name
+            os.environ[f'NN_DEVICE_{i}_TOTAL_MEM'] = str(total_mem)
+            os.environ[f'NN_DEVICE_{i}_FREE_MEM'] = str(total_mem)
+            
+        
+
+    @staticmethod
+    def getDevices():
+        if Devices.all_devices is None:
+            if int(os.environ.get("NN_DEVICES_INITIALIZED", 0)) != 1:
+                raise Exception("nn devices are not initialized. Run initialize_main_env() in main process.")
+            devices = []
+            for i in range ( int(os.environ['NN_DEVICES_COUNT']) ):
+                devices.append ( Device(index=i,
+                                        tf_dev_type=os.environ[f'NN_DEVICE_{i}_TF_DEV_TYPE'],
+                                        name=os.environ[f'NN_DEVICE_{i}_NAME'],
+                                        total_mem=int(os.environ[f'NN_DEVICE_{i}_TOTAL_MEM']),
+                                        free_mem=int(os.environ[f'NN_DEVICE_{i}_FREE_MEM']), )
+                                )
+            Devices.all_devices = Devices(devices)
+
+        return Devices.all_devices
+
+"""
+
+        
+        # {'name'      : name.split(b'\0', 1)[0].decode(),
+        #     'total_mem' : totalMem.value
+        # }
+
+        
+        
+        
+        
+        return
+
+        
+        
+        
+        min_cc = int(os.environ.get("TF_MIN_REQ_CAP", 35))
+        libnames = ('libcuda.so', 'libcuda.dylib', 'nvcuda.dll')
+        for libname in libnames:
+            try:
+                cuda = ctypes.CDLL(libname)
+            except:
+                continue
+            else:
+                break
+        else:
+            return Devices([])
+
+        nGpus = ctypes.c_int()
+        name = b' ' * 200
+        cc_major = ctypes.c_int()
+        cc_minor = ctypes.c_int()
+        freeMem = ctypes.c_size_t()
+        totalMem = ctypes.c_size_t()
+
+        result = ctypes.c_int()
+        device = ctypes.c_int()
+        context = ctypes.c_void_p()
+        error_str = ctypes.c_char_p()
+
+        devices = []
+
+        if cuda.cuInit(0) == 0 and \
+            cuda.cuDeviceGetCount(ctypes.byref(nGpus)) == 0:
+            for i in range(nGpus.value):
+                if cuda.cuDeviceGet(ctypes.byref(device), i) != 0 or \
+                    cuda.cuDeviceGetName(ctypes.c_char_p(name), len(name), device) != 0 or \
+                    cuda.cuDeviceComputeCapability(ctypes.byref(cc_major), ctypes.byref(cc_minor), device) != 0:
+                    continue
+
+                if cuda.cuCtxCreate_v2(ctypes.byref(context), 0, device) == 0:
+                    if cuda.cuMemGetInfo_v2(ctypes.byref(freeMem), ctypes.byref(totalMem)) == 0:
+                        cc = cc_major.value * 10 + cc_minor.value
+                        if cc >= min_cc:
+                            devices.append ( {'name'      : name.split(b'\0', 1)[0].decode(),
+                                              'total_mem' : totalMem.value,
+                                              'free_mem'  : freeMem.value,
+                                              'cc'        : cc
+                                              })
+                    cuda.cuCtxDetach(context)
+
+        os.environ['NN_DEVICES_COUNT'] = str(len(devices))
+        for i, device in enumerate(devices):
+            os.environ[f'NN_DEVICE_{i}_NAME'] = device['name']
+            os.environ[f'NN_DEVICE_{i}_TOTAL_MEM'] = str(device['total_mem'])
+            os.environ[f'NN_DEVICE_{i}_FREE_MEM'] = str(device['free_mem'])
+            os.environ[f'NN_DEVICE_{i}_CC'] = str(device['cc'])
+"""

face_detect/core/leras/layers/AdaIN.py

@@ -0,0 +1,56 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class AdaIN(nn.LayerBase):
+    """
+    """
+    def __init__(self, in_ch, mlp_ch, kernel_initializer=None, dtype=None, **kwargs):
+        self.in_ch = in_ch
+        self.mlp_ch = mlp_ch
+        self.kernel_initializer = kernel_initializer
+
+        if dtype is None:
+            dtype = nn.floatx
+        self.dtype = dtype
+
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        kernel_initializer = self.kernel_initializer
+        if kernel_initializer is None:
+            kernel_initializer = tf.initializers.he_normal()
+
+        self.weight1   = tf.get_variable("weight1", (self.mlp_ch, self.in_ch), dtype=self.dtype, initializer=kernel_initializer)
+        self.bias1     = tf.get_variable("bias1",   (self.in_ch,), dtype=self.dtype, initializer=tf.initializers.zeros())
+        self.weight2   = tf.get_variable("weight2", (self.mlp_ch, self.in_ch), dtype=self.dtype, initializer=kernel_initializer)
+        self.bias2     = tf.get_variable("bias2",   (self.in_ch,), dtype=self.dtype, initializer=tf.initializers.zeros())
+
+    def get_weights(self):
+        return [self.weight1, self.bias1, self.weight2, self.bias2]
+
+    def forward(self, inputs):
+        x, mlp = inputs
+
+        gamma = tf.matmul(mlp, self.weight1)
+        gamma = tf.add(gamma, tf.reshape(self.bias1, (1,self.in_ch) ) )
+
+        beta = tf.matmul(mlp, self.weight2)
+        beta = tf.add(beta, tf.reshape(self.bias2, (1,self.in_ch) ) )
+
+
+        if nn.data_format == "NHWC":
+            shape = (-1,1,1,self.in_ch)
+        else:
+            shape = (-1,self.in_ch,1,1)
+
+        x_mean = tf.reduce_mean(x, axis=nn.conv2d_spatial_axes, keepdims=True )
+        x_std  = tf.math.reduce_std(x, axis=nn.conv2d_spatial_axes, keepdims=True ) + 1e-5
+
+        x = (x - x_mean) / x_std
+        x *= tf.reshape(gamma, shape)
+
+        x += tf.reshape(beta, shape)
+
+        return x
+
+nn.AdaIN = AdaIN

face_detect/core/leras/layers/BatchNorm2D.py

@@ -0,0 +1,42 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class BatchNorm2D(nn.LayerBase):
+    """
+    currently not for training
+    """
+    def __init__(self, dim, eps=1e-05, momentum=0.1, dtype=None, **kwargs):
+        self.dim = dim
+        self.eps = eps
+        self.momentum = momentum
+        if dtype is None:
+            dtype = nn.floatx
+        self.dtype = dtype
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        self.weight       = tf.get_variable("weight",   (self.dim,), dtype=self.dtype, initializer=tf.initializers.ones() )
+        self.bias         = tf.get_variable("bias",     (self.dim,), dtype=self.dtype, initializer=tf.initializers.zeros() )
+        self.running_mean = tf.get_variable("running_mean", (self.dim,), dtype=self.dtype, initializer=tf.initializers.zeros(), trainable=False )
+        self.running_var  = tf.get_variable("running_var",  (self.dim,), dtype=self.dtype, initializer=tf.initializers.zeros(), trainable=False )
+
+    def get_weights(self):
+        return [self.weight, self.bias, self.running_mean, self.running_var]
+
+    def forward(self, x):
+        if nn.data_format == "NHWC":
+            shape = (1,1,1,self.dim)
+        else:
+            shape = (1,self.dim,1,1)
+
+        weight       = tf.reshape ( self.weight      , shape )
+        bias         = tf.reshape ( self.bias        , shape )
+        running_mean = tf.reshape ( self.running_mean, shape )
+        running_var  = tf.reshape ( self.running_var , shape )
+
+        x = (x - running_mean) / tf.sqrt( running_var + self.eps )
+        x *= weight
+        x += bias
+        return x
+
+nn.BatchNorm2D = BatchNorm2D

face_detect/core/leras/layers/BlurPool.py

@@ -0,0 +1,50 @@
+import numpy as np
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class BlurPool(nn.LayerBase):
+    def __init__(self, filt_size=3, stride=2, **kwargs ):
+
+        if nn.data_format == "NHWC":
+            self.strides = [1,stride,stride,1]
+        else:
+            self.strides = [1,1,stride,stride]
+
+        self.filt_size = filt_size
+        pad = [ int(1.*(filt_size-1)/2), int(np.ceil(1.*(filt_size-1)/2)) ]
+
+        if nn.data_format == "NHWC":
+            self.padding = [ [0,0], pad, pad, [0,0] ]
+        else:
+            self.padding = [ [0,0], [0,0], pad, pad ]
+
+        if(self.filt_size==1):
+            a = np.array([1.,])
+        elif(self.filt_size==2):
+            a = np.array([1., 1.])
+        elif(self.filt_size==3):
+            a = np.array([1., 2., 1.])
+        elif(self.filt_size==4):
+            a = np.array([1., 3., 3., 1.])
+        elif(self.filt_size==5):
+            a = np.array([1., 4., 6., 4., 1.])
+        elif(self.filt_size==6):
+            a = np.array([1., 5., 10., 10., 5., 1.])
+        elif(self.filt_size==7):
+            a = np.array([1., 6., 15., 20., 15., 6., 1.])
+
+        a = a[:,None]*a[None,:]
+        a = a / np.sum(a)
+        a = a[:,:,None,None]
+        self.a = a
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        self.k = tf.constant (self.a, dtype=nn.floatx )
+
+    def forward(self, x):
+        k = tf.tile (self.k, (1,1,x.shape[nn.conv2d_ch_axis],1) )
+        x = tf.pad(x, self.padding )
+        x = tf.nn.depthwise_conv2d(x, k, self.strides, 'VALID', data_format=nn.data_format)
+        return x
+nn.BlurPool = BlurPool

face_detect/core/leras/layers/Conv2D.py

@@ -0,0 +1,112 @@
+import numpy as np
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class Conv2D(nn.LayerBase):
+    """
+    default kernel_initializer - CA
+    use_wscale  bool enables equalized learning rate, if kernel_initializer is None, it will be forced to random_normal
+
+
+    """
+    def __init__(self, in_ch, out_ch, kernel_size, strides=1, padding='SAME', dilations=1, use_bias=True, use_wscale=False, kernel_initializer=None, bias_initializer=None, trainable=True, dtype=None, **kwargs ):
+        if not isinstance(strides, int):
+            raise ValueError ("strides must be an int type")
+        if not isinstance(dilations, int):
+            raise ValueError ("dilations must be an int type")
+        kernel_size = int(kernel_size)
+
+        if dtype is None:
+            dtype = nn.floatx
+
+        if isinstance(padding, str):
+            if padding == "SAME":
+                padding = ( (kernel_size - 1) * dilations + 1 ) // 2
+            elif padding == "VALID":
+                padding = 0
+            else:
+                raise ValueError ("Wrong padding type. Should be VALID SAME or INT or 4x INTs")
+
+        if isinstance(padding, int):
+            if padding != 0:
+                if nn.data_format == "NHWC":
+                    padding = [ [0,0], [padding,padding], [padding,padding], [0,0] ]
+                else:
+                    padding = [ [0,0], [0,0], [padding,padding], [padding,padding] ]
+            else:
+                padding = None
+
+        if nn.data_format == "NHWC":
+            strides = [1,strides,strides,1]
+        else:
+            strides = [1,1,strides,strides]
+
+        if nn.data_format == "NHWC":
+            dilations = [1,dilations,dilations,1]
+        else:
+            dilations = [1,1,dilations,dilations]
+
+        self.in_ch = in_ch
+        self.out_ch = out_ch
+        self.kernel_size = kernel_size
+        self.strides = strides
+        self.padding = padding
+        self.dilations = dilations
+        self.use_bias = use_bias
+        self.use_wscale = use_wscale
+        self.kernel_initializer = kernel_initializer
+        self.bias_initializer = bias_initializer
+        self.trainable = trainable
+        self.dtype = dtype
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        kernel_initializer = self.kernel_initializer
+        if self.use_wscale:
+            gain = 1.0 if self.kernel_size == 1 else np.sqrt(2)
+            fan_in = self.kernel_size*self.kernel_size*self.in_ch
+            he_std = gain / np.sqrt(fan_in)
+            self.wscale = tf.constant(he_std, dtype=self.dtype )
+            if kernel_initializer is None:
+                kernel_initializer = tf.initializers.random_normal(0, 1.0, dtype=self.dtype)
+
+        if kernel_initializer is None:
+            kernel_initializer = nn.initializers.ca()
+
+        self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
+
+        if self.use_bias:
+            bias_initializer = self.bias_initializer
+            if bias_initializer is None:
+                bias_initializer = tf.initializers.zeros(dtype=self.dtype)
+
+            self.bias = tf.get_variable("bias", (self.out_ch,), dtype=self.dtype, initializer=bias_initializer, trainable=self.trainable )
+
+    def get_weights(self):
+        weights = [self.weight]
+        if self.use_bias:
+            weights += [self.bias]
+        return weights
+
+    def forward(self, x):
+        weight = self.weight
+        if self.use_wscale:
+            weight = weight * self.wscale
+
+        if self.padding is not None:
+            x = tf.pad (x, self.padding, mode='CONSTANT')
+
+        x = tf.nn.conv2d(x, weight, self.strides, 'VALID', dilations=self.dilations, data_format=nn.data_format)
+        if self.use_bias:
+            if nn.data_format == "NHWC":
+                bias = tf.reshape (self.bias, (1,1,1,self.out_ch) )
+            else:
+                bias = tf.reshape (self.bias, (1,self.out_ch,1,1) )
+            x = tf.add(x, bias)
+        return x
+
+    def __str__(self):
+        r = f"{self.__class__.__name__} : in_ch:{self.in_ch} out_ch:{self.out_ch} "
+
+        return r
+nn.Conv2D = Conv2D

face_detect/core/leras/layers/Conv2DTranspose.py

@@ -0,0 +1,107 @@
+import numpy as np
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class Conv2DTranspose(nn.LayerBase):
+    """
+    use_wscale      enables weight scale (equalized learning rate)
+                    if kernel_initializer is None, it will be forced to random_normal
+    """
+    def __init__(self, in_ch, out_ch, kernel_size, strides=2, padding='SAME', use_bias=True, use_wscale=False, kernel_initializer=None, bias_initializer=None, trainable=True, dtype=None, **kwargs ):
+        if not isinstance(strides, int):
+            raise ValueError ("strides must be an int type")
+        kernel_size = int(kernel_size)
+
+        if dtype is None:
+            dtype = nn.floatx
+
+        self.in_ch = in_ch
+        self.out_ch = out_ch
+        self.kernel_size = kernel_size
+        self.strides = strides
+        self.padding = padding
+        self.use_bias = use_bias
+        self.use_wscale = use_wscale
+        self.kernel_initializer = kernel_initializer
+        self.bias_initializer = bias_initializer
+        self.trainable = trainable
+        self.dtype = dtype
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        kernel_initializer = self.kernel_initializer
+        if self.use_wscale:
+            gain = 1.0 if self.kernel_size == 1 else np.sqrt(2)
+            fan_in = self.kernel_size*self.kernel_size*self.in_ch
+            he_std = gain / np.sqrt(fan_in) # He init
+            self.wscale = tf.constant(he_std, dtype=self.dtype )
+            if kernel_initializer is None:
+                kernel_initializer = tf.initializers.random_normal(0, 1.0, dtype=self.dtype)
+
+        if kernel_initializer is None:
+            kernel_initializer = nn.initializers.ca()
+        self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.out_ch,self.in_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
+
+        if self.use_bias:
+            bias_initializer = self.bias_initializer
+            if bias_initializer is None:
+                bias_initializer = tf.initializers.zeros(dtype=self.dtype)
+
+            self.bias = tf.get_variable("bias", (self.out_ch,), dtype=self.dtype, initializer=bias_initializer, trainable=self.trainable )
+
+    def get_weights(self):
+        weights = [self.weight]
+        if self.use_bias:
+            weights += [self.bias]
+        return weights
+
+    def forward(self, x):
+        shape = x.shape
+
+        if nn.data_format == "NHWC":
+            h,w,c = shape[1], shape[2], shape[3]
+            output_shape = tf.stack ( (tf.shape(x)[0],
+                                    self.deconv_length(w, self.strides, self.kernel_size, self.padding),
+                                    self.deconv_length(h, self.strides, self.kernel_size, self.padding),
+                                    self.out_ch) )
+
+            strides = [1,self.strides,self.strides,1]
+        else:
+            c,h,w = shape[1], shape[2], shape[3]
+            output_shape = tf.stack ( (tf.shape(x)[0],
+                                        self.out_ch,
+                                        self.deconv_length(w, self.strides, self.kernel_size, self.padding),
+                                        self.deconv_length(h, self.strides, self.kernel_size, self.padding),
+                                        ) )
+            strides = [1,1,self.strides,self.strides]
+        weight = self.weight
+        if self.use_wscale:
+            weight = weight * self.wscale
+
+        x = tf.nn.conv2d_transpose(x, weight, output_shape, strides, padding=self.padding, data_format=nn.data_format)
+
+        if self.use_bias:
+            if nn.data_format == "NHWC":
+                bias = tf.reshape (self.bias, (1,1,1,self.out_ch) )
+            else:
+                bias = tf.reshape (self.bias, (1,self.out_ch,1,1) )
+            x = tf.add(x, bias)
+        return x
+
+    def __str__(self):
+        r = f"{self.__class__.__name__} : in_ch:{self.in_ch} out_ch:{self.out_ch} "
+
+        return r
+
+    def deconv_length(self, dim_size, stride_size, kernel_size, padding):
+        assert padding in {'SAME', 'VALID', 'FULL'}
+        if dim_size is None:
+            return None
+        if padding == 'VALID':
+            dim_size = dim_size * stride_size + max(kernel_size - stride_size, 0)
+        elif padding == 'FULL':
+            dim_size = dim_size * stride_size - (stride_size + kernel_size - 2)
+        elif padding == 'SAME':
+            dim_size = dim_size * stride_size
+        return dim_size
+nn.Conv2DTranspose = Conv2DTranspose

face_detect/core/leras/layers/Dense.py

@@ -0,0 +1,76 @@
+import numpy as np
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class Dense(nn.LayerBase):
+    def __init__(self, in_ch, out_ch, use_bias=True, use_wscale=False, maxout_ch=0, kernel_initializer=None, bias_initializer=None, trainable=True, dtype=None, **kwargs ):
+        """
+        use_wscale          enables weight scale (equalized learning rate)
+                            if kernel_initializer is None, it will be forced to random_normal
+
+        maxout_ch     https://link.springer.com/article/10.1186/s40537-019-0233-0
+                            typical 2-4 if you want to enable DenseMaxout behaviour
+        """
+        self.in_ch = in_ch
+        self.out_ch = out_ch
+        self.use_bias = use_bias
+        self.use_wscale = use_wscale
+        self.maxout_ch = maxout_ch
+        self.kernel_initializer = kernel_initializer
+        self.bias_initializer = bias_initializer
+        self.trainable = trainable
+        if dtype is None:
+            dtype = nn.floatx
+
+        self.dtype = dtype
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        if self.maxout_ch > 1:
+            weight_shape = (self.in_ch,self.out_ch*self.maxout_ch)
+        else:
+            weight_shape = (self.in_ch,self.out_ch)
+
+        kernel_initializer = self.kernel_initializer
+
+        if self.use_wscale:
+            gain = 1.0
+            fan_in = np.prod( weight_shape[:-1] )
+            he_std = gain / np.sqrt(fan_in) # He init
+            self.wscale = tf.constant(he_std, dtype=self.dtype )
+            if kernel_initializer is None:
+                kernel_initializer = tf.initializers.random_normal(0, 1.0, dtype=self.dtype)
+
+        if kernel_initializer is None:
+            kernel_initializer = tf.initializers.glorot_uniform(dtype=self.dtype)
+
+        self.weight = tf.get_variable("weight", weight_shape, dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
+
+        if self.use_bias:
+            bias_initializer = self.bias_initializer
+            if bias_initializer is None:
+                bias_initializer = tf.initializers.zeros(dtype=self.dtype)
+            self.bias = tf.get_variable("bias", (self.out_ch,), dtype=self.dtype, initializer=bias_initializer, trainable=self.trainable )
+
+    def get_weights(self):
+        weights = [self.weight]
+        if self.use_bias:
+            weights += [self.bias]
+        return weights
+
+    def forward(self, x):
+        weight = self.weight
+        if self.use_wscale:
+            weight = weight * self.wscale
+
+        x = tf.matmul(x, weight)
+
+        if self.maxout_ch > 1:
+            x = tf.reshape (x, (-1, self.out_ch, self.maxout_ch) )
+            x = tf.reduce_max(x, axis=-1)
+
+        if self.use_bias:
+            x = tf.add(x, tf.reshape(self.bias, (1,self.out_ch) ) )
+
+        return x
+nn.Dense = Dense

face_detect/core/leras/layers/DenseNorm.py

@@ -0,0 +1,16 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class DenseNorm(nn.LayerBase):
+    def __init__(self, dense=False, eps=1e-06, dtype=None, **kwargs):
+        self.dense = dense        
+        if dtype is None:
+            dtype = nn.floatx
+        self.eps = tf.constant(eps, dtype=dtype, name="epsilon")
+
+        super().__init__(**kwargs)
+
+    def __call__(self, x):
+        return x * tf.rsqrt(tf.reduce_mean(tf.square(x), axis=-1, keepdims=True) + self.eps)
+        
+nn.DenseNorm = DenseNorm

face_detect/core/leras/layers/DepthwiseConv2D.py

@@ -0,0 +1,110 @@
+import numpy as np
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class DepthwiseConv2D(nn.LayerBase):
+    """
+    default kernel_initializer - CA
+    use_wscale  bool enables equalized learning rate, if kernel_initializer is None, it will be forced to random_normal
+    """
+    def __init__(self, in_ch, kernel_size, strides=1, padding='SAME', depth_multiplier=1, dilations=1, use_bias=True, use_wscale=False, kernel_initializer=None, bias_initializer=None, trainable=True, dtype=None, **kwargs ):
+        if not isinstance(strides, int):
+            raise ValueError ("strides must be an int type")
+        if not isinstance(dilations, int):
+            raise ValueError ("dilations must be an int type")
+        kernel_size = int(kernel_size)
+
+        if dtype is None:
+            dtype = nn.floatx
+
+        if isinstance(padding, str):
+            if padding == "SAME":
+                padding = ( (kernel_size - 1) * dilations + 1 ) // 2
+            elif padding == "VALID":
+                padding = 0
+            else:
+                raise ValueError ("Wrong padding type. Should be VALID SAME or INT or 4x INTs")
+
+        if isinstance(padding, int):
+            if padding != 0:
+                if nn.data_format == "NHWC":
+                    padding = [ [0,0], [padding,padding], [padding,padding], [0,0] ]
+                else:
+                    padding = [ [0,0], [0,0], [padding,padding], [padding,padding] ]
+            else:
+                padding = None
+
+        if nn.data_format == "NHWC":
+            strides = [1,strides,strides,1]
+        else:
+            strides = [1,1,strides,strides]
+
+        if nn.data_format == "NHWC":
+            dilations = [1,dilations,dilations,1]
+        else:
+            dilations = [1,1,dilations,dilations]
+
+        self.in_ch = in_ch
+        self.depth_multiplier = depth_multiplier
+        self.kernel_size = kernel_size
+        self.strides = strides
+        self.padding = padding
+        self.dilations = dilations
+        self.use_bias = use_bias
+        self.use_wscale = use_wscale
+        self.kernel_initializer = kernel_initializer
+        self.bias_initializer = bias_initializer
+        self.trainable = trainable
+        self.dtype = dtype
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        kernel_initializer = self.kernel_initializer
+        if self.use_wscale:
+            gain = 1.0 if self.kernel_size == 1 else np.sqrt(2)
+            fan_in = self.kernel_size*self.kernel_size*self.in_ch
+            he_std = gain / np.sqrt(fan_in)
+            self.wscale = tf.constant(he_std, dtype=self.dtype )
+            if kernel_initializer is None:
+                kernel_initializer = tf.initializers.random_normal(0, 1.0, dtype=self.dtype)
+
+        if kernel_initializer is None:
+            kernel_initializer = nn.initializers.ca()
+
+        self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.depth_multiplier), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
+
+        if self.use_bias:
+            bias_initializer = self.bias_initializer
+            if bias_initializer is None:
+                bias_initializer = tf.initializers.zeros(dtype=self.dtype)
+
+            self.bias = tf.get_variable("bias", (self.in_ch*self.depth_multiplier,), dtype=self.dtype, initializer=bias_initializer, trainable=self.trainable )
+
+    def get_weights(self):
+        weights = [self.weight]
+        if self.use_bias:
+            weights += [self.bias]
+        return weights
+
+    def forward(self, x):
+        weight = self.weight
+        if self.use_wscale:
+            weight = weight * self.wscale
+
+        if self.padding is not None:
+            x = tf.pad (x, self.padding, mode='CONSTANT')
+
+        x = tf.nn.depthwise_conv2d(x, weight, self.strides, 'VALID', data_format=nn.data_format)
+        if self.use_bias:
+            if nn.data_format == "NHWC":
+                bias = tf.reshape (self.bias, (1,1,1,self.in_ch*self.depth_multiplier) )
+            else:
+                bias = tf.reshape (self.bias, (1,self.in_ch*self.depth_multiplier,1,1) )
+            x = tf.add(x, bias)
+        return x
+
+    def __str__(self):
+        r = f"{self.__class__.__name__} : in_ch:{self.in_ch} depth_multiplier:{self.depth_multiplier} "
+        return r
+
+nn.DepthwiseConv2D = DepthwiseConv2D

face_detect/core/leras/layers/FRNorm2D.py

@@ -0,0 +1,38 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class FRNorm2D(nn.LayerBase):
+    """
+    Tensorflow implementation of
+    Filter Response Normalization Layer: Eliminating Batch Dependence in theTraining of Deep Neural Networks
+    https://arxiv.org/pdf/1911.09737.pdf
+    """
+    def __init__(self, in_ch, dtype=None, **kwargs):
+        self.in_ch = in_ch
+
+        if dtype is None:
+            dtype = nn.floatx
+        self.dtype = dtype
+
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        self.weight      = tf.get_variable("weight", (self.in_ch,), dtype=self.dtype, initializer=tf.initializers.ones() )
+        self.bias        = tf.get_variable("bias",   (self.in_ch,), dtype=self.dtype, initializer=tf.initializers.zeros() )
+        self.eps         = tf.get_variable("eps",    (1,), dtype=self.dtype, initializer=tf.initializers.constant(1e-6) )
+
+    def get_weights(self):
+        return [self.weight, self.bias, self.eps]
+
+    def forward(self, x):
+        if nn.data_format == "NHWC":
+            shape = (1,1,1,self.in_ch)
+        else:
+            shape = (1,self.in_ch,1,1)
+        weight       = tf.reshape ( self.weight, shape )
+        bias         = tf.reshape ( self.bias  , shape )
+        nu2 = tf.reduce_mean(tf.square(x), axis=nn.conv2d_spatial_axes, keepdims=True)
+        x = x * ( 1.0/tf.sqrt(nu2 + tf.abs(self.eps) ) )
+
+        return x*weight + bias
+nn.FRNorm2D = FRNorm2D

face_detect/core/leras/layers/InstanceNorm2D.py

@@ -0,0 +1,40 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class InstanceNorm2D(nn.LayerBase):
+    def __init__(self, in_ch, dtype=None, **kwargs):
+        self.in_ch = in_ch
+
+        if dtype is None:
+            dtype = nn.floatx
+        self.dtype = dtype
+
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        kernel_initializer = tf.initializers.glorot_uniform(dtype=self.dtype)
+        self.weight       = tf.get_variable("weight",   (self.in_ch,), dtype=self.dtype, initializer=kernel_initializer )
+        self.bias         = tf.get_variable("bias",     (self.in_ch,), dtype=self.dtype, initializer=tf.initializers.zeros() )
+
+    def get_weights(self):
+        return [self.weight, self.bias]
+
+    def forward(self, x):
+        if nn.data_format == "NHWC":
+            shape = (1,1,1,self.in_ch)
+        else:
+            shape = (1,self.in_ch,1,1)
+
+        weight       = tf.reshape ( self.weight      , shape )
+        bias         = tf.reshape ( self.bias        , shape )
+
+        x_mean = tf.reduce_mean(x, axis=nn.conv2d_spatial_axes, keepdims=True )
+        x_std  = tf.math.reduce_std(x, axis=nn.conv2d_spatial_axes, keepdims=True ) + 1e-5
+
+        x = (x - x_mean) / x_std
+        x *= weight
+        x += bias
+
+        return x
+
+nn.InstanceNorm2D = InstanceNorm2D

face_detect/core/leras/layers/LayerBase.py

@@ -0,0 +1,16 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class LayerBase(nn.Saveable):
+    #override
+    def build_weights(self):
+        pass
+    
+    #override
+    def forward(self, *args, **kwargs):
+        pass
+    
+    def __call__(self, *args, **kwargs):
+        return self.forward(*args, **kwargs)
+
+nn.LayerBase = LayerBase

face_detect/core/leras/layers/Saveable.py

@@ -0,0 +1,106 @@
+import pickle
+from pathlib import Path
+from face_feature.core import pathex
+import numpy as np
+
+from face_feature.core.leras import nn
+
+tf = nn.tf
+
+class Saveable():
+    def __init__(self, name=None):
+        self.name = name
+
+    #override
+    def get_weights(self):
+        #return tf tensors that should be initialized/loaded/saved
+        return []
+
+    #override
+    def get_weights_np(self):
+        weights = self.get_weights()
+        if len(weights) == 0:
+            return []
+        return nn.tf_sess.run (weights)
+
+    def set_weights(self, new_weights):
+        weights = self.get_weights()
+        if len(weights) != len(new_weights):
+            raise ValueError ('len of lists mismatch')
+
+        tuples = []
+        for w, new_w in zip(weights, new_weights):
+
+            if len(w.shape) != new_w.shape:
+                new_w = new_w.reshape(w.shape)
+
+            tuples.append ( (w, new_w) )
+
+        nn.batch_set_value (tuples)
+
+    def save_weights(self, filename, force_dtype=None):
+        d = {}
+        weights = self.get_weights()
+
+        if self.name is None:
+            raise Exception("name must be defined.")
+
+        name = self.name
+        for w, w_val in zip(weights, nn.tf_sess.run (weights)):
+            w_name_split = w.name.split('/', 1)
+            if name != w_name_split[0]:
+                raise Exception("weight first name != Saveable.name")
+
+            if force_dtype is not None:
+                w_val = w_val.astype(force_dtype)
+
+            d[ w_name_split[1] ] = w_val
+
+        d_dumped = pickle.dumps (d, 4)
+        pathex.write_bytes_safe ( Path(filename), d_dumped )
+
+    def load_weights(self, filename):
+        """
+        returns True if file exists
+        """
+        filepath = Path(filename)
+        if filepath.exists():
+            result = True
+            d_dumped = filepath.read_bytes()
+            d = pickle.loads(d_dumped)
+        else:
+            return False
+
+        weights = self.get_weights()
+
+        if self.name is None:
+            raise Exception("name must be defined.")
+
+        try:
+            tuples = []
+            for w in weights:
+                w_name_split = w.name.split('/')
+                if self.name != w_name_split[0]:
+                    raise Exception("weight first name != Saveable.name")
+
+                sub_w_name = "/".join(w_name_split[1:])
+
+                w_val = d.get(sub_w_name, None)
+
+                if w_val is None:
+                    #io.log_err(f"Weight {w.name} was not loaded from file {filename}")
+                    tuples.append ( (w, w.initializer) )
+                else:
+                    w_val = np.reshape( w_val, w.shape.as_list() )
+                    tuples.append ( (w, w_val) )
+
+            nn.batch_set_value(tuples)
+        except:
+            return False
+            
+        return True
+
+    def init_weights(self):
+        nn.init_weights(self.get_weights())
+    
+nn.Saveable = Saveable

face_detect/core/leras/layers/ScaleAdd.py

@@ -0,0 +1,31 @@
+from face_feature.core.leras import nn
+tf = nn.tf
+
+class ScaleAdd(nn.LayerBase):
+    def __init__(self, ch, dtype=None, **kwargs):
+        if dtype is None:
+            dtype = nn.floatx
+        self.dtype = dtype
+        self.ch = ch
+
+        super().__init__(**kwargs)
+
+    def build_weights(self):
+        self.weight = tf.get_variable("weight",(self.ch,), dtype=self.dtype, initializer=tf.initializers.zeros() )
+
+    def get_weights(self):
+        return [self.weight]
+
+    def forward(self, inputs):
+        if nn.data_format == "NHWC":
+            shape = (1,1,1,self.ch)
+        else:
+            shape = (1,self.ch,1,1)
+
+        weight = tf.reshape ( self.weight, shape )
+
+        x0, x1 = inputs
+        x = x0 + x1*weight
+
+        return x
+nn.ScaleAdd = ScaleAdd