DFL-2.0 initial branch commit

merger/FrameInfo.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+class FrameInfo(object):
+    def __init__(self, filename=None, landmarks_list=None):
+        self.filename = filename
+        self.filename_short = str(Path(filename).name)
+        self.landmarks_list = landmarks_list or []
+        self.motion_deg = 0
+        self.motion_power = 0

merger/MergeAvatar.py

@@ -0,0 +1,41 @@
+import cv2
+import numpy as np
+
+from core import imagelib
+from facelib import FaceType, LandmarksProcessor
+from core.cv2ex import *
+
+def process_frame_info(frame_info, inp_sh):
+    img_uint8 = cv2_imread (frame_info.filename)
+    img_uint8 = imagelib.normalize_channels (img_uint8, 3)        
+    img = img_uint8.astype(np.float32) / 255.0        
+
+    img_mat = LandmarksProcessor.get_transform_mat (frame_info.landmarks_list[0], inp_sh[0], face_type=FaceType.FULL_NO_ALIGN)
+    img = cv2.warpAffine( img, img_mat, inp_sh[0:2], borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_CUBIC )
+    return img
+    
+def MergeFaceAvatar (predictor_func, predictor_input_shape, cfg, prev_temporal_frame_infos, frame_info, next_temporal_frame_infos):
+    inp_sh = predictor_input_shape
+
+    prev_imgs=[]
+    next_imgs=[]
+    for i in range(cfg.temporal_face_count):
+        prev_imgs.append( process_frame_info(prev_temporal_frame_infos[i], inp_sh) )
+        next_imgs.append( process_frame_info(next_temporal_frame_infos[i], inp_sh) )
+    img = process_frame_info(frame_info, inp_sh)
+
+    prd_f = predictor_func ( prev_imgs, img, next_imgs )
+
+    if cfg.super_resolution_mode != 0:
+        prd_f = cfg.superres_func(cfg.super_resolution_mode, prd_f)
+        
+    if cfg.sharpen_mode != 0 and cfg.sharpen_amount != 0:
+        prd_f = cfg.sharpen_func ( prd_f, cfg.sharpen_mode, 3, cfg.sharpen_amount)
+    
+    out_img = np.clip(prd_f, 0.0, 1.0)
+
+    if cfg.add_source_image:
+        out_img = np.concatenate ( [cv2.resize ( img, (prd_f.shape[1], prd_f.shape[0])  ), 
+                                    out_img], axis=1 )
+
+    return (out_img*255).astype(np.uint8)

merger/MergeMasked.py

@@ -0,0 +1,362 @@
+import traceback
+
+import cv2
+import numpy as np
+
+from core import imagelib
+from facelib import FaceType, LandmarksProcessor
+from core.interact import interact as io
+from core.cv2ex import *
+
+def MergeMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, img_bgr_uint8, img_bgr, img_face_landmarks):
+    img_size = img_bgr.shape[1], img_bgr.shape[0]
+    img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr.shape, img_face_landmarks)
+
+    if cfg.mode == 'original':
+        if cfg.export_mask_alpha:
+            img_bgr = np.concatenate ( [img_bgr, img_face_mask_a], -1 )
+        return img_bgr, img_face_mask_a
+
+    out_img = img_bgr.copy()
+    out_merging_mask = None
+
+    output_size = predictor_input_shape[0]
+    if cfg.super_resolution_mode != 0:
+        output_size *= 4
+
+    face_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, output_size, face_type=cfg.face_type)
+    face_output_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, output_size, face_type=cfg.face_type, scale= 1.0 + 0.01*cfg.output_face_scale   )
+
+    dst_face_bgr      = cv2.warpAffine( img_bgr        , face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC )
+    dst_face_bgr      = np.clip(dst_face_bgr, 0, 1)
+    
+    dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC )
+    dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
+
+    predictor_input_bgr      = cv2.resize (dst_face_bgr, predictor_input_shape[0:2] )
+
+    predicted = predictor_func (predictor_input_bgr)
+    if isinstance(predicted, tuple):
+        #merger return bgr,mask
+        prd_face_bgr      = np.clip (predicted[0], 0, 1.0)
+        prd_face_mask_a_0 = np.clip (predicted[1], 0, 1.0)
+        predictor_masked = True
+    else:
+        #merger return bgr only, using dst mask
+        prd_face_bgr      = np.clip (predicted, 0, 1.0 )
+        prd_face_mask_a_0 = cv2.resize (dst_face_mask_a_0, predictor_input_shape[0:2] )
+        predictor_masked = False
+
+    if cfg.super_resolution_mode:
+        prd_face_bgr = cfg.superres_func(cfg.super_resolution_mode, prd_face_bgr)
+        prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
+        
+        if predictor_masked:
+            prd_face_mask_a_0 = cv2.resize (prd_face_mask_a_0,  (output_size, output_size), cv2.INTER_CUBIC)
+        else:
+            prd_face_mask_a_0 = cv2.resize (dst_face_mask_a_0,  (output_size, output_size), cv2.INTER_CUBIC)
+
+    if cfg.mask_mode == 2: #dst
+        prd_face_mask_a_0 = cv2.resize (dst_face_mask_a_0, (output_size,output_size), cv2.INTER_CUBIC)
+    elif cfg.mask_mode >= 3 and cfg.mask_mode <= 8:
+
+        if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6:
+            prd_face_fanseg_bgr = cv2.resize (prd_face_bgr, (cfg.fanseg_input_size,)*2 )
+            prd_face_fanseg_mask = cfg.fanseg_extract_func(FaceType.FULL, prd_face_fanseg_bgr)
+            FAN_prd_face_mask_a_0 = cv2.resize ( prd_face_fanseg_mask, (output_size, output_size), cv2.INTER_CUBIC)
+
+        if cfg.mask_mode >= 4 and cfg.mask_mode <= 7:
+
+            full_face_fanseg_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, cfg.fanseg_input_size, face_type=FaceType.FULL)
+            dst_face_fanseg_bgr = cv2.warpAffine(img_bgr, full_face_fanseg_mat, (cfg.fanseg_input_size,)*2, flags=cv2.INTER_CUBIC )
+            dst_face_fanseg_mask = cfg.fanseg_extract_func( FaceType.FULL, dst_face_fanseg_bgr )
+
+            if cfg.face_type == FaceType.FULL:
+                FAN_dst_face_mask_a_0 = cv2.resize (dst_face_fanseg_mask, (output_size,output_size), cv2.INTER_CUBIC)
+            else:
+                face_fanseg_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, cfg.fanseg_input_size, face_type=cfg.face_type)
+
+                fanseg_rect_corner_pts = np.array ( [ [0,0], [cfg.fanseg_input_size-1,0], [0,cfg.fanseg_input_size-1] ], dtype=np.float32 )
+                a = LandmarksProcessor.transform_points (fanseg_rect_corner_pts, face_fanseg_mat, invert=True )
+                b = LandmarksProcessor.transform_points (a, full_face_fanseg_mat )
+                m = cv2.getAffineTransform(b, fanseg_rect_corner_pts)
+                FAN_dst_face_mask_a_0 = cv2.warpAffine(dst_face_fanseg_mask, m, (cfg.fanseg_input_size,)*2, flags=cv2.INTER_CUBIC )
+                FAN_dst_face_mask_a_0 = cv2.resize (FAN_dst_face_mask_a_0, (output_size,output_size), cv2.INTER_CUBIC)
+
+        if cfg.mask_mode == 3:   #FAN-prd
+            prd_face_mask_a_0 = FAN_prd_face_mask_a_0
+        elif cfg.mask_mode == 4: #FAN-dst
+            prd_face_mask_a_0 = FAN_dst_face_mask_a_0
+        elif cfg.mask_mode == 5:
+            prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
+        elif cfg.mask_mode == 6:
+            prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
+        elif cfg.mask_mode == 7:
+            prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0
+        #elif cfg.mask_mode == 8: #FANCHQ-dst
+        #    prd_face_mask_a_0 = FANCHQ_dst_face_mask_a_0
+
+    prd_face_mask_a_0[ prd_face_mask_a_0 < 0.001 ] = 0.0
+
+    prd_face_mask_a   = prd_face_mask_a_0[...,np.newaxis]
+    prd_face_mask_aaa = np.repeat (prd_face_mask_a, (3,), axis=-1)
+
+    img_face_mask_aaa = cv2.warpAffine( prd_face_mask_aaa, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC )
+    img_face_mask_aaa = np.clip (img_face_mask_aaa, 0.0, 1.0)
+    img_face_mask_aaa [ img_face_mask_aaa <= 0.1 ] = 0.0 #get rid of noise
+
+    if 'raw' in cfg.mode:
+        face_corner_pts = np.array ([ [0,0], [output_size-1,0], [output_size-1,output_size-1],  [0,output_size-1] ], dtype=np.float32)
+        square_mask = np.zeros(img_bgr.shape, dtype=np.float32)
+        cv2.fillConvexPoly(square_mask, \
+                           LandmarksProcessor.transform_points (face_corner_pts, face_output_mat, invert=True ).astype(np.int), \
+                           (1,1,1) )
+
+        if cfg.mode == 'raw-rgb':
+            out_merging_mask = square_mask
+
+        if cfg.mode == 'raw-rgb' or cfg.mode == 'raw-rgb-mask':
+            out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
+
+        if cfg.mode == 'raw-rgb-mask':
+            out_img = np.concatenate ( [out_img, np.expand_dims (img_face_mask_aaa[:,:,0],-1)], -1 )
+            out_merging_mask = square_mask
+
+        elif cfg.mode == 'raw-mask-only':
+            out_img = img_face_mask_aaa
+            out_merging_mask = img_face_mask_aaa
+        elif cfg.mode == 'raw-predicted-only':
+            out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
+            out_merging_mask = square_mask
+
+        out_img = np.clip (out_img, 0.0, 1.0 )
+    else:
+        #averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
+        ar = []
+        for i in range(1, 10):
+            maxregion = np.argwhere( img_face_mask_aaa > i / 10.0 )
+            if maxregion.size != 0:
+                miny,minx = maxregion.min(axis=0)[:2]
+                maxy,maxx = maxregion.max(axis=0)[:2]
+                lenx = maxx - minx
+                leny = maxy - miny
+                if min(lenx,leny) >= 4:
+                    ar += [ [ lenx, leny]  ]
+
+        if len(ar) > 0:
+            lenx, leny = np.mean ( ar, axis=0 )
+            lowest_len = min (lenx, leny)
+
+            if cfg.erode_mask_modifier != 0:
+                ero  = int( lowest_len * ( 0.126 - lowest_len * 0.00004551365 ) * 0.01*cfg.erode_mask_modifier )
+                if ero > 0:
+                    img_face_mask_aaa = cv2.erode(img_face_mask_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 )
+                elif ero < 0:
+                    img_face_mask_aaa = cv2.dilate(img_face_mask_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(-ero,-ero)), iterations = 1 )
+
+            if cfg.clip_hborder_mask_per > 0: #clip hborder before blur
+                prd_hborder_rect_mask_a = np.ones ( prd_face_mask_a.shape, dtype=np.float32)
+                prd_border_size = int ( prd_hborder_rect_mask_a.shape[1] * cfg.clip_hborder_mask_per )
+                prd_hborder_rect_mask_a[:,0:prd_border_size,:] = 0
+                prd_hborder_rect_mask_a[:,-prd_border_size:,:] = 0
+                prd_hborder_rect_mask_a[-prd_border_size:,:,:] = 0
+                prd_hborder_rect_mask_a = np.expand_dims(cv2.blur(prd_hborder_rect_mask_a, (prd_border_size, prd_border_size) ),-1)
+
+                img_prd_hborder_rect_mask_a = cv2.warpAffine( prd_hborder_rect_mask_a, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC )
+                img_prd_hborder_rect_mask_a = np.expand_dims (img_prd_hborder_rect_mask_a, -1)
+                img_face_mask_aaa *= img_prd_hborder_rect_mask_a
+                img_face_mask_aaa = np.clip( img_face_mask_aaa, 0, 1.0 )
+
+            if cfg.blur_mask_modifier > 0:
+                blur = int( lowest_len * 0.10 * 0.01*cfg.blur_mask_modifier )
+                if blur > 0:
+                    img_face_mask_aaa = cv2.blur(img_face_mask_aaa, (blur, blur) )
+
+            img_face_mask_aaa = np.clip( img_face_mask_aaa, 0, 1.0 )
+
+            if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
+                if cfg.color_transfer_mode == 1: #rct
+                    prd_face_bgr = imagelib.reinhard_color_transfer ( (prd_face_bgr*255).astype(np.uint8),
+                                                                      (dst_face_bgr*255).astype(np.uint8),
+                                                                      source_mask=prd_face_mask_a, target_mask=prd_face_mask_a)
+                    prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
+
+                elif cfg.color_transfer_mode == 2: #lct
+                    prd_face_bgr = imagelib.linear_color_transfer (prd_face_bgr, dst_face_bgr)
+                    prd_face_bgr = np.clip( prd_face_bgr, 0.0, 1.0)
+                elif cfg.color_transfer_mode == 3: #mkl
+                    prd_face_bgr = imagelib.color_transfer_mkl (prd_face_bgr, dst_face_bgr)
+                elif cfg.color_transfer_mode == 4: #mkl-m
+                    prd_face_bgr = imagelib.color_transfer_mkl (prd_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                elif cfg.color_transfer_mode == 5: #idt
+                    prd_face_bgr = imagelib.color_transfer_idt (prd_face_bgr, dst_face_bgr)
+                elif cfg.color_transfer_mode == 6: #idt-m
+                    prd_face_bgr = imagelib.color_transfer_idt (prd_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                elif cfg.color_transfer_mode == 7: #sot-m                    
+                    prd_face_bgr = imagelib.color_transfer_sot (prd_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                    prd_face_bgr = np.clip (prd_face_bgr, 0.0, 1.0)
+                elif cfg.color_transfer_mode == 8: #mix-m
+                    prd_face_bgr = imagelib.color_transfer_mix (prd_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                    
+            if cfg.mode == 'hist-match-bw':
+                prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY)
+                prd_face_bgr = np.repeat( np.expand_dims (prd_face_bgr, -1), (3,), -1 )
+
+            if cfg.mode == 'hist-match' or cfg.mode == 'hist-match-bw':
+                hist_mask_a = np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=np.float32)
+
+                if cfg.masked_hist_match:
+                    hist_mask_a *= prd_face_mask_a
+
+                white =  (1.0-hist_mask_a)* np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=np.float32)
+
+                hist_match_1 = prd_face_bgr*hist_mask_a + white
+                hist_match_1[ hist_match_1 > 1.0 ] = 1.0
+
+                hist_match_2 = dst_face_bgr*hist_mask_a + white
+                hist_match_2[ hist_match_1 > 1.0 ] = 1.0
+
+                prd_face_bgr = imagelib.color_hist_match(hist_match_1, hist_match_2, cfg.hist_match_threshold ).astype(dtype=np.float32)
+
+            if cfg.mode == 'hist-match-bw':
+                prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
+
+            if 'seamless' in cfg.mode:
+                #mask used for cv2.seamlessClone
+                img_face_mask_a = img_face_mask_aaa[...,0:1]
+
+                img_face_seamless_mask_a = None
+                for i in range(1,10):
+                    a = img_face_mask_a > i / 10.0
+                    if len(np.argwhere(a)) == 0:
+                        continue
+                    img_face_seamless_mask_a = img_face_mask_a.copy()
+                    img_face_seamless_mask_a[a] = 1.0
+                    img_face_seamless_mask_a[img_face_seamless_mask_a <= i / 10.0] = 0.0
+                    break
+
+            out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
+            
+            out_img = np.clip(out_img, 0.0, 1.0)
+
+            if 'seamless' in cfg.mode:
+                try:
+                    #calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
+                    l,t,w,h = cv2.boundingRect( (img_face_seamless_mask_a*255).astype(np.uint8) )
+                    s_maskx, s_masky = int(l+w/2), int(t+h/2)
+                    out_img = cv2.seamlessClone( (out_img*255).astype(np.uint8), img_bgr_uint8, (img_face_seamless_mask_a*255).astype(np.uint8), (s_maskx,s_masky) , cv2.NORMAL_CLONE )
+                    out_img = out_img.astype(dtype=np.float32) / 255.0
+                except Exception as e:
+                    #seamlessClone may fail in some cases
+                    e_str = traceback.format_exc()
+
+                    if 'MemoryError' in e_str:
+                        raise Exception("Seamless fail: " + e_str) #reraise MemoryError in order to reprocess this data by other processes
+                    else:
+                        print ("Seamless fail: " + e_str)
+            
+    
+            out_img = img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa)
+
+            out_face_bgr = cv2.warpAffine( out_img, face_mat, (output_size, output_size) )
+
+            if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
+                if cfg.color_transfer_mode == 1:
+                    face_mask_aaa = cv2.warpAffine( img_face_mask_aaa, face_mat, (output_size, output_size) )
+
+                    out_face_bgr = imagelib.reinhard_color_transfer ( (out_face_bgr*255).astype(np.uint8),
+                                                                      (dst_face_bgr*255).astype(np.uint8),
+                                                                            source_mask=face_mask_aaa, target_mask=face_mask_aaa)
+                    out_face_bgr = np.clip( out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
+                elif cfg.color_transfer_mode == 2: #lct
+                    out_face_bgr = imagelib.linear_color_transfer (out_face_bgr, dst_face_bgr)
+                    out_face_bgr = np.clip( out_face_bgr, 0.0, 1.0)
+                elif cfg.color_transfer_mode == 3: #mkl
+                    out_face_bgr = imagelib.color_transfer_mkl (out_face_bgr, dst_face_bgr)
+                elif cfg.color_transfer_mode == 4: #mkl-m
+                    out_face_bgr = imagelib.color_transfer_mkl (out_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                elif cfg.color_transfer_mode == 5: #idt
+                    out_face_bgr = imagelib.color_transfer_idt (out_face_bgr, dst_face_bgr)
+                elif cfg.color_transfer_mode == 6: #idt-m
+                    out_face_bgr = imagelib.color_transfer_idt (out_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                elif cfg.color_transfer_mode == 7: #sot-m                    
+                    out_face_bgr = imagelib.color_transfer_sot (out_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                    out_face_bgr = np.clip (out_face_bgr, 0.0, 1.0)
+                elif cfg.color_transfer_mode == 8: #mix-m
+                    out_face_bgr = imagelib.color_transfer_mix (out_face_bgr*prd_face_mask_a, dst_face_bgr*prd_face_mask_a)
+                    
+            if cfg.mode == 'seamless-hist-match':
+                out_face_bgr = imagelib.color_hist_match(out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
+
+            cfg_mp = cfg.motion_blur_power / 100.0
+            if cfg_mp != 0:
+                k_size = int(frame_info.motion_power*cfg_mp)
+                if k_size >= 1:
+                    k_size = np.clip (k_size+1, 2, 50)
+                    if cfg.super_resolution_mode:
+                        k_size *= 2
+                    out_face_bgr = imagelib.LinearMotionBlur (out_face_bgr, k_size , frame_info.motion_deg)
+
+            if cfg.blursharpen_amount != 0:
+                out_face_bgr = cfg.blursharpen_func ( out_face_bgr, cfg.sharpen_mode, 3, cfg.blursharpen_amount)
+
+
+            if cfg.image_denoise_power != 0:
+                n = cfg.image_denoise_power
+                while n > 0:
+                    img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
+                    if int(n / 100) != 0:
+                        img_bgr = img_bgr_denoised
+                    else:
+                        pass_power = (n % 100) / 100.0
+                        img_bgr = img_bgr*(1.0-pass_power)+img_bgr_denoised*pass_power
+                    n = max(n-10,0)
+
+            if cfg.bicubic_degrade_power != 0:
+                p = 1.0 - cfg.bicubic_degrade_power / 101.0
+                img_bgr_downscaled = cv2.resize (img_bgr, ( int(img_size[0]*p), int(img_size[1]*p ) ), cv2.INTER_CUBIC)
+                img_bgr = cv2.resize (img_bgr_downscaled, img_size, cv2.INTER_CUBIC)
+
+            new_out = cv2.warpAffine( out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
+            out_img =  np.clip( img_bgr*(1-img_face_mask_aaa) + (new_out*img_face_mask_aaa) , 0, 1.0 )
+
+            if cfg.color_degrade_power != 0:
+                out_img_reduced = imagelib.reduce_colors(out_img, 256)
+                if cfg.color_degrade_power == 100:
+                    out_img = out_img_reduced
+                else:
+                    alpha = cfg.color_degrade_power / 100.0
+                    out_img = (out_img*(1.0-alpha) + out_img_reduced*alpha)
+                    
+        out_merging_mask = img_face_mask_aaa
+
+    return out_img, out_merging_mask[...,0:1]
+
+
+def MergeMasked (predictor_func, predictor_input_shape, cfg, frame_info):
+    img_bgr_uint8 = cv2_imread(frame_info.filename)
+    img_bgr_uint8 = imagelib.normalize_channels (img_bgr_uint8, 3)
+    img_bgr = img_bgr_uint8.astype(np.float32) / 255.0
+
+    outs = []
+    for face_num, img_landmarks in enumerate( frame_info.landmarks_list ):
+        out_img, out_img_merging_mask = MergeMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, img_bgr_uint8, img_bgr, img_landmarks)
+        outs += [ (out_img, out_img_merging_mask) ]
+
+    #Combining multiple face outputs
+    final_img = None
+    final_mask = None
+    for img, merging_mask in outs:
+        h,w,c = img.shape
+
+        if final_img is None:
+            final_img = img
+            final_mask = merging_mask
+        else:
+            final_img = final_img*(1-merging_mask) + img*merging_mask            
+            final_mask = np.clip (final_mask + merging_mask, 0, 1 )
+
+    if cfg.export_mask_alpha:
+        final_img = np.concatenate ( [final_img, final_mask], -1)
+        
+    return (final_img*255).astype(np.uint8)

merger/MergerConfig.py

@@ -0,0 +1,371 @@
+import numpy as np
+import copy
+
+from facelib import FaceType
+from core.interact import interact as io
+
+
+class MergerConfig(object):
+    TYPE_NONE = 0
+    TYPE_MASKED = 1
+    TYPE_FACE_AVATAR = 2
+    ####
+
+    TYPE_IMAGE = 3
+    TYPE_IMAGE_WITH_LANDMARKS = 4
+
+    def __init__(self, type=0,
+
+                       super_resolution_mode=0,
+                       sharpen_mode=0,
+                       blursharpen_amount=0,
+                       **kwargs
+                       ):
+        self.type = type
+
+        self.superres_func = None
+        self.blursharpen_func = None
+        self.fanseg_input_size = None
+        self.fanseg_extract_func = None
+
+        self.super_res_dict = {0:"None", 1:'FaceEnhancer'}
+        self.sharpen_dict = {0:"None", 1:'box', 2:'gaussian'}
+
+        #default changeable params
+        self.super_resolution_mode = super_resolution_mode
+        self.sharpen_mode = sharpen_mode
+        self.blursharpen_amount = blursharpen_amount
+
+    def copy(self):
+        return copy.copy(self)
+
+    #overridable
+    def ask_settings(self):
+        s = """Choose sharpen mode: \n"""
+        for key in self.sharpen_dict.keys():
+            s += f"""({key}) {self.sharpen_dict[key]}\n"""            
+        io.log_info(s)
+        self.sharpen_mode = io.input_int ("", 0, valid_list=self.sharpen_dict.keys(), help_message="Enhance details by applying sharpen filter.")
+
+        if self.sharpen_mode != 0:
+            self.blursharpen_amount = np.clip ( io.input_int ("Choose blur/sharpen amount", 0, add_info="-100..100"), -100, 100 )
+
+        s = """Choose super resolution mode: \n"""
+        for key in self.super_res_dict.keys():
+            s += f"""({key}) {self.super_res_dict[key]}\n"""
+        io.log_info(s)
+        self.super_resolution_mode = io.input_int ("", 0, valid_list=self.super_res_dict.keys(), help_message="Enhance details by applying superresolution network.")
+
+    def toggle_sharpen_mode(self):
+        a = list( self.sharpen_dict.keys() )
+        self.sharpen_mode = a[ (a.index(self.sharpen_mode)+1) % len(a) ]
+
+    def add_blursharpen_amount(self, diff):
+        self.blursharpen_amount = np.clip ( self.blursharpen_amount+diff, -100, 100)
+
+    def toggle_super_resolution_mode(self):
+        a = list( self.super_res_dict.keys() )
+        self.super_resolution_mode = a[ (a.index(self.super_resolution_mode)+1) % len(a) ]
+
+    #overridable
+    def get_config(self):
+        d = self.__dict__.copy()
+        d.pop('type')
+        return d
+        return {'sharpen_mode':self.sharpen_mode,
+                'blursharpen_amount':self.blursharpen_amount,
+                'super_resolution_mode':self.super_resolution_mode
+                }
+
+    #overridable
+    def __eq__(self, other):
+        #check equality of changeable params
+
+        if isinstance(other, MergerConfig):
+            return self.sharpen_mode == other.sharpen_mode and \
+                   self.blursharpen_amount == other.blursharpen_amount and \
+                   self.super_resolution_mode == other.super_resolution_mode
+
+        return False
+
+    #overridable
+    def to_string(self, filename):
+        r = ""
+        r += f"sharpen_mode : {self.sharpen_dict[self.sharpen_mode]}\n"
+        r += f"blursharpen_amount : {self.blursharpen_amount}\n"
+        r += f"super_resolution_mode : {self.super_res_dict[self.super_resolution_mode]}\n"
+        return r
+
+mode_dict = {0:'original',
+             1:'overlay',
+             2:'hist-match',
+             3:'seamless',
+             4:'seamless-hist-match',
+             5:'raw-rgb',
+             6:'raw-rgb-mask',
+             7:'raw-mask-only',
+             8:'raw-predicted-only'}
+
+mode_str_dict = {}
+
+for key in mode_dict.keys():
+    mode_str_dict[ mode_dict[key] ] = key
+
+full_face_mask_mode_dict = {1:'learned',
+                                    2:'dst',
+                                    3:'FAN-prd',
+                                    4:'FAN-dst',
+                                    5:'FAN-prd*FAN-dst',
+                                    6:'learned*FAN-prd*FAN-dst'}
+
+half_face_mask_mode_dict = {1:'learned',
+                                    2:'dst',
+                                    4:'FAN-dst',
+                                    7:'learned*FAN-dst'}
+
+ctm_dict = { 0: "None", 1:"rct", 2:"lct", 3:"mkl", 4:"mkl-m", 5:"idt", 6:"idt-m", 7:"sot-m", 8:"mix-m" }
+ctm_str_dict = {None:0, "rct":1, "lct":2, "mkl":3, "mkl-m":4, "idt":5, "idt-m":6, "sot-m":7, "mix-m":8 }
+
+class MergerConfigMasked(MergerConfig):
+
+    def __init__(self, face_type=FaceType.FULL,
+                       default_mode = 'overlay',
+                       clip_hborder_mask_per = 0,
+
+                       mode='overlay',
+                       masked_hist_match=True,
+                       hist_match_threshold = 238,
+                       mask_mode = 1,
+                       erode_mask_modifier = 50,
+                       blur_mask_modifier = 50,
+                       motion_blur_power = 0,
+                       output_face_scale = 0,
+                       color_transfer_mode = ctm_str_dict['rct'],
+                       image_denoise_power = 0,
+                       bicubic_degrade_power = 0,
+                       color_degrade_power = 0,
+                       export_mask_alpha = False,
+                       **kwargs
+                       ):
+
+        super().__init__(type=MergerConfig.TYPE_MASKED, **kwargs)
+
+        self.face_type = face_type
+        if self.face_type not in [FaceType.HALF, FaceType.MID_FULL, FaceType.FULL ]:
+            raise ValueError("MergerConfigMasked does not support this type of face.")
+
+        self.default_mode = default_mode
+        self.clip_hborder_mask_per = clip_hborder_mask_per
+
+        #default changeable params
+        self.mode = mode
+        self.masked_hist_match = masked_hist_match
+        self.hist_match_threshold = hist_match_threshold
+        self.mask_mode = mask_mode
+        self.erode_mask_modifier = erode_mask_modifier
+        self.blur_mask_modifier = blur_mask_modifier
+        self.motion_blur_power = motion_blur_power
+        self.output_face_scale = output_face_scale
+        self.color_transfer_mode = color_transfer_mode
+        self.image_denoise_power = image_denoise_power
+        self.bicubic_degrade_power = bicubic_degrade_power
+        self.color_degrade_power = color_degrade_power
+        self.export_mask_alpha = export_mask_alpha
+
+    def copy(self):
+        return copy.copy(self)
+
+    def set_mode (self, mode):
+        self.mode = mode_dict.get (mode, self.default_mode)
+
+    def toggle_masked_hist_match(self):
+        if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
+            self.masked_hist_match = not self.masked_hist_match
+
+    def add_hist_match_threshold(self, diff):
+        if self.mode == 'hist-match' or self.mode == 'hist-match-bw' or self.mode == 'seamless-hist-match':
+            self.hist_match_threshold = np.clip ( self.hist_match_threshold+diff , 0, 255)
+
+    def toggle_mask_mode(self):
+        if self.face_type == FaceType.FULL:
+            a = list( full_face_mask_mode_dict.keys() )
+        else:
+            a = list( half_face_mask_mode_dict.keys() )
+        self.mask_mode = a[ (a.index(self.mask_mode)+1) % len(a) ]
+
+    def add_erode_mask_modifier(self, diff):
+        self.erode_mask_modifier = np.clip ( self.erode_mask_modifier+diff , -400, 400)
+
+    def add_blur_mask_modifier(self, diff):
+        self.blur_mask_modifier = np.clip ( self.blur_mask_modifier+diff , -400, 400)
+
+    def add_motion_blur_power(self, diff):
+        self.motion_blur_power = np.clip ( self.motion_blur_power+diff, 0, 100)
+
+    def add_output_face_scale(self, diff):
+        self.output_face_scale = np.clip ( self.output_face_scale+diff , -50, 50)
+
+    def toggle_color_transfer_mode(self):
+        self.color_transfer_mode = (self.color_transfer_mode+1) % ( max(ctm_dict.keys())+1 )
+
+    def add_color_degrade_power(self, diff):
+        self.color_degrade_power = np.clip ( self.color_degrade_power+diff , 0, 100)
+
+    def add_image_denoise_power(self, diff):
+        self.image_denoise_power = np.clip ( self.image_denoise_power+diff, 0, 500)
+
+    def add_bicubic_degrade_power(self, diff):
+        self.bicubic_degrade_power = np.clip ( self.bicubic_degrade_power+diff, 0, 100)
+
+    def toggle_export_mask_alpha(self):
+        self.export_mask_alpha = not self.export_mask_alpha
+
+    def ask_settings(self):
+        s = """Choose mode: \n"""
+        for key in mode_dict.keys():
+            s += f"""({key}) {mode_dict[key]}\n"""
+        io.log_info(s)
+        mode = io.input_int ("", mode_str_dict.get(self.default_mode, 1) )
+
+        self.mode = mode_dict.get (mode, self.default_mode )
+
+        if 'raw' not in self.mode:
+            if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
+                self.masked_hist_match = io.input_bool("Masked hist match?", True)
+
+            if self.mode == 'hist-match' or self.mode == 'hist-match-bw' or self.mode == 'seamless-hist-match':
+                self.hist_match_threshold = np.clip ( io.input_int("Hist match threshold", 255, add_info="0..255"), 0, 255)
+
+        if self.face_type == FaceType.FULL:
+            s = """Choose mask mode: \n"""
+            for key in full_face_mask_mode_dict.keys():
+                s += f"""({key}) {full_face_mask_mode_dict[key]}\n"""
+            io.log_info(s)
+
+            self.mask_mode = io.input_int ("", 1, valid_list=full_face_mask_mode_dict.keys(), help_message="If you learned the mask, then option 1 should be choosed. 'dst' mask is raw shaky mask from dst aligned images. 'FAN-prd' - using super smooth mask by pretrained FAN-model from predicted face. 'FAN-dst' - using super smooth mask by pretrained FAN-model from dst face. 'FAN-prd*FAN-dst' or 'learned*FAN-prd*FAN-dst' - using multiplied masks.")
+        else:
+            s = """Choose mask mode: \n"""
+            for key in half_face_mask_mode_dict.keys():
+                s += f"""({key}) {half_face_mask_mode_dict[key]}\n"""
+            io.log_info(s)
+            self.mask_mode = io.input_int ("", 1, valid_list=half_face_mask_mode_dict.keys(), help_message="If you learned the mask, then option 1 should be choosed. 'dst' mask is raw shaky mask from dst aligned images.")
+
+        if 'raw' not in self.mode:
+            self.erode_mask_modifier = np.clip ( io.input_int ("Choose erode mask modifier", 0, add_info="-400..400"), -400, 400)
+            self.blur_mask_modifier =  np.clip ( io.input_int ("Choose blur mask modifier", 0, add_info="-400..400"), -400, 400)
+            self.motion_blur_power = np.clip ( io.input_int ("Choose motion blur power", 0, add_info="0..100"), 0, 100)
+
+        self.output_face_scale = np.clip (io.input_int ("Choose output face scale modifier", 0, add_info="-50..50" ), -50, 50)
+
+        if 'raw' not in self.mode:
+            self.color_transfer_mode = io.input_str ( "Color transfer to predicted face", None, valid_list=list(ctm_str_dict.keys())[1:] )
+            self.color_transfer_mode = ctm_str_dict[self.color_transfer_mode]
+
+        super().ask_settings()
+
+        if 'raw' not in self.mode:
+            self.image_denoise_power = np.clip ( io.input_int ("Choose image degrade by denoise power", 0, add_info="0..500"), 0, 500)
+            self.bicubic_degrade_power = np.clip ( io.input_int ("Choose image degrade by bicubic rescale power", 0, add_info="0..100"), 0, 100)
+            self.color_degrade_power = np.clip (  io.input_int ("Degrade color power of final image", 0, add_info="0..100"), 0, 100)
+            self.export_mask_alpha = io.input_bool("Export png with alpha channel of the mask?", False)
+
+        io.log_info ("")
+
+    def __eq__(self, other):
+        #check equality of changeable params
+
+        if isinstance(other, MergerConfigMasked):
+            return super().__eq__(other) and \
+                   self.mode == other.mode and \
+                   self.masked_hist_match == other.masked_hist_match and \
+                   self.hist_match_threshold == other.hist_match_threshold and \
+                   self.mask_mode == other.mask_mode and \
+                   self.erode_mask_modifier == other.erode_mask_modifier and \
+                   self.blur_mask_modifier == other.blur_mask_modifier and \
+                   self.motion_blur_power == other.motion_blur_power and \
+                   self.output_face_scale == other.output_face_scale and \
+                   self.color_transfer_mode == other.color_transfer_mode and \
+                   self.image_denoise_power == other.image_denoise_power and \
+                   self.bicubic_degrade_power == other.bicubic_degrade_power and \
+                   self.color_degrade_power == other.color_degrade_power and \
+                   self.export_mask_alpha == other.export_mask_alpha
+
+        return False
+
+    def to_string(self, filename):
+        r = (
+            f"""MergerConfig {filename}:\n"""
+            f"""Mode: {self.mode}\n"""
+            )
+
+        if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
+            r += f"""masked_hist_match: {self.masked_hist_match}\n"""
+
+        if self.mode == 'hist-match' or self.mode == 'hist-match-bw' or self.mode == 'seamless-hist-match':
+            r += f"""hist_match_threshold: {self.hist_match_threshold}\n"""
+
+        if self.face_type == FaceType.FULL:
+            r += f"""mask_mode: { full_face_mask_mode_dict[self.mask_mode] }\n"""
+        else:
+            r += f"""mask_mode: { half_face_mask_mode_dict[self.mask_mode] }\n"""
+
+        if 'raw' not in self.mode:
+            r += (f"""erode_mask_modifier: {self.erode_mask_modifier}\n"""
+                  f"""blur_mask_modifier: {self.blur_mask_modifier}\n"""
+                  f"""motion_blur_power: {self.motion_blur_power}\n""")
+
+        r += f"""output_face_scale: {self.output_face_scale}\n"""
+
+        if 'raw' not in self.mode:
+            r += f"""color_transfer_mode: { ctm_dict[self.color_transfer_mode]}\n"""
+
+        r += super().to_string(filename)
+
+        if 'raw' not in self.mode:
+            r += (f"""image_denoise_power: {self.image_denoise_power}\n"""
+                  f"""bicubic_degrade_power: {self.bicubic_degrade_power}\n"""
+                  f"""color_degrade_power: {self.color_degrade_power}\n"""
+                  f"""export_mask_alpha: {self.export_mask_alpha}\n""")
+
+        r += "================"
+
+        return r
+
+
+class MergerConfigFaceAvatar(MergerConfig):
+
+    def __init__(self, temporal_face_count=0,
+                       add_source_image=False):
+        super().__init__(type=MergerConfig.TYPE_FACE_AVATAR)
+        self.temporal_face_count = temporal_face_count
+
+        #changeable params
+        self.add_source_image = add_source_image
+
+    def copy(self):
+        return copy.copy(self)
+
+    #override
+    def ask_settings(self):
+        self.add_source_image = io.input_bool("Add source image?", False, help_message="Add source image for comparison.")
+        super().ask_settings()
+
+    def toggle_add_source_image(self):
+        self.add_source_image = not self.add_source_image
+
+    #override
+    def __eq__(self, other):
+        #check equality of changeable params
+
+        if isinstance(other, MergerConfigFaceAvatar):
+            return super().__eq__(other) and \
+                   self.add_source_image == other.add_source_image
+
+        return False
+
+    #override
+    def to_string(self, filename):
+        return (f"MergerConfig {filename}:\n"
+                f"add_source_image : {self.add_source_image}\n") + \
+                super().to_string(filename) + "================"
+

merger/__init__.py

@@ -0,0 +1,4 @@
+from .FrameInfo import FrameInfo
+from .MergerConfig import MergerConfig, MergerConfigMasked, MergerConfigFaceAvatar
+from .MergeMasked import MergeMasked
+from .MergeAvatar import MergeFaceAvatar