improved face align,

More stable and precise version of the face transformation matrix.
Now full_faces are aligned with the upper and lateral boundaries of the frame,
result: fix of cutted mouth, increase area of the cheeks of side faces
before/after https://i.imgur.com/t9IyGZv.jpg
therefore, additional training is required for existing models.
Optionally, you can re-extract dst faces of your project, if they have problems with cutted mouth or cheeks.

facelib/FANExtractor.py

@@ -83,7 +83,7 @@ class FANExtractor(object):
             for i, lmrks in enumerate(landmarks):
                 try:
                     if lmrks is not None:
-                        image_to_face_mat = LandmarksProcessor.get_transform_mat (lmrks, 256, FaceType.FULL)
+                        image_to_face_mat = LandmarksProcessor.get_transform_mat (lmrks, 256, FaceType.FULL, full_face_align_top=False)
                         face_image = cv2.warpAffine(input_image, image_to_face_mat, (256, 256), cv2.INTER_CUBIC )
 
                         rects2 = second_pass_extractor.extract(face_image, is_bgr=is_bgr)

facelib/LandmarksProcessor.py

@@ -183,6 +183,15 @@ landmarks_68_3D = np.array( [
 [0.205322    , 31.408738    , -21.903670  ],
 [-7.198266   , 30.844876    , -20.328022  ] ], dtype=np.float32)
 
+FaceType_to_padding_remove_align = {
+    FaceType.HALF: (0.0, False),
+    FaceType.MID_FULL: (0.0675, False),
+    FaceType.FULL: (0.2109375, False),
+    FaceType.FULL_NO_ALIGN: (0.2109375, True),
+    FaceType.HEAD: (0.369140625, False),
+    FaceType.HEAD_NO_ALIGN: (0.369140625, True),
+}
+
 def convert_98_to_68(lmrks):
     #jaw
     result = [ lmrks[0] ]
@@ -240,66 +249,63 @@ def transform_points(points, mat, invert=False):
     points = np.squeeze(points)
     return points
 
-def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
+def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0, full_face_align_top=True):
     if not isinstance(image_landmarks, np.ndarray):
         image_landmarks = np.array (image_landmarks)
 
-    """
+    padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
-    if face_type == FaceType.AVATAR:
-        centroid = np.mean (image_landmarks, axis=0)
 
-        mat = umeyama(image_landmarks[17:], landmarks_2D, True)[0:2]
-        a, c = mat[0,0], mat[1,0]
-        scale = math.sqrt((a * a) + (c * c))
-
-        padding = (output_size / 64) * 32
-
-        mat = np.eye ( 2,3 )
-        mat[0,2] = -centroid[0]
-        mat[1,2] = -centroid[1]
-        mat = mat * scale * (output_size / 3)
-        mat[:,2] += output_size / 2
-    else:
-    """
-    remove_align = False
-    if face_type == FaceType.FULL_NO_ALIGN:
-        face_type = FaceType.FULL
-        remove_align = True
-    elif face_type == FaceType.HEAD_NO_ALIGN:
-        face_type = FaceType.HEAD
-        remove_align = True
-
-    if face_type == FaceType.HALF:
-        padding = 0
-    elif face_type == FaceType.MID_FULL:
-        padding = int(output_size * 0.06)
-    elif face_type == FaceType.FULL:
-        padding = (output_size / 64) * 12
-    elif face_type == FaceType.HEAD:
-        padding = (output_size / 64) * 21
-    else:
-        raise ValueError ('wrong face_type: ', face_type)
-
-    #mat = umeyama(image_landmarks[17:], landmarks_2D, True)[0:2]
     mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
+    l_p = transform_points (  np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True)
+    l_c = l_p[4]
 
-    mat = mat * (output_size - 2 * padding)
+    tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32)
-    mat[:,2] += padding
+    tb_diag_vec /= npla.norm(tb_diag_vec)
-    mat *= (1 / scale)
+    bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32)
-    mat[:,2] += -output_size*( ( (1 / scale) - 1.0 ) / 2 )
+    bt_diag_vec /= npla.norm(bt_diag_vec)
+    
+    mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) )
+    
+    l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), 
+                      np.round( l_c + bt_diag_vec*mod ), 
+                      np.round( l_c + tb_diag_vec*mod ) ] )    
+
+    pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
+    mat = cv2.getAffineTransform(l_t,pts2)
+    
+    if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN):
+        lmrks2 = expand_eyebrows(image_landmarks)    
+        lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False )     
+        y_diff = np.float32( (0,np.min(lmrks2_[:,1])) )   
+
+        y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True)
+        y_diff = y_diff[1]-y_diff[0]
+        
+        x_diff = np.float32((0,0))
+        
+        lmrks2_ = transform_points( [ lmrks2[0], lmrks2[16] ], mat, False )   
+        if lmrks2_[0,0] < 0:
+            x_diff = lmrks2_[0,0]        
+            x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
+            x_diff = x_diff[1]-x_diff[0]        
+        elif lmrks2_[1,0] >= output_size:
+            x_diff = lmrks2_[1,0]-(output_size-1)
+            x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
+            x_diff = x_diff[1]-x_diff[0]    
+        
+        mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2)
         
     if remove_align:
-        bbox = transform_points ( [ (0,0), (0,output_size-1), (output_size-1, output_size-1), (output_size-1,0) ], mat, True)
+        bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True)
         area = mathlib.polygon_area(bbox[:,0], bbox[:,1] )
         side = math.sqrt(area) / 2
         center = transform_points ( [(output_size/2,output_size/2)], mat, True)
-
+        pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[-side,side] ))
-        pts1 = np.float32([ center+[-side,-side], center+[side,-side], center+[-side,side] ])
-        pts2 = np.float32([[0,0],[output_size-1,0],[0,output_size-1]])
         mat = cv2.getAffineTransform(pts1,pts2)
 
     return mat
 
+
 def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0):
     if len(lmrks) != 68:
         raise Exception('works only with 68 landmarks')

samplelib/SampleGeneratorFace.py

@@ -54,7 +54,7 @@ class SampleGeneratorFace(SampleGeneratorBase):
         if self.samples_len == 0:
             raise ValueError('No training data provided.')
         
-        ct_samples = SampleLoader.load (SampleType.FACE, random_ct_samples_path) if random_ct_samples_path is not None else None
+        ct_samples = SampleLoader.load (SampleType.FACE, random_ct_samples_path, use_caching=use_caching) if random_ct_samples_path is not None else None
 
         if self.debug:
             self.generators_count = 1

samplelib/SampleProcessor.py

@@ -177,8 +177,8 @@ class SampleProcessor(object):
                             if len(mask.shape) == 2:
                                 mask = mask[...,np.newaxis]
 
-                            img = np.concatenate( (img, mask ), -1 )
+                            
-                        return img
+                        return img, mask
 
                     img = sample_bgr
 
@@ -222,14 +222,20 @@ class SampleProcessor(object):
                             img =  cv2.warpAffine( img, LandmarksProcessor.get_transform_mat (sample.landmarks, sample.shape[0], target_ft), (sample.shape[0],sample.shape[0]), flags=cv2.INTER_CUBIC )
                             mask = cv2.warpAffine( mask, LandmarksProcessor.get_transform_mat (sample.landmarks, sample.shape[0], target_ft), (sample.shape[0],sample.shape[0]), flags=cv2.INTER_CUBIC )
                             #then apply transforms
-                            img = do_transform (img, mask)
+                            img, mask = do_transform (img, mask)
+                            img = np.concatenate( (img, mask ), -1 )
                             img = cv2.resize( img, (resolution,resolution), cv2.INTER_CUBIC )
                         else:
-                            img = do_transform (img, mask)
+                            img, mask = do_transform (img, mask)
-                            img = cv2.warpAffine( img, LandmarksProcessor.get_transform_mat (sample.landmarks, resolution, target_ft), (resolution,resolution), borderMode=(cv2.BORDER_REPLICATE if border_replicate else cv2.BORDER_CONSTANT), flags=cv2.INTER_CUBIC )
+                            
+                            mat = LandmarksProcessor.get_transform_mat (sample.landmarks, resolution, target_ft)
+                            img = cv2.warpAffine( img, mat, (resolution,resolution), borderMode=(cv2.BORDER_REPLICATE if border_replicate else cv2.BORDER_CONSTANT), flags=cv2.INTER_CUBIC )
+                            mask = cv2.warpAffine( mask, mat, (resolution,resolution), borderMode=cv2.BORDER_CONSTANT, flags=cv2.INTER_CUBIC )
+                            img = np.concatenate( (img, mask[...,None] ), -1 )
 
                     else:
-                        img = do_transform (img, mask)
+                        img, mask = do_transform (img, mask)
+                        img = np.concatenate( (img, mask ), -1 )
                         img = cv2.resize( img, (resolution,resolution), cv2.INTER_CUBIC )
 
                     if random_sub_res != 0: