Fix in-progress

facelib/LandmarksProcessor.py

@@ -38,7 +38,6 @@ landmarks_68_pt = { "mouth": (48,68),
                    "nose": (27, 36),  # missed one point
                    "jaw": (0, 17)}
 
-
 landmarks_68_3D = np.array([
     [-73.393523, -29.801432, 47.667532],
     [-72.775014, -10.949766, 45.909403],
@@ -109,6 +108,7 @@ landmarks_68_3D = np.array( [
     [0.205322, 31.408738, -21.903670],
     [-7.198266, 30.844876, -20.328022]], dtype=np.float32)
 
+
 def transform_points(points, mat, invert=False):
     if invert:
         mat = cv2.invertAffineTransform(mat)
@@ -117,6 +117,39 @@ def transform_points(points, mat, invert=False):
     points = np.squeeze(points)
     return points
 
+
+def get_translation_scale_tan_rotation_of_mat(mat):
+    # TODO
+    # extracting rotation, scale values from 2d transformation matrix
+    # https://math.stackexchange.com/questions/13150/extracting-rotation-scale-values-from-2d-transformation-matrix/13165#13165
+    a, b, tx = mat[0, :]
+    c, d, ty = mat[1, :]
+
+    sx = np.sign(a) * math.sqrt(a ** 2 + b ** 2)
+    sy = np.sign(d) * math.sqrt(c ** 2 + d ** 2)
+
+    tan_psi = -b / a
+    return {
+        'tx': tx,
+        'ty': ty,
+        'sx': sx,
+        'sy': sy,
+        'tan_psi': tan_psi
+    }
+
+
+def get_scale_of_mat(mat):
+    # TODO
+    return np.mean(np.sqrt(np.sum(np.square(mat[:, :2]), axis=1)))
+
+
+def calc_image_size_for_unscaled(image_landmarks, face_type, scale=1.0):
+    # TODO
+    mat = get_transform_mat(image_landmarks, 1, face_type, scale=scale)
+    scale = get_scale_of_mat(mat)
+    return int(1 / scale)
+
+
 def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0):
     if not isinstance(image_landmarks, np.ndarray):
         image_landmarks = np.array(image_landmarks)
@@ -145,19 +178,32 @@ def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
     elif face_type == FaceType.FULL:
         padding = (output_size / 64) * 12
     elif face_type == FaceType.HEAD:
-        padding = (output_size / 64) * 24
+        padding = (output_size / 64) * 18
     else:
         raise ValueError('wrong face_type: ', face_type)
 
-
     mat = umeyama(image_landmarks[17:], landmarks_2D, True)[0:2]
+
+    # TODO
+    if output_size != 1:
+        print(
+            f'PREPAD - get_translation_scale_tan_rotation_of_mat: {get_translation_scale_tan_rotation_of_mat(mat)}')
     mat = mat * (output_size - 2 * padding)
     mat[:, 2] += padding
     mat *= (1 / scale)
     mat[:, 2] += -output_size * (((1 / scale) - 1.0) / 2)
 
+    # TODO
+    if output_size != 1:
+        print(
+            f'POSTPAD - get_translation_scale_tan_rotation_of_mat: {get_translation_scale_tan_rotation_of_mat(mat)}')
+    else:
+        print(
+            f'CALC SCALE - get_translation_scale_tan_rotation_of_mat: {get_translation_scale_tan_rotation_of_mat(mat)}')
+
     if remove_align:
-        bbox = transform_points ( [ (0,0), (0,output_size-1), (output_size-1, output_size-1), (output_size-1,0) ], mat, True)
+        bbox = transform_points([(0, 0), (0, output_size - 1), (output_size - 1, output_size - 1),
+                                 (output_size - 1, 0)], mat, True)
         area = mathlib.polygon_area(bbox[:, 0], bbox[:, 1])
         side = math.sqrt(area) / 2
         center = transform_points([(output_size / 2, output_size / 2)], mat, True)
@@ -168,6 +214,7 @@ def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
 
     return mat
 
+
 def get_image_hull_mask(image_shape, image_landmarks, ie_polys=None):
     if len(image_landmarks) != 68:
         raise Exception('get_image_hull_mask works only with 68 landmarks')
@@ -214,6 +261,7 @@ def get_image_hull_mask (image_shape, image_landmarks, ie_polys=None):
 
     return hull_mask
 
+
 def get_image_eye_mask(image_shape, image_landmarks):
     if len(image_landmarks) != 68:
         raise Exception('get_image_eye_mask works only with 68 landmarks')
@@ -225,8 +273,8 @@ def get_image_eye_mask (image_shape, image_landmarks):
 
     return hull_mask
 
-def blur_image_hull_mask (hull_mask):
 
+def blur_image_hull_mask(hull_mask):
     maxregion = np.argwhere(hull_mask == 1.0)
     miny, minx = maxregion.min(axis=0)[:2]
     maxy, maxx = maxregion.max(axis=0)[:2]
@@ -238,12 +286,14 @@ def blur_image_hull_mask (hull_mask):
     ero = int(lowest_len * 0.085)
     blur = int(lowest_len * 0.10)
 
-    hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 )
+    hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)),
+                          iterations=1)
     hull_mask = cv2.blur(hull_mask, (blur, blur))
     hull_mask = np.expand_dims(hull_mask, -1)
 
     return hull_mask
 
+
 mirror_idxs = [
     [0, 16],
     [1, 15],
@@ -279,6 +329,7 @@ mirror_idxs = [
     [60, 64],
     [61, 63]]
 
+
 def mirror_landmarks(landmarks, val):
     result = landmarks.copy()
 
@@ -288,7 +339,9 @@ def mirror_landmarks (landmarks, val):
     result[:, 0] = val - result[:, 0] - 1
     return result
 
-def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=False, ie_polys=None):
+
+def draw_landmarks(image, image_landmarks, color=(0, 255, 0), transparent_mask=False,
+                   ie_polys=None):
     if len(image_landmarks) != 68:
         raise Exception('get_image_eye_mask works only with 68 landmarks')
 
@@ -303,13 +356,15 @@ def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=Fa
     nose = int_lmrks[slice(*landmarks_68_pt["nose"])]
 
     # open shapes
-    cv2.polylines(image, tuple(np.array([v]) for v in ( right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])) )),
+    cv2.polylines(image, tuple(np.array([v]) for v in (
+    right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])))),
                   False, color, lineType=cv2.LINE_AA)
     # closed shapes
     cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)),
                   True, color, lineType=cv2.LINE_AA)
     # the rest of the cicles
-    for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0):
+    for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose),
+                               axis=0):
         cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA)
     # jaw big circles
     for x, y in jaw:
@@ -319,14 +374,20 @@ def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=Fa
         mask = get_image_hull_mask(image.shape, image_landmarks, ie_polys)
         image[...] = (image * (1 - mask) + image * mask / 2)[...]
 
-def draw_rect_landmarks (image, rect, image_landmarks, face_size, face_type, transparent_mask=False, ie_polys=None, landmarks_color=(0,255,0) ):
+
-    draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask, ie_polys=ie_polys)
+def draw_rect_landmarks(image, rect, image_landmarks, face_size, face_type, transparent_mask=False,
+                        ie_polys=None, landmarks_color=(0, 255, 0)):
+    draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask,
+                   ie_polys=ie_polys)
     imagelib.draw_rect(image, rect, (255, 0, 0), 2)
 
     image_to_face_mat = get_transform_mat(image_landmarks, face_size, face_type)
-    points = transform_points ( [ (0,0), (0,face_size-1), (face_size-1, face_size-1), (face_size-1,0) ], image_to_face_mat, True)
+    points = transform_points(
+        [(0, 0), (0, face_size - 1), (face_size - 1, face_size - 1), (face_size - 1, 0)],
+        image_to_face_mat, True)
     imagelib.draw_polygon(image, points, (0, 0, 255), 2)
 
+
 def calc_face_pitch(landmarks):
     if not isinstance(landmarks, np.ndarray):
         landmarks = np.array(landmarks)
@@ -334,13 +395,17 @@ def calc_face_pitch(landmarks):
     b = landmarks[8][1]
     return float(b - t)
 
+
 def calc_face_yaw(landmarks):
     if not isinstance(landmarks, np.ndarray):
         landmarks = np.array(landmarks)
-    l = ( (landmarks[27][0]-landmarks[0][0]) + (landmarks[28][0]-landmarks[1][0]) + (landmarks[29][0]-landmarks[2][0]) ) / 3.0
+    l = ((landmarks[27][0] - landmarks[0][0]) + (landmarks[28][0] - landmarks[1][0]) + (
-    r = ( (landmarks[16][0]-landmarks[27][0]) + (landmarks[15][0]-landmarks[28][0]) + (landmarks[14][0]-landmarks[29][0]) ) / 3.0
+                landmarks[29][0] - landmarks[2][0])) / 3.0
+    r = ((landmarks[16][0] - landmarks[27][0]) + (landmarks[15][0] - landmarks[28][0]) + (
+                landmarks[14][0] - landmarks[29][0])) / 3.0
     return float(r - l)
 
+
 # returns pitch,yaw,roll [-1...+1]
 def estimate_pitch_yaw_roll(aligned_256px_landmarks):
     shape = (256, 256)

mainscripts/Extractor.py

@@ -48,6 +48,7 @@ class ExtractSubprocessor(Subprocessor):
             self.final_output_path  = Path(client_dict['final_output_dir']) if 'final_output_dir' in client_dict.keys() else None
             self.debug_dir    = client_dict['debug_dir']
             self.image_size = client_dict['image_size']
+            self.log_info(f'on_initialize: {client_dict}')
 
 
 
@@ -220,7 +221,8 @@ class ExtractSubprocessor(Subprocessor):
                     debug_image = image.copy()
 
                 if src_dflimg is not None and len(rects) != 1:
-                    #if re-extracting from dflimg and more than 1 or zero faces detected - dont process and just copy it
+                    # if re-extracting from dflimg and more than 1 or zero faces detected:
+                    # don't process and just copy it
                     print("src_dflimg is not None and len(rects) != 1", str(filename_path) )
                     output_file = str(self.final_output_path / filename_path.name)
                     if str(filename_path) != str(output_file):
@@ -238,30 +240,36 @@ class ExtractSubprocessor(Subprocessor):
 
                         rect = np.array(rect)
                         rect_area = mathlib.polygon_area(np.array(rect[[0, 2, 2, 0]]), np.array(rect[[1, 1, 3, 3]]))
-                        if self.image_size == 0:
-                            self.image_size = int(math.sqrt(rect_area))
 
+                        # `self.image_size` is the output size for the entire process,
+                        # we don't want to overwrite it
+                        face_image_size = self.image_size
+                        # TODO
+                        self.log_info(f'BEFORE if face_image_size==0: {face_image_size}')
+                        if face_image_size == 0:
+                            face_image_size = LandmarksProcessor.calc_image_size_for_unscaled(image_landmarks, 1, self.face_type)
+                            # TODO
+                            self.log_info(f'AFTER if face_image_size==0: {face_image_size}')
 
                         if self.face_type == FaceType.MARK_ONLY:
                             image_to_face_mat = None
                             face_image = image
                             face_image_landmarks = image_landmarks
                         else:
-                            image_to_face_mat = LandmarksProcessor.get_transform_mat (image_landmarks, self.image_size, self.face_type)
+                            image_to_face_mat = LandmarksProcessor.get_transform_mat(image_landmarks, face_image_size, self.face_type)
-
+                            face_image = cv2.warpAffine(image, image_to_face_mat, (face_image_size, face_image_size), cv2.INTER_LANCZOS4)
-                            face_image = cv2.warpAffine(image, image_to_face_mat, (self.image_size, self.image_size), cv2.INTER_LANCZOS4)
+                            # TODO
+                            self.log_info(f'warpAffine size: {face_image.shape[[1, 0]]}')
                             face_image_landmarks = LandmarksProcessor.transform_points(image_landmarks, image_to_face_mat)
-
+                            landmarks_bbox = LandmarksProcessor.transform_points([(0,0), (0, face_image_size-1), (face_image_size-1, face_image_size-1), (face_image_size-1,0) ], image_to_face_mat, True)
-                            landmarks_bbox = LandmarksProcessor.transform_points ( [ (0,0), (0,self.image_size-1), (self.image_size-1, self.image_size-1), (self.image_size-1,0) ], image_to_face_mat, True)
-
 
                             landmarks_area = mathlib.polygon_area(landmarks_bbox[:,0], landmarks_bbox[:,1] )
 
-                            if landmarks_area > 4*rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area
+                            if self.face_type is not FaceType.HEAD and landmarks_area > 4*rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area
                                 continue
 
                             if self.debug_dir is not None:
-                                LandmarksProcessor.draw_rect_landmarks (debug_image, rect, image_landmarks, self.image_size, self.face_type, transparent_mask=True)
+                                LandmarksProcessor.draw_rect_landmarks(debug_image, rect, image_landmarks, face_image_size, self.face_type, transparent_mask=True)
 
                         if filename_path.suffix == '.jpg':