FaceAligner: added 'head_mode', can be used with HEAD models.

apps/DeepFaceLive/backend/FaceAligner.py

@@ -37,6 +37,7 @@ class FaceAlignerWorker(BackendWorker):
         cs.face_coverage.call_on_number(self.on_cs_face_coverage)
         cs.resolution.call_on_number(self.on_cs_resolution)
         cs.exclude_moving_parts.call_on_flag(self.on_cs_exclude_moving_parts)
+        cs.head_mode.call_on_flag(self.on_cs_head_mode)
         cs.x_offset.call_on_number(self.on_cs_x_offset)
         cs.y_offset.call_on_number(self.on_cs_y_offset)
 
@@ -50,6 +51,9 @@ class FaceAlignerWorker(BackendWorker):
 
         cs.exclude_moving_parts.enable()
         cs.exclude_moving_parts.set_flag(state.exclude_moving_parts if state.exclude_moving_parts is not None else True)
+    
+        cs.head_mode.enable()
+        cs.head_mode.set_flag(state.head_mode if state.head_mode is not None else False)
 
         cs.x_offset.enable()
         cs.x_offset.set_config(lib_csw.Number.Config(min=-1, max=1, step=0.01, decimals=2, allow_instant_update=True))
@@ -82,6 +86,12 @@ class FaceAlignerWorker(BackendWorker):
         self.save_state()
         self.reemit_frame_signal.send()
 
+    def on_cs_head_mode(self, head_mode):
+        state, cs = self.get_state(), self.get_control_sheet()
+        state.head_mode = head_mode
+        self.save_state()
+        self.reemit_frame_signal.send()
+    
     def on_cs_x_offset(self, x_offset):
         state, cs = self.get_state(), self.get_control_sheet()
         cfg = cs.x_offset.get_config()
@@ -113,13 +123,20 @@ class FaceAlignerWorker(BackendWorker):
 
                 if all_is_not_None(state.face_coverage, state.resolution, frame_name, frame_image):
                     for face_id,face_mark in enumerate( bcd.get_face_mark_list() ):
-                        face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_468)
+                        face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_468)
                         if face_ulmrks is None:
-                            face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_68)
+                            face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_68)
-
+                        
+                        head_yaw = None
+                        if state.head_mode:
+                            face_pose = face_mark.get_face_pose()
+                            if face_pose is not None:
+                                head_yaw = face_pose.as_radians()[1]
+                                
                         if face_ulmrks is not None:
                             face_image, uni_mat = face_ulmrks.cut(frame_image, state.face_coverage, state.resolution,
                                                                   exclude_moving_parts=state.exclude_moving_parts,
+                                                                  head_yaw=head_yaw,
                                                                   x_offset=state.x_offset,
                                                                   y_offset=state.y_offset)
 
@@ -155,6 +172,7 @@ class Sheet:
             self.face_coverage = lib_csw.Number.Client()
             self.resolution = lib_csw.Number.Client()
             self.exclude_moving_parts = lib_csw.Flag.Client()
+            self.head_mode = lib_csw.Flag.Client()
             self.x_offset = lib_csw.Number.Client()
             self.y_offset = lib_csw.Number.Client()
 
@@ -164,6 +182,7 @@ class Sheet:
             self.face_coverage = lib_csw.Number.Host()
             self.resolution = lib_csw.Number.Host()
             self.exclude_moving_parts = lib_csw.Flag.Host()
+            self.head_mode = lib_csw.Flag.Host()
             self.x_offset = lib_csw.Number.Host()
             self.y_offset = lib_csw.Number.Host()
 
@@ -171,5 +190,6 @@ class WorkerState(BackendWorkerState):
     face_coverage : float = None
     resolution    : int = None
     exclude_moving_parts : bool = None
+    head_mode : bool = None
     x_offset : float = None
     y_offset : float = None

apps/DeepFaceLive/backend/FaceMarker.py

@@ -4,9 +4,8 @@ import numpy as np
 from modelhub import onnx as onnx_models
 from modelhub import cv as cv_models
 
-from xlib import cv as lib_cv
 from xlib import os as lib_os
-from xlib.facemeta import FaceULandmarks
+from xlib.facemeta import FaceULandmarks, FacePose
 from xlib.image import ImageProcessor
 from xlib.mp import csw as lib_csw
 from xlib.python import all_is_not_None
@@ -15,7 +14,6 @@ from .BackendBase import (BackendConnection, BackendDB, BackendHost,
                           BackendSignal, BackendWeakHeap, BackendWorker,
                           BackendWorkerState)
 
-
 class MarkerType(IntEnum):
     OPENCV_LBF = 0
     GOOGLE_FACEMESH = 1
@@ -175,9 +173,7 @@ class FaceMarkerWorker(BackendWorker):
                                 if is_opencv_lbf:
                                     lmrks = self.opencv_lbf.extract(face_image)[0]
                                 elif is_google_facemesh:
-                                    lmrks = self.google_facemesh.extract(face_image)[0][...,0:2]
+                                    lmrks = self.google_facemesh.extract(face_image)[0]
-
-                                lmrks /= (W,H)
 
                                 if marker_state.temporal_smoothing != 1:
                                     if not is_frame_reemitted or len(self.temporal_lmrks[face_id]) == 0:
@@ -186,11 +182,21 @@ class FaceMarkerWorker(BackendWorker):
 
                                     lmrks = np.mean(self.temporal_lmrks[face_id],0 )
 
-                                face_ulmrks = FaceULandmarks.create (FaceULandmarks.Type.LANDMARKS_2D_68 if is_opencv_lbf else \
+                                if is_google_facemesh:
-                                                                        FaceULandmarks.Type.LANDMARKS_2D_468 if is_google_facemesh else None, lmrks)
+                                    face_mark.set_face_pose(FacePose.from_3D_468_landmarks(lmrks))
+
+                                if is_opencv_lbf:
+                                    lmrks /= (W,H)
+                                elif is_google_facemesh:
+                                    lmrks = lmrks[...,0:2] / (W,H)
+
+                                face_ulmrks = FaceULandmarks.create (FaceULandmarks.Type.LANDMARKS_68 if is_opencv_lbf else \
+                                                                     FaceULandmarks.Type.LANDMARKS_468 if is_google_facemesh else None, lmrks)
 
                                 face_ulmrks = face_ulmrks.transform(face_uni_mat, invert=True)
                                 face_mark.add_face_ulandmarks (face_ulmrks)
+                                
+                               
 
                     self.stop_profile_timing()
                     self.pending_bcd = bcd

apps/DeepFaceLive/ui/QFaceAligner.py

@@ -24,6 +24,9 @@ class QFaceAligner(QBackendPanel):
         q_exclude_moving_parts_label = QLabelPopupInfo(label=L('@QFaceAligner.exclude_moving_parts'), popup_info_text=L('@QFaceAligner.help.exclude_moving_parts') )
         q_exclude_moving_parts = QCheckBoxCSWFlag(cs.exclude_moving_parts, reflect_state_widgets=[q_exclude_moving_parts_label])
 
+        q_head_mode_label = QLabelPopupInfo(label=L('@QFaceAligner.head_mode'), popup_info_text=L('@QFaceAligner.help.head_mode') )
+        q_head_mode = QCheckBoxCSWFlag(cs.head_mode, reflect_state_widgets=[q_head_mode_label])
+
         q_x_offset_label = QLabelPopupInfo(label=L('@QFaceAligner.x_offset'))
         q_x_offset       = QSpinBoxCSWNumber(cs.x_offset, reflect_state_widgets=[q_x_offset_label])
 
@@ -41,6 +44,9 @@ class QFaceAligner(QBackendPanel):
         grid_l.addWidget(q_exclude_moving_parts_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
         grid_l.addWidget(q_exclude_moving_parts, row, 1, alignment=Qt.AlignmentFlag.AlignLeft )
         row += 1
+        grid_l.addWidget(q_head_mode_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
+        grid_l.addWidget(q_head_mode, row, 1, alignment=Qt.AlignmentFlag.AlignLeft )
+        row += 1
         grid_l.addLayout( lib_qt.QXVBoxLayout([q_x_offset_label, q_y_offset_label]), row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
         grid_l.addLayout( lib_qt.QXHBoxLayout([q_x_offset, q_y_offset]), row, 1, alignment=Qt.AlignmentFlag.AlignLeft )
         row += 1

apps/DeepFaceLive/ui/widgets/QBCFaceAlignViewer.py

@@ -57,9 +57,9 @@ class QBCFaceAlignViewer(lib_qt.QXCollapsibleSection):
                         if all_is_not_None(face_align_image_name):
                             face_image = bcd.get_image(face_align_image_name).copy()
 
-                            face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_468)
+                            face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_468)
                             if face_ulmrks is None:
-                                face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_68)
+                                face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_68)
 
                             if face_ulmrks is not None:
                                 lmrks_layer = np.zeros( (self._preview_width, self._preview_width, 4), dtype=np.uint8)

localization/localization.py

@@ -336,6 +336,16 @@ class Localization:
                 'en-US' : 'Increase stabilization by excluding landmarks of moving parts of the face, such as mouth and other.',
                 'ru-RU' : 'Улучшить стабилизацию исключением лицевых точек\nдвижущихся частей лица, таких как рот и других.',
                 'zh-CN' : '通过排除面部移动部分（例如嘴巴和其他你懂的）的特征点来提高稳定性。'},
+    
+    'QFaceAligner.head_mode':{
+                'en-US' : 'Head mode',
+                'ru-RU' : 'Режим головы',
+                'zh-CN' : 'Head mode(没有翻译)'},
+
+    'QFaceAligner.help.head_mode':{
+                'en-US' : 'Head mode. Used with HEAD model.',
+                'ru-RU' : 'Режим головы. Используется с HEAD моделью.',
+                'zh-CN' : 'Head mode. Used with HEAD model.(没有翻译)'},
 
     'QFaceAligner.x_offset':{
                 'en-US' : 'X offset',

xlib/facemeta/FacePose.py

@@ -0,0 +1,50 @@
+from typing import Tuple
+import numpy as np
+from xlib import math as lib_math
+
+
+class FacePose:
+    """
+    Describes face pitch/yaw/roll
+    """
+    def __init__(self):
+        self._pyr : np.ndarray = None
+
+    def __getstate__(self):
+        return self.__dict__.copy()
+
+    def __setstate__(self, d):
+        self.__init__()
+        self.__dict__.update(d)
+
+    def as_radians(self) -> Tuple[float, float, float]:
+        """
+        returns pitch,yaw,roll in radians
+        """
+        return self._pyr.copy()
+
+    def as_degress(self) -> Tuple[float, float, float]:
+        """
+        returns pitch,yaw,roll in degrees
+        """
+        return np.degrees(self._pyr)
+
+    @staticmethod
+    def from_radians(pitch, yaw, roll):
+        """
+        """
+        face_rect = FacePose()
+        face_rect._pyr = np.array([pitch, yaw, roll], np.float32)
+        return face_rect
+
+    @staticmethod
+    def from_3D_468_landmarks(lmrks):
+        """
+        """
+        mat = np.empty((3,3))
+        mat[0,:] = (lmrks[454] - lmrks[234])/np.linalg.norm(lmrks[454] - lmrks[234])
+        mat[1,:] = (lmrks[152] - lmrks[6])/np.linalg.norm(lmrks[152] - lmrks[6])
+        mat[2,:] = np.cross(mat[0, :], mat[1, :])
+        pitch, yaw, roll = lib_math.rotation_matrix_to_euler(mat)
+
+        return FacePose.from_radians(pitch, yaw, roll)

xlib/facemeta/FaceULandmarks.py

@@ -9,13 +9,13 @@ from xlib.math import Affine2DMat, Affine2DUniMat
 
 class FaceULandmarks:
     """
-    Describes face landmarks in uniform coordinates
+    Describes 2D face landmarks in uniform coordinates
     """
 
     class Type(IntEnum):
-        LANDMARKS_2D_5 = 0
+        LANDMARKS_5 = 0
-        LANDMARKS_2D_68 = 1
+        LANDMARKS_68 = 1
-        LANDMARKS_2D_468 = 2
+        LANDMARKS_468 = 2
 
     def __init__(self):
         self._type : FaceULandmarks.Type = None
@@ -32,7 +32,7 @@ class FaceULandmarks:
     def create( type : 'FaceULandmarks.Type', ulmrks : np.ndarray):
         """
 
-         ulmrks np.ndarray  (*,2)
+         ulmrks np.ndarray  (*,2|3)
         """
 
         if not isinstance(type, FaceULandmarks.Type):
@@ -41,17 +41,18 @@ class FaceULandmarks:
         ulmrks = np.float32(ulmrks)
         if len(ulmrks.shape) != 2:
             raise ValueError('ulmrks shape must have rank 2')
-        if type in [FaceULandmarks.Type.LANDMARKS_2D_5, FaceULandmarks.Type.LANDMARKS_2D_68, FaceULandmarks.Type.LANDMARKS_2D_468]:
+
-            if ulmrks.shape[1] != 2:
+        if ulmrks.shape[1] != 2:
-                raise ValueError('ulmrks dim must be == 2')
+            raise ValueError('ulmrks dim must be == 2')
+
         ulmrks_count = ulmrks.shape[0]
-        if type == FaceULandmarks.Type.LANDMARKS_2D_5:
+        if type == FaceULandmarks.Type.LANDMARKS_5:
             if ulmrks_count != 5:
                 raise ValueError('ulmrks_count must be == 5')
-        elif type == FaceULandmarks.Type.LANDMARKS_2D_68:
+        elif type == FaceULandmarks.Type.LANDMARKS_68:
             if ulmrks_count != 68:
                 raise ValueError('ulmrks_count must be == 68')
-        elif type == FaceULandmarks.Type.LANDMARKS_2D_468:
+        elif type == FaceULandmarks.Type.LANDMARKS_468:
             if ulmrks_count != 468:
                 raise ValueError('ulmrks_count must be == 468')
 
@@ -87,13 +88,18 @@ class FaceULandmarks:
 
         if invert:
             mat = cv2.invertAffineTransform (mat)
+
         ulmrks = self._ulmrks.copy()
         ulmrks = np.expand_dims(ulmrks, axis=1)
         ulmrks = cv2.transform(ulmrks, mat, ulmrks.shape).squeeze()
+
         return FaceULandmarks.create(type=self._type, ulmrks=ulmrks)
 
 
-    def calc_cut(self, w_h, coverage : float, output_size : int, exclude_moving_parts : bool, x_offset : float = 0, y_offset : float = 0):
+    def calc_cut(self, w_h, coverage : float, output_size : int,
+                       exclude_moving_parts : bool,
+                       head_yaw : float = None,
+                       x_offset : float = 0, y_offset : float = 0):
         """
         Calculates affine mat for face cut.
 
@@ -102,14 +108,13 @@ class FaceULandmarks:
              mat,       matrix to transform img space to face_image space
              uni_mat    matrix to transform uniform img space to uniform face_image space
         """
-
-        lmrks = (self._ulmrks * w_h ).astype(np.float32)
         type = self._type
+        lmrks = (self._ulmrks * w_h).astype(np.float32)
 
         # estimate landmarks transform from global space to local aligned space with bounds [0..1]
-        if type == FaceULandmarks.Type.LANDMARKS_2D_68:
+        if type == FaceULandmarks.Type.LANDMARKS_68:
             mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:49] , lmrks[54:55] ]), uni_landmarks_68)
-        elif type == FaceULandmarks.Type.LANDMARKS_2D_468:
+        elif type == FaceULandmarks.Type.LANDMARKS_468:
             src_lmrks = lmrks
             dst_lmrks = uni_landmarks_468
             if exclude_moving_parts:
@@ -138,6 +143,10 @@ class FaceULandmarks:
         h_vec = (g_p[1]-g_p[0]).astype(np.float32)
         v_vec = (g_p[3]-g_p[0]).astype(np.float32)
 
+        if head_yaw is not None:
+            # Damp near zero
+            x_offset += -(head_yaw * np.abs(np.tanh(head_yaw*2)) ) * 0.5
+
         g_c += h_vec*x_offset + v_vec*(y_offset-0.08)
 
         l_t = np.array( [ g_c - tb_diag_vec*mod,
@@ -151,7 +160,13 @@ class FaceULandmarks:
         return mat, uni_mat
 
 
-    def cut(self, img : np.ndarray, coverage : float, output_size : int, exclude_moving_parts=False, x_offset : float = 0, y_offset : float = 0) -> Tuple[Affine2DMat, Affine2DUniMat]:
+    def cut(self, img : np.ndarray,
+                  coverage : float,
+                  output_size : int,
+                  exclude_moving_parts : bool = False,
+                  head_yaw : float = None,
+                  x_offset : float = 0,
+                  y_offset : float = 0) -> Tuple[Affine2DMat, Affine2DUniMat]:
         """
         Cut the face to square of output_size from img using landmarks with given parameters
 
@@ -165,15 +180,17 @@ class FaceULandmarks:
 
             exclude_moving_parts(False)     exclude moving parts of the face, such as eyebrows and jaw
 
-            v_offset
+            head_yaw(None)    float     fit the head in center using provided yaw radian value.
-            h_offset    float   uniform h/v offset
+
+            x_offset
+            y_offset    float   uniform x/y offset
 
         returns face_image,
                 uni_mat         uniform affine matrix to transform uniform img space to uniform face_image space
         """
         h,w = img.shape[0:2]
 
-        mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts, x_offset=x_offset, y_offset=y_offset)
+        mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts, head_yaw=head_yaw, x_offset=x_offset, y_offset=y_offset)
 
         face_image = cv2.warpAffine(img, mat, (output_size, output_size), cv2.INTER_CUBIC )
         return face_image, uni_mat

xlib/facemeta/__init__.py

@@ -15,9 +15,9 @@ FaceMark - (mean single face data referencing any image)
            .FaceURect        - a rectangle of the face in source image space
            .list[FaceULandmarks] - a list of unique types of landmarks of the face in source image space
                                     types:
-                                        LANDMARKS_2D_5
+                                        LANDMARKS_5
-                                        LANDMARKS_2D_68
+                                        LANDMARKS_68
-                                        LANDMARKS_2D_468
+                                        LANDMARKS_468
            
            .FaceAlign -  an aligned face from FaceMark
            
@@ -44,5 +44,5 @@ FaceMark - (mean single face data referencing any image)
 
 """
 
-from .face import FaceMark, FaceAlign, FaceSwap, FaceMask, FaceURect, FaceULandmarks
+from .face import FaceMark, FaceAlign, FaceSwap, FaceMask, FaceURect, FaceULandmarks, FacePose
 from .Faceset import Faceset

xlib/facemeta/face.py

@@ -4,7 +4,7 @@ from xlib import math as lib_math
 
 from .FaceULandmarks import FaceULandmarks
 from .FaceURect import FaceURect
-
+from .FacePose import FacePose
 
 class _part_picklable_expandable:
     def __getstate__(self):
@@ -135,6 +135,16 @@ class _part_face_swap:
             raise ValueError('face_swap must be an instance of FaceSwap')
         self._face_swap = face_swap
 
+class _part_face_pose:
+    def __init__(self):
+        self._face_pose : Union['FacePose', None] = None
+
+    def get_face_pose(self) -> Union['FacePose', None]: return self._face_pose
+    def set_face_pose(self, face_pose : 'FacePose'):
+        if not isinstance(face_pose, FacePose):
+            raise ValueError('face_pose must be an instance of FacePose')
+        self._face_pose = face_pose
+
 
 class FaceMark(_part_picklable_expandable,
                 _part_image_name,
@@ -142,6 +152,7 @@ class FaceMark(_part_picklable_expandable,
                 _part_face_urect,
                 _part_face_ulandmarks_list,
                 _part_face_align,
+                _part_face_pose,
                 ):
     """
     Describes meta data of single face.
@@ -154,6 +165,7 @@ class FaceMark(_part_picklable_expandable,
         _part_face_urect.__init__(self)
         _part_face_ulandmarks_list.__init__(self)
         _part_face_align.__init__(self)
+        _part_face_pose.__init__(self)
 
 class FaceAlign(_part_picklable_expandable,
                 _part_image_name,

xlib/math/__init__.py

@@ -1,4 +1,4 @@
 from .Affine2DMat import Affine2DMat, Affine2DUniMat
-from .math_ import (intersect_two_line, polygon_area, segment_length,
+from .math_ import (intersect_two_line, polygon_area, rotation_matrix_to_euler,
-                    segment_to_vector)
+                    segment_length, segment_to_vector)
 from .nms import nms

xlib/math/math_.py

@@ -1,7 +1,23 @@
+import math
+
 import numpy as np
 import numpy.linalg as npla
 
 
+def rotation_matrix_to_euler(R : np.ndarray) -> np.ndarray:
+    sy = math.sqrt(R[0,0] * R[0,0] +  R[1,0] * R[1,0])
+    singular = sy < 1e-6
+    if  not singular :
+        x = math.atan2(R[2,1] , R[2,2])
+        y = math.atan2(-R[2,0], sy)
+        z = math.atan2(R[1,0], R[0,0])
+    else :
+        x = math.atan2(-R[1,2], R[1,1])
+        y = math.atan2(-R[2,0], sy)
+        z = 0
+    return np.array([x, y, z])
+
+
 def segment_length(p1 : np.ndarray, p2 : np.ndarray):
     """
         p1  (2,)