From 72166cc1907ad54f7b5ea50455903d7b1692d113 Mon Sep 17 00:00:00 2001 From: iperov Date: Sat, 7 Aug 2021 19:16:25 +0400 Subject: [PATCH] FaceAligner: added 'head_mode', can be used with HEAD models. --- apps/DeepFaceLive/backend/FaceAligner.py | 26 ++++++++- apps/DeepFaceLive/backend/FaceMarker.py | 22 ++++--- apps/DeepFaceLive/ui/QFaceAligner.py | 6 ++ .../ui/widgets/QBCFaceAlignViewer.py | 4 +- localization/localization.py | 10 ++++ xlib/facemeta/FacePose.py | 50 ++++++++++++++++ xlib/facemeta/FaceULandmarks.py | 57 ++++++++++++------- xlib/facemeta/__init__.py | 8 +-- xlib/facemeta/face.py | 14 ++++- xlib/math/__init__.py | 4 +- xlib/math/math_.py | 16 ++++++ 11 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 xlib/facemeta/FacePose.py diff --git a/apps/DeepFaceLive/backend/FaceAligner.py b/apps/DeepFaceLive/backend/FaceAligner.py index 14392ee..e3a9d7d 100644 --- a/apps/DeepFaceLive/backend/FaceAligner.py +++ b/apps/DeepFaceLive/backend/FaceAligner.py @@ -37,6 +37,7 @@ class FaceAlignerWorker(BackendWorker): cs.face_coverage.call_on_number(self.on_cs_face_coverage) cs.resolution.call_on_number(self.on_cs_resolution) cs.exclude_moving_parts.call_on_flag(self.on_cs_exclude_moving_parts) + cs.head_mode.call_on_flag(self.on_cs_head_mode) cs.x_offset.call_on_number(self.on_cs_x_offset) cs.y_offset.call_on_number(self.on_cs_y_offset) @@ -50,6 +51,9 @@ class FaceAlignerWorker(BackendWorker): cs.exclude_moving_parts.enable() cs.exclude_moving_parts.set_flag(state.exclude_moving_parts if state.exclude_moving_parts is not None else True) + + cs.head_mode.enable() + cs.head_mode.set_flag(state.head_mode if state.head_mode is not None else False) cs.x_offset.enable() cs.x_offset.set_config(lib_csw.Number.Config(min=-1, max=1, step=0.01, decimals=2, allow_instant_update=True)) @@ -82,6 +86,12 @@ class FaceAlignerWorker(BackendWorker): self.save_state() self.reemit_frame_signal.send() + def on_cs_head_mode(self, head_mode): + state, cs = self.get_state(), self.get_control_sheet() + state.head_mode = head_mode + self.save_state() + self.reemit_frame_signal.send() + def on_cs_x_offset(self, x_offset): state, cs = self.get_state(), self.get_control_sheet() cfg = cs.x_offset.get_config() @@ -113,13 +123,20 @@ class FaceAlignerWorker(BackendWorker): if all_is_not_None(state.face_coverage, state.resolution, frame_name, frame_image): for face_id,face_mark in enumerate( bcd.get_face_mark_list() ): - face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_468) + face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_468) if face_ulmrks is None: - face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_68) - + face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_68) + + head_yaw = None + if state.head_mode: + face_pose = face_mark.get_face_pose() + if face_pose is not None: + head_yaw = face_pose.as_radians()[1] + if face_ulmrks is not None: face_image, uni_mat = face_ulmrks.cut(frame_image, state.face_coverage, state.resolution, exclude_moving_parts=state.exclude_moving_parts, + head_yaw=head_yaw, x_offset=state.x_offset, y_offset=state.y_offset) @@ -155,6 +172,7 @@ class Sheet: self.face_coverage = lib_csw.Number.Client() self.resolution = lib_csw.Number.Client() self.exclude_moving_parts = lib_csw.Flag.Client() + self.head_mode = lib_csw.Flag.Client() self.x_offset = lib_csw.Number.Client() self.y_offset = lib_csw.Number.Client() @@ -164,6 +182,7 @@ class Sheet: self.face_coverage = lib_csw.Number.Host() self.resolution = lib_csw.Number.Host() self.exclude_moving_parts = lib_csw.Flag.Host() + self.head_mode = lib_csw.Flag.Host() self.x_offset = lib_csw.Number.Host() self.y_offset = lib_csw.Number.Host() @@ -171,5 +190,6 @@ class WorkerState(BackendWorkerState): face_coverage : float = None resolution : int = None exclude_moving_parts : bool = None + head_mode : bool = None x_offset : float = None y_offset : float = None diff --git a/apps/DeepFaceLive/backend/FaceMarker.py b/apps/DeepFaceLive/backend/FaceMarker.py index c679f4f..5479c82 100644 --- a/apps/DeepFaceLive/backend/FaceMarker.py +++ b/apps/DeepFaceLive/backend/FaceMarker.py @@ -4,9 +4,8 @@ import numpy as np from modelhub import onnx as onnx_models from modelhub import cv as cv_models -from xlib import cv as lib_cv from xlib import os as lib_os -from xlib.facemeta import FaceULandmarks +from xlib.facemeta import FaceULandmarks, FacePose from xlib.image import ImageProcessor from xlib.mp import csw as lib_csw from xlib.python import all_is_not_None @@ -15,7 +14,6 @@ from .BackendBase import (BackendConnection, BackendDB, BackendHost, BackendSignal, BackendWeakHeap, BackendWorker, BackendWorkerState) - class MarkerType(IntEnum): OPENCV_LBF = 0 GOOGLE_FACEMESH = 1 @@ -175,9 +173,7 @@ class FaceMarkerWorker(BackendWorker): if is_opencv_lbf: lmrks = self.opencv_lbf.extract(face_image)[0] elif is_google_facemesh: - lmrks = self.google_facemesh.extract(face_image)[0][...,0:2] - - lmrks /= (W,H) + lmrks = self.google_facemesh.extract(face_image)[0] if marker_state.temporal_smoothing != 1: if not is_frame_reemitted or len(self.temporal_lmrks[face_id]) == 0: @@ -186,11 +182,21 @@ class FaceMarkerWorker(BackendWorker): lmrks = np.mean(self.temporal_lmrks[face_id],0 ) - face_ulmrks = FaceULandmarks.create (FaceULandmarks.Type.LANDMARKS_2D_68 if is_opencv_lbf else \ - FaceULandmarks.Type.LANDMARKS_2D_468 if is_google_facemesh else None, lmrks) + if is_google_facemesh: + face_mark.set_face_pose(FacePose.from_3D_468_landmarks(lmrks)) + + if is_opencv_lbf: + lmrks /= (W,H) + elif is_google_facemesh: + lmrks = lmrks[...,0:2] / (W,H) + + face_ulmrks = FaceULandmarks.create (FaceULandmarks.Type.LANDMARKS_68 if is_opencv_lbf else \ + FaceULandmarks.Type.LANDMARKS_468 if is_google_facemesh else None, lmrks) face_ulmrks = face_ulmrks.transform(face_uni_mat, invert=True) face_mark.add_face_ulandmarks (face_ulmrks) + + self.stop_profile_timing() self.pending_bcd = bcd diff --git a/apps/DeepFaceLive/ui/QFaceAligner.py b/apps/DeepFaceLive/ui/QFaceAligner.py index 2eeaeb9..5ad08c4 100644 --- a/apps/DeepFaceLive/ui/QFaceAligner.py +++ b/apps/DeepFaceLive/ui/QFaceAligner.py @@ -24,6 +24,9 @@ class QFaceAligner(QBackendPanel): q_exclude_moving_parts_label = QLabelPopupInfo(label=L('@QFaceAligner.exclude_moving_parts'), popup_info_text=L('@QFaceAligner.help.exclude_moving_parts') ) q_exclude_moving_parts = QCheckBoxCSWFlag(cs.exclude_moving_parts, reflect_state_widgets=[q_exclude_moving_parts_label]) + q_head_mode_label = QLabelPopupInfo(label=L('@QFaceAligner.head_mode'), popup_info_text=L('@QFaceAligner.help.head_mode') ) + q_head_mode = QCheckBoxCSWFlag(cs.head_mode, reflect_state_widgets=[q_head_mode_label]) + q_x_offset_label = QLabelPopupInfo(label=L('@QFaceAligner.x_offset')) q_x_offset = QSpinBoxCSWNumber(cs.x_offset, reflect_state_widgets=[q_x_offset_label]) @@ -41,6 +44,9 @@ class QFaceAligner(QBackendPanel): grid_l.addWidget(q_exclude_moving_parts_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter ) grid_l.addWidget(q_exclude_moving_parts, row, 1, alignment=Qt.AlignmentFlag.AlignLeft ) row += 1 + grid_l.addWidget(q_head_mode_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter ) + grid_l.addWidget(q_head_mode, row, 1, alignment=Qt.AlignmentFlag.AlignLeft ) + row += 1 grid_l.addLayout( lib_qt.QXVBoxLayout([q_x_offset_label, q_y_offset_label]), row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter ) grid_l.addLayout( lib_qt.QXHBoxLayout([q_x_offset, q_y_offset]), row, 1, alignment=Qt.AlignmentFlag.AlignLeft ) row += 1 diff --git a/apps/DeepFaceLive/ui/widgets/QBCFaceAlignViewer.py b/apps/DeepFaceLive/ui/widgets/QBCFaceAlignViewer.py index 88c4d58..f3f9c40 100644 --- a/apps/DeepFaceLive/ui/widgets/QBCFaceAlignViewer.py +++ b/apps/DeepFaceLive/ui/widgets/QBCFaceAlignViewer.py @@ -57,9 +57,9 @@ class QBCFaceAlignViewer(lib_qt.QXCollapsibleSection): if all_is_not_None(face_align_image_name): face_image = bcd.get_image(face_align_image_name).copy() - face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_468) + face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_468) if face_ulmrks is None: - face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_68) + face_ulmrks = face_align.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_68) if face_ulmrks is not None: lmrks_layer = np.zeros( (self._preview_width, self._preview_width, 4), dtype=np.uint8) diff --git a/localization/localization.py b/localization/localization.py index d6fd387..0983ede 100644 --- a/localization/localization.py +++ b/localization/localization.py @@ -336,6 +336,16 @@ class Localization: 'en-US' : 'Increase stabilization by excluding landmarks of moving parts of the face, such as mouth and other.', 'ru-RU' : 'Улучшить стабилизацию исключением лицевых точек\nдвижущихся частей лица, таких как рот и других.', 'zh-CN' : '通过排除面部移动部分（例如嘴巴和其他你懂的）的特征点来提高稳定性。'}, + + 'QFaceAligner.head_mode':{ + 'en-US' : 'Head mode', + 'ru-RU' : 'Режим головы', + 'zh-CN' : 'Head mode(没有翻译)'}, + + 'QFaceAligner.help.head_mode':{ + 'en-US' : 'Head mode. Used with HEAD model.', + 'ru-RU' : 'Режим головы. Используется с HEAD моделью.', + 'zh-CN' : 'Head mode. Used with HEAD model.(没有翻译)'}, 'QFaceAligner.x_offset':{ 'en-US' : 'X offset', diff --git a/xlib/facemeta/FacePose.py b/xlib/facemeta/FacePose.py new file mode 100644 index 0000000..f1fda79 --- /dev/null +++ b/xlib/facemeta/FacePose.py @@ -0,0 +1,50 @@ +from typing import Tuple +import numpy as np +from xlib import math as lib_math + + +class FacePose: + """ + Describes face pitch/yaw/roll + """ + def __init__(self): + self._pyr : np.ndarray = None + + def __getstate__(self): + return self.__dict__.copy() + + def __setstate__(self, d): + self.__init__() + self.__dict__.update(d) + + def as_radians(self) -> Tuple[float, float, float]: + """ + returns pitch,yaw,roll in radians + """ + return self._pyr.copy() + + def as_degress(self) -> Tuple[float, float, float]: + """ + returns pitch,yaw,roll in degrees + """ + return np.degrees(self._pyr) + + @staticmethod + def from_radians(pitch, yaw, roll): + """ + """ + face_rect = FacePose() + face_rect._pyr = np.array([pitch, yaw, roll], np.float32) + return face_rect + + @staticmethod + def from_3D_468_landmarks(lmrks): + """ + """ + mat = np.empty((3,3)) + mat[0,:] = (lmrks[454] - lmrks[234])/np.linalg.norm(lmrks[454] - lmrks[234]) + mat[1,:] = (lmrks[152] - lmrks[6])/np.linalg.norm(lmrks[152] - lmrks[6]) + mat[2,:] = np.cross(mat[0, :], mat[1, :]) + pitch, yaw, roll = lib_math.rotation_matrix_to_euler(mat) + + return FacePose.from_radians(pitch, yaw, roll) diff --git a/xlib/facemeta/FaceULandmarks.py b/xlib/facemeta/FaceULandmarks.py index aa0c3b5..a878dd7 100644 --- a/xlib/facemeta/FaceULandmarks.py +++ b/xlib/facemeta/FaceULandmarks.py @@ -9,13 +9,13 @@ from xlib.math import Affine2DMat, Affine2DUniMat class FaceULandmarks: """ - Describes face landmarks in uniform coordinates + Describes 2D face landmarks in uniform coordinates """ class Type(IntEnum): - LANDMARKS_2D_5 = 0 - LANDMARKS_2D_68 = 1 - LANDMARKS_2D_468 = 2 + LANDMARKS_5 = 0 + LANDMARKS_68 = 1 + LANDMARKS_468 = 2 def __init__(self): self._type : FaceULandmarks.Type = None @@ -32,7 +32,7 @@ class FaceULandmarks: def create( type : 'FaceULandmarks.Type', ulmrks : np.ndarray): """ - ulmrks np.ndarray (*,2) + ulmrks np.ndarray (*,2|3) """ if not isinstance(type, FaceULandmarks.Type): @@ -41,17 +41,18 @@ class FaceULandmarks: ulmrks = np.float32(ulmrks) if len(ulmrks.shape) != 2: raise ValueError('ulmrks shape must have rank 2') - if type in [FaceULandmarks.Type.LANDMARKS_2D_5, FaceULandmarks.Type.LANDMARKS_2D_68, FaceULandmarks.Type.LANDMARKS_2D_468]: - if ulmrks.shape[1] != 2: - raise ValueError('ulmrks dim must be == 2') + + if ulmrks.shape[1] != 2: + raise ValueError('ulmrks dim must be == 2') + ulmrks_count = ulmrks.shape[0] - if type == FaceULandmarks.Type.LANDMARKS_2D_5: + if type == FaceULandmarks.Type.LANDMARKS_5: if ulmrks_count != 5: raise ValueError('ulmrks_count must be == 5') - elif type == FaceULandmarks.Type.LANDMARKS_2D_68: + elif type == FaceULandmarks.Type.LANDMARKS_68: if ulmrks_count != 68: raise ValueError('ulmrks_count must be == 68') - elif type == FaceULandmarks.Type.LANDMARKS_2D_468: + elif type == FaceULandmarks.Type.LANDMARKS_468: if ulmrks_count != 468: raise ValueError('ulmrks_count must be == 468') @@ -87,13 +88,18 @@ class FaceULandmarks: if invert: mat = cv2.invertAffineTransform (mat) + ulmrks = self._ulmrks.copy() ulmrks = np.expand_dims(ulmrks, axis=1) ulmrks = cv2.transform(ulmrks, mat, ulmrks.shape).squeeze() + return FaceULandmarks.create(type=self._type, ulmrks=ulmrks) - def calc_cut(self, w_h, coverage : float, output_size : int, exclude_moving_parts : bool, x_offset : float = 0, y_offset : float = 0): + def calc_cut(self, w_h, coverage : float, output_size : int, + exclude_moving_parts : bool, + head_yaw : float = None, + x_offset : float = 0, y_offset : float = 0): """ Calculates affine mat for face cut. @@ -102,14 +108,13 @@ class FaceULandmarks: mat, matrix to transform img space to face_image space uni_mat matrix to transform uniform img space to uniform face_image space """ - - lmrks = (self._ulmrks * w_h ).astype(np.float32) type = self._type + lmrks = (self._ulmrks * w_h).astype(np.float32) # estimate landmarks transform from global space to local aligned space with bounds [0..1] - if type == FaceULandmarks.Type.LANDMARKS_2D_68: + if type == FaceULandmarks.Type.LANDMARKS_68: mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:49] , lmrks[54:55] ]), uni_landmarks_68) - elif type == FaceULandmarks.Type.LANDMARKS_2D_468: + elif type == FaceULandmarks.Type.LANDMARKS_468: src_lmrks = lmrks dst_lmrks = uni_landmarks_468 if exclude_moving_parts: @@ -138,6 +143,10 @@ class FaceULandmarks: h_vec = (g_p[1]-g_p[0]).astype(np.float32) v_vec = (g_p[3]-g_p[0]).astype(np.float32) + if head_yaw is not None: + # Damp near zero + x_offset += -(head_yaw * np.abs(np.tanh(head_yaw*2)) ) * 0.5 + g_c += h_vec*x_offset + v_vec*(y_offset-0.08) l_t = np.array( [ g_c - tb_diag_vec*mod, @@ -151,7 +160,13 @@ class FaceULandmarks: return mat, uni_mat - def cut(self, img : np.ndarray, coverage : float, output_size : int, exclude_moving_parts=False, x_offset : float = 0, y_offset : float = 0) -> Tuple[Affine2DMat, Affine2DUniMat]: + def cut(self, img : np.ndarray, + coverage : float, + output_size : int, + exclude_moving_parts : bool = False, + head_yaw : float = None, + x_offset : float = 0, + y_offset : float = 0) -> Tuple[Affine2DMat, Affine2DUniMat]: """ Cut the face to square of output_size from img using landmarks with given parameters @@ -165,15 +180,17 @@ class FaceULandmarks: exclude_moving_parts(False) exclude moving parts of the face, such as eyebrows and jaw - v_offset - h_offset float uniform h/v offset + head_yaw(None) float fit the head in center using provided yaw radian value. + + x_offset + y_offset float uniform x/y offset returns face_image, uni_mat uniform affine matrix to transform uniform img space to uniform face_image space """ h,w = img.shape[0:2] - mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts, x_offset=x_offset, y_offset=y_offset) + mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts, head_yaw=head_yaw, x_offset=x_offset, y_offset=y_offset) face_image = cv2.warpAffine(img, mat, (output_size, output_size), cv2.INTER_CUBIC ) return face_image, uni_mat diff --git a/xlib/facemeta/__init__.py b/xlib/facemeta/__init__.py index 96fb5e0..3baefcc 100644 --- a/xlib/facemeta/__init__.py +++ b/xlib/facemeta/__init__.py @@ -15,9 +15,9 @@ FaceMark - (mean single face data referencing any image) .FaceURect - a rectangle of the face in source image space .list[FaceULandmarks] - a list of unique types of landmarks of the face in source image space types: - LANDMARKS_2D_5 - LANDMARKS_2D_68 - LANDMARKS_2D_468 + LANDMARKS_5 + LANDMARKS_68 + LANDMARKS_468 .FaceAlign - an aligned face from FaceMark @@ -44,5 +44,5 @@ FaceMark - (mean single face data referencing any image) """ -from .face import FaceMark, FaceAlign, FaceSwap, FaceMask, FaceURect, FaceULandmarks +from .face import FaceMark, FaceAlign, FaceSwap, FaceMask, FaceURect, FaceULandmarks, FacePose from .Faceset import Faceset diff --git a/xlib/facemeta/face.py b/xlib/facemeta/face.py index 719750e..e2599a9 100644 --- a/xlib/facemeta/face.py +++ b/xlib/facemeta/face.py @@ -4,7 +4,7 @@ from xlib import math as lib_math from .FaceULandmarks import FaceULandmarks from .FaceURect import FaceURect - +from .FacePose import FacePose class _part_picklable_expandable: def __getstate__(self): @@ -135,6 +135,16 @@ class _part_face_swap: raise ValueError('face_swap must be an instance of FaceSwap') self._face_swap = face_swap +class _part_face_pose: + def __init__(self): + self._face_pose : Union['FacePose', None] = None + + def get_face_pose(self) -> Union['FacePose', None]: return self._face_pose + def set_face_pose(self, face_pose : 'FacePose'): + if not isinstance(face_pose, FacePose): + raise ValueError('face_pose must be an instance of FacePose') + self._face_pose = face_pose + class FaceMark(_part_picklable_expandable, _part_image_name, @@ -142,6 +152,7 @@ class FaceMark(_part_picklable_expandable, _part_face_urect, _part_face_ulandmarks_list, _part_face_align, + _part_face_pose, ): """ Describes meta data of single face. @@ -154,6 +165,7 @@ class FaceMark(_part_picklable_expandable, _part_face_urect.__init__(self) _part_face_ulandmarks_list.__init__(self) _part_face_align.__init__(self) + _part_face_pose.__init__(self) class FaceAlign(_part_picklable_expandable, _part_image_name, diff --git a/xlib/math/__init__.py b/xlib/math/__init__.py index b299631..8191bb9 100644 --- a/xlib/math/__init__.py +++ b/xlib/math/__init__.py @@ -1,4 +1,4 @@ from .Affine2DMat import Affine2DMat, Affine2DUniMat -from .math_ import (intersect_two_line, polygon_area, segment_length, - segment_to_vector) +from .math_ import (intersect_two_line, polygon_area, rotation_matrix_to_euler, + segment_length, segment_to_vector) from .nms import nms diff --git a/xlib/math/math_.py b/xlib/math/math_.py index 690a800..67b9d83 100644 --- a/xlib/math/math_.py +++ b/xlib/math/math_.py @@ -1,7 +1,23 @@ +import math + import numpy as np import numpy.linalg as npla +def rotation_matrix_to_euler(R : np.ndarray) -> np.ndarray: + sy = math.sqrt(R[0,0] * R[0,0] + R[1,0] * R[1,0]) + singular = sy < 1e-6 + if not singular : + x = math.atan2(R[2,1] , R[2,2]) + y = math.atan2(-R[2,0], sy) + z = math.atan2(R[1,0], R[0,0]) + else : + x = math.atan2(-R[1,2], R[1,1]) + y = math.atan2(-R[2,0], sy) + z = 0 + return np.array([x, y, z]) + + def segment_length(p1 : np.ndarray, p2 : np.ndarray): """ p1 (2,)