diff --git a/apps/DeepFaceLive/backend/FaceMarker.py b/apps/DeepFaceLive/backend/FaceMarker.py index eb720a2..e5334a1 100644 --- a/apps/DeepFaceLive/backend/FaceMarker.py +++ b/apps/DeepFaceLive/backend/FaceMarker.py @@ -16,8 +16,9 @@ from .BackendBase import (BackendConnection, BackendDB, BackendHost, class MarkerType(IntEnum): OPENCV_LBF = 0 GOOGLE_FACEMESH = 1 + INSIGHT_2D106 = 2 -MarkerTypeNames = ['OpenCV LBF','Google FaceMesh'] +MarkerTypeNames = ['OpenCV LBF','Google FaceMesh','InsightFace_2D106'] class FaceMarker(BackendHost): def __init__(self, weak_heap : BackendWeakHeap, reemit_frame_signal : BackendSignal, bc_in : BackendConnection, bc_out : BackendConnection, backend_db : BackendDB = None): @@ -46,6 +47,7 @@ class FaceMarkerWorker(BackendWorker): self.pending_bcd = None self.opencv_lbf = None self.google_facemesh = None + self.insightface_2d106 = None self.temporal_lmrks = [] lib_os.set_timer_resolution(1) @@ -58,7 +60,7 @@ class FaceMarkerWorker(BackendWorker): cs.marker_type.enable() cs.marker_type.set_choices(MarkerType, MarkerTypeNames, none_choice_name=None) - cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.GOOGLE_FACEMESH) + cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.INSIGHT_2D106) def on_cs_marker_type(self, idx, marker_type): state, cs = self.get_state(), self.get_control_sheet() @@ -71,6 +73,10 @@ class FaceMarkerWorker(BackendWorker): elif marker_type == MarkerType.GOOGLE_FACEMESH: cs.device.set_choices(onnx_models.FaceMesh.get_available_devices(), none_choice_name='@misc.menu_select') cs.device.select(state.google_facemesh_state.device) + elif marker_type == MarkerType.INSIGHT_2D106: + cs.device.set_choices(onnx_models.InsightFace2D106.get_available_devices(), none_choice_name='@misc.menu_select') + cs.device.select(state.insightface_2d106_state.device) + else: state.marker_type = marker_type self.save_state() @@ -82,13 +88,16 @@ class FaceMarkerWorker(BackendWorker): if device is not None and \ ( (marker_type == MarkerType.OPENCV_LBF and state.opencv_lbf_state.device == device) or \ - (marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) ): + (marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) or \ + (marker_type == MarkerType.INSIGHT_2D106 and state.insightface_2d106_state.device == device) ): marker_state = state.get_marker_state() if state.marker_type == MarkerType.OPENCV_LBF: self.opencv_lbf = cv_models.FaceMarkerLBF() elif state.marker_type == MarkerType.GOOGLE_FACEMESH: self.google_facemesh = onnx_models.FaceMesh(state.google_facemesh_state.device) + elif state.marker_type == MarkerType.INSIGHT_2D106: + self.insightface_2d106 = onnx_models.InsightFace2D106(state.insightface_2d106_state.device) cs.marker_coverage.enable() cs.marker_coverage.set_config(lib_csw.Number.Config(min=0.1, max=3.0, step=0.1, decimals=1, allow_instant_update=True)) @@ -99,6 +108,8 @@ class FaceMarkerWorker(BackendWorker): marker_coverage = 1.1 elif marker_type == MarkerType.GOOGLE_FACEMESH: marker_coverage = 1.4 + elif marker_type == MarkerType.INSIGHT_2D106: + marker_coverage = 1.6 cs.marker_coverage.set_number(marker_coverage) cs.temporal_smoothing.enable() @@ -110,6 +121,8 @@ class FaceMarkerWorker(BackendWorker): state.opencv_lbf_state.device = device elif marker_type == MarkerType.GOOGLE_FACEMESH: state.google_facemesh_state.device = device + elif marker_type == MarkerType.INSIGHT_2D106: + state.insightface_2d106_state.device = device self.save_state() self.restart() @@ -150,11 +163,13 @@ class FaceMarkerWorker(BackendWorker): is_opencv_lbf = marker_type == MarkerType.OPENCV_LBF and self.opencv_lbf is not None is_google_facemesh = marker_type == MarkerType.GOOGLE_FACEMESH and self.google_facemesh is not None + is_insightface_2d106 = marker_type == MarkerType.INSIGHT_2D106 and self.insightface_2d106 is not None + is_marker_loaded = is_opencv_lbf or is_google_facemesh or is_insightface_2d106 if marker_type is not None: frame_image = bcd.get_image(bcd.get_frame_image_name()) - if frame_image is not None and (is_opencv_lbf or is_google_facemesh): + if frame_image is not None and is_marker_loaded: fsi_list = bcd.get_face_swap_info_list() if marker_state.temporal_smoothing != 1 and \ len(self.temporal_lmrks) != len(fsi_list): @@ -164,19 +179,20 @@ class FaceMarkerWorker(BackendWorker): if fsi.face_urect is not None: # Cut the face to feed to the face marker face_image, face_uni_mat = fsi.face_urect.cut(frame_image, marker_state.marker_coverage, 256 if is_opencv_lbf else \ - 192 if is_google_facemesh else 0 ) + 192 if is_google_facemesh else \ + 192 if is_insightface_2d106 else 0 ) _,H,W,_ = ImageProcessor(face_image).get_dims() - if is_opencv_lbf: lmrks = self.opencv_lbf.extract(face_image)[0] elif is_google_facemesh: lmrks = self.google_facemesh.extract(face_image)[0] + elif is_insightface_2d106: + lmrks = self.insightface_2d106.extract(face_image)[0] if marker_state.temporal_smoothing != 1: if not is_frame_reemitted or len(self.temporal_lmrks[face_id]) == 0: self.temporal_lmrks[face_id].append(lmrks) self.temporal_lmrks[face_id] = self.temporal_lmrks[face_id][-marker_state.temporal_smoothing:] - lmrks = np.mean(self.temporal_lmrks[face_id],0 ) if is_google_facemesh: @@ -186,9 +202,12 @@ class FaceMarkerWorker(BackendWorker): lmrks /= (W,H) elif is_google_facemesh: lmrks = lmrks[...,0:2] / (W,H) + elif is_insightface_2d106: + lmrks = lmrks[...,0:2] / (W,H) face_ulmrks = FLandmarks2D.create (ELandmarks2D.L68 if is_opencv_lbf else \ - ELandmarks2D.L468 if is_google_facemesh else None, lmrks) + ELandmarks2D.L468 if is_google_facemesh else \ + ELandmarks2D.L106 if is_insightface_2d106 else None, lmrks) face_ulmrks = face_ulmrks.transform(face_uni_mat, invert=True) fsi.face_ulmrks = face_ulmrks @@ -212,12 +231,16 @@ class OpenCVLBFState(BackendWorkerState): class GoogleFaceMeshState(BackendWorkerState): device = None +class Insight2D106State(BackendWorkerState): + device = None + class WorkerState(BackendWorkerState): def __init__(self): self.marker_type : MarkerType = None self.marker_state = {} self.opencv_lbf_state = OpenCVLBFState() self.google_facemesh_state = GoogleFaceMeshState() + self.insightface_2d106_state = Insight2D106State() def get_marker_state(self) -> MarkerState: state = self.marker_state.get(self.marker_type, None) diff --git a/modelhub/onnx/InsightFace2d106/InsightFace2D106.onnx b/modelhub/onnx/InsightFace2d106/InsightFace2D106.onnx new file mode 100644 index 0000000..35725cb Binary files /dev/null and b/modelhub/onnx/InsightFace2d106/InsightFace2D106.onnx differ diff --git a/modelhub/onnx/InsightFace2d106/InsightFace2D106.py b/modelhub/onnx/InsightFace2d106/InsightFace2D106.py new file mode 100644 index 0000000..0a84683 --- /dev/null +++ b/modelhub/onnx/InsightFace2d106/InsightFace2D106.py @@ -0,0 +1,61 @@ +from pathlib import Path +from typing import List + +from xlib.image import ImageProcessor +from xlib.onnxruntime import (InferenceSession_with_device, ORTDeviceInfo, + get_available_devices_info) + +class InsightFace2D106: + """ + arguments + + device_info ORTDeviceInfo + + use InsightFace2D106.get_available_devices() + to determine a list of avaliable devices accepted by model + + raises + Exception + """ + + @staticmethod + def get_available_devices() -> List[ORTDeviceInfo]: + return get_available_devices_info() + + def __init__(self, device_info : ORTDeviceInfo): + if device_info not in InsightFace2D106.get_available_devices(): + raise Exception(f'device_info {device_info} is not in available devices for InsightFace2D106') + + path = Path(__file__).parent / 'InsightFace2D106.onnx' + if not path.exists(): + raise FileNotFoundError(f'{path} not found') + + self._sess = sess = InferenceSession_with_device(str(path), device_info) + self._input_name = sess.get_inputs()[0].name + self._input_width = 192 + self._input_height = 192 + + def extract(self, img): + """ + arguments + + img np.ndarray HW,HWC,NHWC uint8/float32 + + returns (N,106,2) + """ + ip = ImageProcessor(img) + N,H,W,_ = ip.get_dims() + + h_scale = H / self._input_height + w_scale = W / self._input_width + + feed_img = ip.resize( (self._input_width, self._input_height) ).swap_ch().as_float32().ch(3).get_image('NCHW') + + lmrks = self._sess.run(None, {self._input_name: feed_img})[0] + lmrks = lmrks.reshape( (N,106,2)) + lmrks /= 2.0 + lmrks += (0.5, 0.5) + lmrks *= (w_scale, h_scale) + lmrks *= (W, H) + + return lmrks diff --git a/modelhub/onnx/__init__.py b/modelhub/onnx/__init__.py index b54745b..c70ca7e 100644 --- a/modelhub/onnx/__init__.py +++ b/modelhub/onnx/__init__.py @@ -2,3 +2,4 @@ from .CenterFace.CenterFace import CenterFace from .FaceMesh.FaceMesh import FaceMesh from .S3FD.S3FD import S3FD from .YoloV5Face.YoloV5Face import YoloV5Face +from .InsightFace2d106.InsightFace2D106 import InsightFace2D106 \ No newline at end of file diff --git a/xlib/face/ELandmarks2D.py b/xlib/face/ELandmarks2D.py index 3f749ed..5200b60 100644 --- a/xlib/face/ELandmarks2D.py +++ b/xlib/face/ELandmarks2D.py @@ -3,4 +3,5 @@ from enum import IntEnum class ELandmarks2D(IntEnum): L5 = 0 L68 = 1 - L468 = 2 \ No newline at end of file + L106 = 2 + L468 = 3 \ No newline at end of file diff --git a/xlib/face/FLandmarks2D.py b/xlib/face/FLandmarks2D.py index f159f6a..67a36da 100644 --- a/xlib/face/FLandmarks2D.py +++ b/xlib/face/FLandmarks2D.py @@ -48,6 +48,9 @@ class FLandmarks2D(IState): elif type == ELandmarks2D.L68: if ulmrks_count != 68: raise ValueError('ulmrks_count must be == 68') + elif type == ELandmarks2D.L106: + if ulmrks_count != 106: + raise ValueError('ulmrks_count must be == 106') elif type == ELandmarks2D.L468: if ulmrks_count != 468: raise ValueError('ulmrks_count must be == 468') @@ -122,8 +125,14 @@ class FLandmarks2D(IState): lmrks = (self._ulmrks * (w,h)).astype(np.float32) # estimate landmarks transform from global space to local aligned space with bounds [0..1] + if type == ELandmarks2D.L106: + type = ELandmarks2D.L68 + lmrks = lmrks[ lmrks_106_to_68_mean_pairs ] + lmrks = lmrks.reshape( (68,2,2)).mean(1) + if type == ELandmarks2D.L68: - mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:49] , lmrks[54:55] ]), uni_landmarks_68) + mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:36], lmrks[36:37], lmrks[39:40], lmrks[42:43], lmrks[45:46], lmrks[48:49], lmrks[54:55] ]), uni_landmarks_68) + elif type == ELandmarks2D.L468: src_lmrks = lmrks dst_lmrks = uni_landmarks_468 @@ -227,8 +236,14 @@ class FLandmarks2D(IState): cv2.fillConvexPoly( mask, cv2.convexHull(lmrks), color) return mask - - +lmrks_106_to_68_mean_pairs = [1,9, 10,11, 12,13, 14,15, 16,2, 3,4, 5,6, 7,8, 0,0, 24,23, 22,21, 20,19, 18,32, 31,30, 29,28, 27,26,25,17, + 43,43, 48,44, 49,45, 51,47, 50,46, + 102,97, 103,98, 104,99, 105,100, 101,101, + 72,72, 73,73, 74,74, 86,86, 77,78, 78,79, 80,80, 85,84, 84,83, + 35,35, 41,40, 40,42, 39,39, 37,33, 33,36, + 89,89, 95,94, 94,96, 93,93, 91,87, 87,90, + 52,52, 64,64, 63,63, 71,71, 67,67, 68,68, 61,61, 58,58, 59,59, 53,53, 56,56, 55,55, 65,65, 66,66, 62,62, 70,70, 69,69, 57,57, 60,60, 54,54] + uni_landmarks_68 = np.float32([ [ 0.000213256, 0.106454 ], #17 [ 0.0752622, 0.038915 ], #18 @@ -251,18 +266,18 @@ uni_landmarks_68 = np.float32([ [ 0.613373, 0.587326 ], #35 [ 0.121737, 0.216423 ], #36 -[ 0.187122, 0.178758 ], #37 -[ 0.265825, 0.179852 ], #38 +#[ 0.187122, 0.178758 ], #37 +#[ 0.265825, 0.179852 ], #38 [ 0.334606, 0.231733 ], #39 -[ 0.260918, 0.245099 ], #40 -[ 0.182743, 0.244077 ], #41 +#[ 0.260918, 0.245099 ], #40 +#[ 0.182743, 0.244077 ], #41 [ 0.645647, 0.231733 ], #42 -[ 0.714428, 0.179852 ], #43 -[ 0.793132, 0.178758 ], #44 +#[ 0.714428, 0.179852 ], #43 +#[ 0.793132, 0.178758 ], #44 [ 0.858516, 0.216423 ], #45 -[ 0.79751, 0.244077 ], #46 -[ 0.719335, 0.245099 ], #47 +#[ 0.79751, 0.244077 ], #46 +#[ 0.719335, 0.245099 ], #47 [ 0.254149, 0.780233 ], #48 [ 0.726104, 0.780233 ], #54