added face marker model InsightFace2D106 that provides better face alignment

apps/DeepFaceLive/backend/FaceMarker.py

@@ -16,8 +16,9 @@ from .BackendBase import (BackendConnection, BackendDB, BackendHost,
 class MarkerType(IntEnum):
     OPENCV_LBF = 0
     GOOGLE_FACEMESH = 1
+    INSIGHT_2D106 = 2
 
-MarkerTypeNames = ['OpenCV LBF','Google FaceMesh']
+MarkerTypeNames = ['OpenCV LBF','Google FaceMesh','InsightFace_2D106']
 
 class FaceMarker(BackendHost):
     def __init__(self, weak_heap : BackendWeakHeap, reemit_frame_signal : BackendSignal, bc_in : BackendConnection, bc_out : BackendConnection, backend_db : BackendDB = None):
@@ -46,6 +47,7 @@ class FaceMarkerWorker(BackendWorker):
         self.pending_bcd = None
         self.opencv_lbf = None
         self.google_facemesh = None
+        self.insightface_2d106 = None
         self.temporal_lmrks = []
 
         lib_os.set_timer_resolution(1)
@@ -58,7 +60,7 @@ class FaceMarkerWorker(BackendWorker):
 
         cs.marker_type.enable()
         cs.marker_type.set_choices(MarkerType, MarkerTypeNames, none_choice_name=None)
-        cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.GOOGLE_FACEMESH)
+        cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.INSIGHT_2D106)
 
     def on_cs_marker_type(self, idx, marker_type):
         state, cs = self.get_state(), self.get_control_sheet()
@@ -71,6 +73,10 @@ class FaceMarkerWorker(BackendWorker):
             elif marker_type == MarkerType.GOOGLE_FACEMESH:
                 cs.device.set_choices(onnx_models.FaceMesh.get_available_devices(), none_choice_name='@misc.menu_select')
                 cs.device.select(state.google_facemesh_state.device)
+            elif marker_type == MarkerType.INSIGHT_2D106:
+                cs.device.set_choices(onnx_models.InsightFace2D106.get_available_devices(), none_choice_name='@misc.menu_select')
+                cs.device.select(state.insightface_2d106_state.device)
+
         else:
             state.marker_type = marker_type
             self.save_state()
@@ -82,13 +88,16 @@ class FaceMarkerWorker(BackendWorker):
 
         if device is not None and \
             ( (marker_type == MarkerType.OPENCV_LBF and state.opencv_lbf_state.device == device) or \
-              (marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) ):
+              (marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) or \
+              (marker_type == MarkerType.INSIGHT_2D106 and state.insightface_2d106_state.device == device) ):
             marker_state = state.get_marker_state()
 
             if state.marker_type == MarkerType.OPENCV_LBF:
                 self.opencv_lbf = cv_models.FaceMarkerLBF()
             elif state.marker_type == MarkerType.GOOGLE_FACEMESH:
                 self.google_facemesh = onnx_models.FaceMesh(state.google_facemesh_state.device)
+            elif state.marker_type == MarkerType.INSIGHT_2D106:
+                self.insightface_2d106 = onnx_models.InsightFace2D106(state.insightface_2d106_state.device)
 
             cs.marker_coverage.enable()
             cs.marker_coverage.set_config(lib_csw.Number.Config(min=0.1, max=3.0, step=0.1, decimals=1, allow_instant_update=True))
@@ -99,6 +108,8 @@ class FaceMarkerWorker(BackendWorker):
                     marker_coverage = 1.1
                 elif marker_type == MarkerType.GOOGLE_FACEMESH:
                     marker_coverage = 1.4
+                elif marker_type == MarkerType.INSIGHT_2D106:
+                    marker_coverage = 1.6
             cs.marker_coverage.set_number(marker_coverage)
 
             cs.temporal_smoothing.enable()
@@ -110,6 +121,8 @@ class FaceMarkerWorker(BackendWorker):
                 state.opencv_lbf_state.device = device
             elif marker_type == MarkerType.GOOGLE_FACEMESH:
                 state.google_facemesh_state.device = device
+            elif marker_type == MarkerType.INSIGHT_2D106:
+                state.insightface_2d106_state.device = device
             self.save_state()
             self.restart()
 
@@ -150,11 +163,13 @@ class FaceMarkerWorker(BackendWorker):
 
                 is_opencv_lbf = marker_type == MarkerType.OPENCV_LBF and self.opencv_lbf is not None
                 is_google_facemesh = marker_type == MarkerType.GOOGLE_FACEMESH and self.google_facemesh is not None
+                is_insightface_2d106 = marker_type == MarkerType.INSIGHT_2D106 and self.insightface_2d106 is not None
+                is_marker_loaded = is_opencv_lbf or is_google_facemesh or is_insightface_2d106
 
                 if marker_type is not None:
                     frame_image = bcd.get_image(bcd.get_frame_image_name())
 
-                    if frame_image is not None and (is_opencv_lbf or is_google_facemesh):
+                    if frame_image is not None and is_marker_loaded:
                         fsi_list = bcd.get_face_swap_info_list()
                         if marker_state.temporal_smoothing != 1 and \
                             len(self.temporal_lmrks) != len(fsi_list):
@@ -164,19 +179,20 @@ class FaceMarkerWorker(BackendWorker):
                             if fsi.face_urect is not None:
                                 # Cut the face to feed to the face marker
                                 face_image, face_uni_mat = fsi.face_urect.cut(frame_image, marker_state.marker_coverage, 256 if is_opencv_lbf else \
-                                                                                                                         192 if is_google_facemesh else 0 )
+                                                                                                                         192 if is_google_facemesh else \
+                                                                                                                         192 if is_insightface_2d106 else 0 )
                                 _,H,W,_ = ImageProcessor(face_image).get_dims()
-
                                 if is_opencv_lbf:
                                     lmrks = self.opencv_lbf.extract(face_image)[0]
                                 elif is_google_facemesh:
                                     lmrks = self.google_facemesh.extract(face_image)[0]
+                                elif is_insightface_2d106:
+                                    lmrks = self.insightface_2d106.extract(face_image)[0]
 
                                 if marker_state.temporal_smoothing != 1:
                                     if not is_frame_reemitted or len(self.temporal_lmrks[face_id]) == 0:
                                         self.temporal_lmrks[face_id].append(lmrks)
                                     self.temporal_lmrks[face_id] = self.temporal_lmrks[face_id][-marker_state.temporal_smoothing:]
-
                                     lmrks = np.mean(self.temporal_lmrks[face_id],0 )
 
                                 if is_google_facemesh:
@@ -186,9 +202,12 @@ class FaceMarkerWorker(BackendWorker):
                                     lmrks /= (W,H)
                                 elif is_google_facemesh:
                                     lmrks = lmrks[...,0:2] / (W,H)
+                                elif is_insightface_2d106:
+                                    lmrks = lmrks[...,0:2] / (W,H)
 
                                 face_ulmrks = FLandmarks2D.create (ELandmarks2D.L68 if is_opencv_lbf else \
-                                                                   ELandmarks2D.L468 if is_google_facemesh else None, lmrks)
+                                                                   ELandmarks2D.L468 if is_google_facemesh else \
+                                                                   ELandmarks2D.L106 if is_insightface_2d106 else None, lmrks)
                                 face_ulmrks = face_ulmrks.transform(face_uni_mat, invert=True)
                                 fsi.face_ulmrks = face_ulmrks
 
@@ -212,12 +231,16 @@ class OpenCVLBFState(BackendWorkerState):
 class GoogleFaceMeshState(BackendWorkerState):
     device = None
 
+class Insight2D106State(BackendWorkerState):
+    device = None
+
 class WorkerState(BackendWorkerState):
     def __init__(self):
         self.marker_type : MarkerType = None
         self.marker_state = {}
         self.opencv_lbf_state = OpenCVLBFState()
         self.google_facemesh_state = GoogleFaceMeshState()
+        self.insightface_2d106_state = Insight2D106State()
 
     def get_marker_state(self) -> MarkerState:
         state = self.marker_state.get(self.marker_type, None)

modelhub/onnx/InsightFace2d106/InsightFace2D106.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+from typing import List
+
+from xlib.image import ImageProcessor
+from xlib.onnxruntime import (InferenceSession_with_device, ORTDeviceInfo,
+                              get_available_devices_info)
+
+class InsightFace2D106:
+    """
+    arguments
+
+     device_info    ORTDeviceInfo
+
+        use InsightFace2D106.get_available_devices()
+        to determine a list of avaliable devices accepted by model
+
+    raises
+     Exception
+    """
+
+    @staticmethod
+    def get_available_devices() -> List[ORTDeviceInfo]:
+        return get_available_devices_info()
+
+    def __init__(self, device_info : ORTDeviceInfo):
+        if device_info not in InsightFace2D106.get_available_devices():
+            raise Exception(f'device_info {device_info} is not in available devices for InsightFace2D106')
+
+        path = Path(__file__).parent / 'InsightFace2D106.onnx'
+        if not path.exists():
+            raise FileNotFoundError(f'{path} not found')
+            
+        self._sess = sess = InferenceSession_with_device(str(path), device_info)
+        self._input_name = sess.get_inputs()[0].name
+        self._input_width = 192
+        self._input_height = 192
+
+    def extract(self, img):
+        """
+        arguments
+
+         img    np.ndarray      HW,HWC,NHWC uint8/float32
+
+        returns (N,106,2)
+        """
+        ip = ImageProcessor(img)
+        N,H,W,_ = ip.get_dims()
+
+        h_scale = H / self._input_height
+        w_scale = W / self._input_width
+
+        feed_img = ip.resize( (self._input_width, self._input_height) ).swap_ch().as_float32().ch(3).get_image('NCHW')
+
+        lmrks = self._sess.run(None, {self._input_name: feed_img})[0]
+        lmrks = lmrks.reshape( (N,106,2))
+        lmrks /= 2.0
+        lmrks += (0.5, 0.5)
+        lmrks *= (w_scale, h_scale)
+        lmrks *= (W, H)
+        
+        return lmrks

modelhub/onnx/__init__.py

@@ -2,3 +2,4 @@ from .CenterFace.CenterFace import CenterFace
 from .FaceMesh.FaceMesh import FaceMesh
 from .S3FD.S3FD import S3FD
 from .YoloV5Face.YoloV5Face import YoloV5Face
+from .InsightFace2d106.InsightFace2D106 import InsightFace2D106

xlib/face/ELandmarks2D.py

@@ -3,4 +3,5 @@ from enum import IntEnum
 class ELandmarks2D(IntEnum):
     L5 = 0
     L68 = 1
-    L468 = 2
+    L106 = 2
+    L468 = 3

xlib/face/FLandmarks2D.py

@@ -48,6 +48,9 @@ class FLandmarks2D(IState):
         elif type == ELandmarks2D.L68:
             if ulmrks_count != 68:
                 raise ValueError('ulmrks_count must be == 68')
+        elif type == ELandmarks2D.L106:
+            if ulmrks_count != 106:
+                raise ValueError('ulmrks_count must be == 106')
         elif type == ELandmarks2D.L468:
             if ulmrks_count != 468:
                 raise ValueError('ulmrks_count must be == 468')
@@ -122,8 +125,14 @@ class FLandmarks2D(IState):
         lmrks = (self._ulmrks * (w,h)).astype(np.float32)
 
         # estimate landmarks transform from global space to local aligned space with bounds [0..1]
+        if type == ELandmarks2D.L106:
+            type = ELandmarks2D.L68
+            lmrks = lmrks[ lmrks_106_to_68_mean_pairs ]
+            lmrks = lmrks.reshape( (68,2,2)).mean(1)
+            
         if type == ELandmarks2D.L68:
-            mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:49] , lmrks[54:55] ]), uni_landmarks_68)
+            mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:36], lmrks[36:37], lmrks[39:40], lmrks[42:43], lmrks[45:46], lmrks[48:49], lmrks[54:55] ]), uni_landmarks_68)
+            
         elif type == ELandmarks2D.L468:
             src_lmrks = lmrks
             dst_lmrks = uni_landmarks_468
@@ -227,7 +236,13 @@ class FLandmarks2D(IState):
         cv2.fillConvexPoly( mask, cv2.convexHull(lmrks), color)
         return mask
 
-
+lmrks_106_to_68_mean_pairs = [1,9, 10,11, 12,13, 14,15, 16,2, 3,4, 5,6, 7,8, 0,0, 24,23, 22,21, 20,19, 18,32, 31,30, 29,28, 27,26,25,17,
+                   43,43, 48,44, 49,45, 51,47, 50,46,
+                   102,97, 103,98, 104,99, 105,100, 101,101,
+                   72,72, 73,73, 74,74, 86,86, 77,78, 78,79, 80,80, 85,84, 84,83,
+                   35,35, 41,40, 40,42, 39,39, 37,33, 33,36,
+                   89,89, 95,94, 94,96, 93,93, 91,87, 87,90,
+                52,52, 64,64, 63,63, 71,71, 67,67, 68,68, 61,61, 58,58, 59,59, 53,53, 56,56, 55,55, 65,65, 66,66, 62,62, 70,70, 69,69, 57,57, 60,60, 54,54]
                     
 uni_landmarks_68 = np.float32([
 [ 0.000213256,  0.106454  ], #17
@@ -251,18 +266,18 @@ uni_landmarks_68 = np.float32([
 [ 0.613373,     0.587326  ], #35
 
 [ 0.121737,     0.216423  ], #36
-[ 0.187122,     0.178758  ], #37
+#[ 0.187122,     0.178758  ], #37
-[ 0.265825,     0.179852  ], #38
+#[ 0.265825,     0.179852  ], #38
 [ 0.334606,     0.231733  ], #39
-[ 0.260918,     0.245099  ], #40
+#[ 0.260918,     0.245099  ], #40
-[ 0.182743,     0.244077  ], #41
+#[ 0.182743,     0.244077  ], #41
 
 [ 0.645647,     0.231733  ], #42
-[ 0.714428,     0.179852  ], #43
+#[ 0.714428,     0.179852  ], #43
-[ 0.793132,     0.178758  ], #44
+#[ 0.793132,     0.178758  ], #44
 [ 0.858516,     0.216423  ], #45
-[ 0.79751,      0.244077  ], #46
+#[ 0.79751,      0.244077  ], #46
-[ 0.719335,     0.245099  ], #47
+#[ 0.719335,     0.245099  ], #47
 
 [ 0.254149,     0.780233  ], #48
 [ 0.726104,     0.780233  ], #54