mirror of
https://github.com/iperov/DeepFaceLive
synced 2025-08-20 21:43:22 -07:00
added face marker model InsightFace2D106 that provides better face alignment
This commit is contained in:
iperov
parent
4043de5260
commit
0c240b53aa
6 changed files with 121 additions and 20 deletions
|
|
@ -16,8 +16,9 @@ from .BackendBase import (BackendConnection, BackendDB, BackendHost,
|
|||
class MarkerType(IntEnum):
|
||||
OPENCV_LBF = 0
|
||||
GOOGLE_FACEMESH = 1
|
||||
INSIGHT_2D106 = 2
|
||||
|
||||
MarkerTypeNames = ['OpenCV LBF','Google FaceMesh']
|
||||
MarkerTypeNames = ['OpenCV LBF','Google FaceMesh','InsightFace_2D106']
|
||||
|
||||
class FaceMarker(BackendHost):
|
||||
def __init__(self, weak_heap : BackendWeakHeap, reemit_frame_signal : BackendSignal, bc_in : BackendConnection, bc_out : BackendConnection, backend_db : BackendDB = None):
|
||||
|
|
@ -46,6 +47,7 @@ class FaceMarkerWorker(BackendWorker):
|
|||
self.pending_bcd = None
|
||||
self.opencv_lbf = None
|
||||
self.google_facemesh = None
|
||||
self.insightface_2d106 = None
|
||||
self.temporal_lmrks = []
|
||||
|
||||
lib_os.set_timer_resolution(1)
|
||||
|
|
@ -58,7 +60,7 @@ class FaceMarkerWorker(BackendWorker):
|
|||
|
||||
cs.marker_type.enable()
|
||||
cs.marker_type.set_choices(MarkerType, MarkerTypeNames, none_choice_name=None)
|
||||
cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.GOOGLE_FACEMESH)
|
||||
cs.marker_type.select(state.marker_type if state.marker_type is not None else MarkerType.INSIGHT_2D106)
|
||||
|
||||
def on_cs_marker_type(self, idx, marker_type):
|
||||
state, cs = self.get_state(), self.get_control_sheet()
|
||||
|
|
@ -71,6 +73,10 @@ class FaceMarkerWorker(BackendWorker):
|
|||
elif marker_type == MarkerType.GOOGLE_FACEMESH:
|
||||
cs.device.set_choices(onnx_models.FaceMesh.get_available_devices(), none_choice_name='@misc.menu_select')
|
||||
cs.device.select(state.google_facemesh_state.device)
|
||||
elif marker_type == MarkerType.INSIGHT_2D106:
|
||||
cs.device.set_choices(onnx_models.InsightFace2D106.get_available_devices(), none_choice_name='@misc.menu_select')
|
||||
cs.device.select(state.insightface_2d106_state.device)
|
||||
|
||||
else:
|
||||
state.marker_type = marker_type
|
||||
self.save_state()
|
||||
|
|
@ -82,13 +88,16 @@ class FaceMarkerWorker(BackendWorker):
|
|||
|
||||
if device is not None and \
|
||||
( (marker_type == MarkerType.OPENCV_LBF and state.opencv_lbf_state.device == device) or \
|
||||
(marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) ):
|
||||
(marker_type == MarkerType.GOOGLE_FACEMESH and state.google_facemesh_state.device == device) or \
|
||||
(marker_type == MarkerType.INSIGHT_2D106 and state.insightface_2d106_state.device == device) ):
|
||||
marker_state = state.get_marker_state()
|
||||
|
||||
if state.marker_type == MarkerType.OPENCV_LBF:
|
||||
self.opencv_lbf = cv_models.FaceMarkerLBF()
|
||||
elif state.marker_type == MarkerType.GOOGLE_FACEMESH:
|
||||
self.google_facemesh = onnx_models.FaceMesh(state.google_facemesh_state.device)
|
||||
elif state.marker_type == MarkerType.INSIGHT_2D106:
|
||||
self.insightface_2d106 = onnx_models.InsightFace2D106(state.insightface_2d106_state.device)
|
||||
|
||||
cs.marker_coverage.enable()
|
||||
cs.marker_coverage.set_config(lib_csw.Number.Config(min=0.1, max=3.0, step=0.1, decimals=1, allow_instant_update=True))
|
||||
|
|
@ -99,6 +108,8 @@ class FaceMarkerWorker(BackendWorker):
|
|||
marker_coverage = 1.1
|
||||
elif marker_type == MarkerType.GOOGLE_FACEMESH:
|
||||
marker_coverage = 1.4
|
||||
elif marker_type == MarkerType.INSIGHT_2D106:
|
||||
marker_coverage = 1.6
|
||||
cs.marker_coverage.set_number(marker_coverage)
|
||||
|
||||
cs.temporal_smoothing.enable()
|
||||
|
|
@ -110,6 +121,8 @@ class FaceMarkerWorker(BackendWorker):
|
|||
state.opencv_lbf_state.device = device
|
||||
elif marker_type == MarkerType.GOOGLE_FACEMESH:
|
||||
state.google_facemesh_state.device = device
|
||||
elif marker_type == MarkerType.INSIGHT_2D106:
|
||||
state.insightface_2d106_state.device = device
|
||||
self.save_state()
|
||||
self.restart()
|
||||
|
||||
|
|
@ -150,11 +163,13 @@ class FaceMarkerWorker(BackendWorker):
|
|||
|
||||
is_opencv_lbf = marker_type == MarkerType.OPENCV_LBF and self.opencv_lbf is not None
|
||||
is_google_facemesh = marker_type == MarkerType.GOOGLE_FACEMESH and self.google_facemesh is not None
|
||||
is_insightface_2d106 = marker_type == MarkerType.INSIGHT_2D106 and self.insightface_2d106 is not None
|
||||
is_marker_loaded = is_opencv_lbf or is_google_facemesh or is_insightface_2d106
|
||||
|
||||
if marker_type is not None:
|
||||
frame_image = bcd.get_image(bcd.get_frame_image_name())
|
||||
|
||||
if frame_image is not None and (is_opencv_lbf or is_google_facemesh):
|
||||
if frame_image is not None and is_marker_loaded:
|
||||
fsi_list = bcd.get_face_swap_info_list()
|
||||
if marker_state.temporal_smoothing != 1 and \
|
||||
len(self.temporal_lmrks) != len(fsi_list):
|
||||
|
|
@ -164,19 +179,20 @@ class FaceMarkerWorker(BackendWorker):
|
|||
if fsi.face_urect is not None:
|
||||
# Cut the face to feed to the face marker
|
||||
face_image, face_uni_mat = fsi.face_urect.cut(frame_image, marker_state.marker_coverage, 256 if is_opencv_lbf else \
|
||||
192 if is_google_facemesh else 0 )
|
||||
192 if is_google_facemesh else \
|
||||
192 if is_insightface_2d106 else 0 )
|
||||
_,H,W,_ = ImageProcessor(face_image).get_dims()
|
||||
|
||||
if is_opencv_lbf:
|
||||
lmrks = self.opencv_lbf.extract(face_image)[0]
|
||||
elif is_google_facemesh:
|
||||
lmrks = self.google_facemesh.extract(face_image)[0]
|
||||
elif is_insightface_2d106:
|
||||
lmrks = self.insightface_2d106.extract(face_image)[0]
|
||||
|
||||
if marker_state.temporal_smoothing != 1:
|
||||
if not is_frame_reemitted or len(self.temporal_lmrks[face_id]) == 0:
|
||||
self.temporal_lmrks[face_id].append(lmrks)
|
||||
self.temporal_lmrks[face_id] = self.temporal_lmrks[face_id][-marker_state.temporal_smoothing:]
|
||||
|
||||
lmrks = np.mean(self.temporal_lmrks[face_id],0 )
|
||||
|
||||
if is_google_facemesh:
|
||||
|
|
@ -186,9 +202,12 @@ class FaceMarkerWorker(BackendWorker):
|
|||
lmrks /= (W,H)
|
||||
elif is_google_facemesh:
|
||||
lmrks = lmrks[...,0:2] / (W,H)
|
||||
elif is_insightface_2d106:
|
||||
lmrks = lmrks[...,0:2] / (W,H)
|
||||
|
||||
face_ulmrks = FLandmarks2D.create (ELandmarks2D.L68 if is_opencv_lbf else \
|
||||
ELandmarks2D.L468 if is_google_facemesh else None, lmrks)
|
||||
ELandmarks2D.L468 if is_google_facemesh else \
|
||||
ELandmarks2D.L106 if is_insightface_2d106 else None, lmrks)
|
||||
face_ulmrks = face_ulmrks.transform(face_uni_mat, invert=True)
|
||||
fsi.face_ulmrks = face_ulmrks
|
||||
|
||||
|
|
@ -212,12 +231,16 @@ class OpenCVLBFState(BackendWorkerState):
|
|||
class GoogleFaceMeshState(BackendWorkerState):
|
||||
device = None
|
||||
|
||||
class Insight2D106State(BackendWorkerState):
|
||||
device = None
|
||||
|
||||
class WorkerState(BackendWorkerState):
|
||||
def __init__(self):
|
||||
self.marker_type : MarkerType = None
|
||||
self.marker_state = {}
|
||||
self.opencv_lbf_state = OpenCVLBFState()
|
||||
self.google_facemesh_state = GoogleFaceMeshState()
|
||||
self.insightface_2d106_state = Insight2D106State()
|
||||
|
||||
def get_marker_state(self) -> MarkerState:
|
||||
state = self.marker_state.get(self.marker_type, None)
|
||||
|
|
|
|||
BIN
modelhub/onnx/InsightFace2d106/InsightFace2D106.onnx
Normal file
BIN
modelhub/onnx/InsightFace2d106/InsightFace2D106.onnx
Normal file
Binary file not shown.
61
modelhub/onnx/InsightFace2d106/InsightFace2D106.py
Normal file
61
modelhub/onnx/InsightFace2d106/InsightFace2D106.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from xlib.image import ImageProcessor
|
||||
from xlib.onnxruntime import (InferenceSession_with_device, ORTDeviceInfo,
|
||||
get_available_devices_info)
|
||||
|
||||
class InsightFace2D106:
|
||||
"""
|
||||
arguments
|
||||
|
||||
device_info ORTDeviceInfo
|
||||
|
||||
use InsightFace2D106.get_available_devices()
|
||||
to determine a list of avaliable devices accepted by model
|
||||
|
||||
raises
|
||||
Exception
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_available_devices() -> List[ORTDeviceInfo]:
|
||||
return get_available_devices_info()
|
||||
|
||||
def __init__(self, device_info : ORTDeviceInfo):
|
||||
if device_info not in InsightFace2D106.get_available_devices():
|
||||
raise Exception(f'device_info {device_info} is not in available devices for InsightFace2D106')
|
||||
|
||||
path = Path(__file__).parent / 'InsightFace2D106.onnx'
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f'{path} not found')
|
||||
|
||||
self._sess = sess = InferenceSession_with_device(str(path), device_info)
|
||||
self._input_name = sess.get_inputs()[0].name
|
||||
self._input_width = 192
|
||||
self._input_height = 192
|
||||
|
||||
def extract(self, img):
|
||||
"""
|
||||
arguments
|
||||
|
||||
img np.ndarray HW,HWC,NHWC uint8/float32
|
||||
|
||||
returns (N,106,2)
|
||||
"""
|
||||
ip = ImageProcessor(img)
|
||||
N,H,W,_ = ip.get_dims()
|
||||
|
||||
h_scale = H / self._input_height
|
||||
w_scale = W / self._input_width
|
||||
|
||||
feed_img = ip.resize( (self._input_width, self._input_height) ).swap_ch().as_float32().ch(3).get_image('NCHW')
|
||||
|
||||
lmrks = self._sess.run(None, {self._input_name: feed_img})[0]
|
||||
lmrks = lmrks.reshape( (N,106,2))
|
||||
lmrks /= 2.0
|
||||
lmrks += (0.5, 0.5)
|
||||
lmrks *= (w_scale, h_scale)
|
||||
lmrks *= (W, H)
|
||||
|
||||
return lmrks
|
||||
|
|
@ -2,3 +2,4 @@ from .CenterFace.CenterFace import CenterFace
|
|||
from .FaceMesh.FaceMesh import FaceMesh
|
||||
from .S3FD.S3FD import S3FD
|
||||
from .YoloV5Face.YoloV5Face import YoloV5Face
|
||||
from .InsightFace2d106.InsightFace2D106 import InsightFace2D106
|
||||
|
|
@ -3,4 +3,5 @@ from enum import IntEnum
|
|||
class ELandmarks2D(IntEnum):
|
||||
L5 = 0
|
||||
L68 = 1
|
||||
L468 = 2
|
||||
L106 = 2
|
||||
L468 = 3
|
||||
|
|
@ -48,6 +48,9 @@ class FLandmarks2D(IState):
|
|||
elif type == ELandmarks2D.L68:
|
||||
if ulmrks_count != 68:
|
||||
raise ValueError('ulmrks_count must be == 68')
|
||||
elif type == ELandmarks2D.L106:
|
||||
if ulmrks_count != 106:
|
||||
raise ValueError('ulmrks_count must be == 106')
|
||||
elif type == ELandmarks2D.L468:
|
||||
if ulmrks_count != 468:
|
||||
raise ValueError('ulmrks_count must be == 468')
|
||||
|
|
@ -122,8 +125,14 @@ class FLandmarks2D(IState):
|
|||
lmrks = (self._ulmrks * (w,h)).astype(np.float32)
|
||||
|
||||
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
|
||||
if type == ELandmarks2D.L106:
|
||||
type = ELandmarks2D.L68
|
||||
lmrks = lmrks[ lmrks_106_to_68_mean_pairs ]
|
||||
lmrks = lmrks.reshape( (68,2,2)).mean(1)
|
||||
|
||||
if type == ELandmarks2D.L68:
|
||||
mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:49] , lmrks[54:55] ]), uni_landmarks_68)
|
||||
mat = Affine2DMat.umeyama( np.concatenate ([ lmrks[17:36], lmrks[36:37], lmrks[39:40], lmrks[42:43], lmrks[45:46], lmrks[48:49], lmrks[54:55] ]), uni_landmarks_68)
|
||||
|
||||
elif type == ELandmarks2D.L468:
|
||||
src_lmrks = lmrks
|
||||
dst_lmrks = uni_landmarks_468
|
||||
|
|
@ -227,8 +236,14 @@ class FLandmarks2D(IState):
|
|||
cv2.fillConvexPoly( mask, cv2.convexHull(lmrks), color)
|
||||
return mask
|
||||
|
||||
|
||||
|
||||
lmrks_106_to_68_mean_pairs = [1,9, 10,11, 12,13, 14,15, 16,2, 3,4, 5,6, 7,8, 0,0, 24,23, 22,21, 20,19, 18,32, 31,30, 29,28, 27,26,25,17,
|
||||
43,43, 48,44, 49,45, 51,47, 50,46,
|
||||
102,97, 103,98, 104,99, 105,100, 101,101,
|
||||
72,72, 73,73, 74,74, 86,86, 77,78, 78,79, 80,80, 85,84, 84,83,
|
||||
35,35, 41,40, 40,42, 39,39, 37,33, 33,36,
|
||||
89,89, 95,94, 94,96, 93,93, 91,87, 87,90,
|
||||
52,52, 64,64, 63,63, 71,71, 67,67, 68,68, 61,61, 58,58, 59,59, 53,53, 56,56, 55,55, 65,65, 66,66, 62,62, 70,70, 69,69, 57,57, 60,60, 54,54]
|
||||
|
||||
uni_landmarks_68 = np.float32([
|
||||
[ 0.000213256, 0.106454 ], #17
|
||||
[ 0.0752622, 0.038915 ], #18
|
||||
|
|
@ -251,18 +266,18 @@ uni_landmarks_68 = np.float32([
|
|||
[ 0.613373, 0.587326 ], #35
|
||||
|
||||
[ 0.121737, 0.216423 ], #36
|
||||
[ 0.187122, 0.178758 ], #37
|
||||
[ 0.265825, 0.179852 ], #38
|
||||
#[ 0.187122, 0.178758 ], #37
|
||||
#[ 0.265825, 0.179852 ], #38
|
||||
[ 0.334606, 0.231733 ], #39
|
||||
[ 0.260918, 0.245099 ], #40
|
||||
[ 0.182743, 0.244077 ], #41
|
||||
#[ 0.260918, 0.245099 ], #40
|
||||
#[ 0.182743, 0.244077 ], #41
|
||||
|
||||
[ 0.645647, 0.231733 ], #42
|
||||
[ 0.714428, 0.179852 ], #43
|
||||
[ 0.793132, 0.178758 ], #44
|
||||
#[ 0.714428, 0.179852 ], #43
|
||||
#[ 0.793132, 0.178758 ], #44
|
||||
[ 0.858516, 0.216423 ], #45
|
||||
[ 0.79751, 0.244077 ], #46
|
||||
[ 0.719335, 0.245099 ], #47
|
||||
#[ 0.79751, 0.244077 ], #46
|
||||
#[ 0.719335, 0.245099 ], #47
|
||||
|
||||
[ 0.254149, 0.780233 ], #48
|
||||
[ 0.726104, 0.780233 ], #54
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue