FaceAligner : added x/y offset

apps/DeepFaceLive/backend/FaceAligner.py

@@ -37,6 +37,8 @@ class FaceAlignerWorker(BackendWorker):
         cs.face_coverage.call_on_number(self.on_cs_face_coverage)
         cs.resolution.call_on_number(self.on_cs_resolution)
         cs.exclude_moving_parts.call_on_flag(self.on_cs_exclude_moving_parts)
+        cs.x_offset.call_on_number(self.on_cs_x_offset)
+        cs.y_offset.call_on_number(self.on_cs_y_offset)
 
         cs.face_coverage.enable()
         cs.face_coverage.set_config(lib_csw.Number.Config(min=0.1, max=4.0, step=0.1, decimals=1, allow_instant_update=True))
@@ -47,9 +49,17 @@ class FaceAlignerWorker(BackendWorker):
         cs.resolution.set_number(state.resolution if state.resolution is not None else 224)
 
         cs.exclude_moving_parts.enable()
-
         cs.exclude_moving_parts.set_flag(state.exclude_moving_parts if state.exclude_moving_parts is not None else True)
 
+        cs.x_offset.enable()
+        cs.x_offset.set_config(lib_csw.Number.Config(min=-1, max=1, step=0.01, decimals=2, allow_instant_update=True))
+        cs.x_offset.set_number(state.x_offset if state.x_offset is not None else 0)
+
+        cs.y_offset.enable()
+        cs.y_offset.set_config(lib_csw.Number.Config(min=-1, max=1, step=0.01, decimals=2, allow_instant_update=True))
+        cs.y_offset.set_number(state.y_offset if state.y_offset is not None else 0)
+
+
     def on_cs_face_coverage(self, face_coverage):
         state, cs = self.get_state(), self.get_control_sheet()
         cfg = cs.face_coverage.get_config()
@@ -72,6 +82,22 @@ class FaceAlignerWorker(BackendWorker):
         self.save_state()
         self.reemit_frame_signal.send()
 
+    def on_cs_x_offset(self, x_offset):
+        state, cs = self.get_state(), self.get_control_sheet()
+        cfg = cs.x_offset.get_config()
+        x_offset = state.x_offset = float(np.clip(x_offset, cfg.min, cfg.max))
+        cs.x_offset.set_number(x_offset)
+        self.save_state()
+        self.reemit_frame_signal.send()
+
+    def on_cs_y_offset(self, y_offset):
+        state, cs = self.get_state(), self.get_control_sheet()
+        cfg = cs.y_offset.get_config()
+        y_offset = state.y_offset = float(np.clip(y_offset, cfg.min, cfg.max))
+        cs.y_offset.set_number(y_offset)
+        self.save_state()
+        self.reemit_frame_signal.send()
+
     def on_tick(self):
         state, cs = self.get_state(), self.get_control_sheet()
 
@@ -92,7 +118,10 @@ class FaceAlignerWorker(BackendWorker):
                             face_ulmrks = face_mark.get_face_ulandmarks_by_type(FaceULandmarks.Type.LANDMARKS_2D_68)
 
                         if face_ulmrks is not None:
-                            face_image, uni_mat = face_ulmrks.cut(frame_image, state.face_coverage, state.resolution, exclude_moving_parts=state.exclude_moving_parts)
+                            face_image, uni_mat = face_ulmrks.cut(frame_image, state.face_coverage, state.resolution,
+                                                                  exclude_moving_parts=state.exclude_moving_parts,
+                                                                  x_offset=state.x_offset,
+                                                                  y_offset=state.y_offset)
 
                             face_align_image_name = f'{frame_name}_{face_id}_aligned'
 
@@ -126,6 +155,8 @@ class Sheet:
             self.face_coverage = lib_csw.Number.Client()
             self.resolution = lib_csw.Number.Client()
             self.exclude_moving_parts = lib_csw.Flag.Client()
+            self.x_offset = lib_csw.Number.Client()
+            self.y_offset = lib_csw.Number.Client()
 
     class Worker(lib_csw.Sheet.Worker):
         def __init__(self):
@@ -133,8 +164,12 @@ class Sheet:
             self.face_coverage = lib_csw.Number.Host()
             self.resolution = lib_csw.Number.Host()
             self.exclude_moving_parts = lib_csw.Flag.Host()
+            self.x_offset = lib_csw.Number.Host()
+            self.y_offset = lib_csw.Number.Host()
 
 class WorkerState(BackendWorkerState):
     face_coverage : float = None
     resolution    : int = None
     exclude_moving_parts : bool = None
+    x_offset : float = None
+    y_offset : float = None

apps/DeepFaceLive/ui/QFaceAligner.py

@@ -24,6 +24,12 @@ class QFaceAligner(QBackendPanel):
         q_exclude_moving_parts_label = QLabelPopupInfo(label=L('@QFaceAligner.exclude_moving_parts'), popup_info_text=L('@QFaceAligner.help.exclude_moving_parts') )
         q_exclude_moving_parts = QCheckBoxCSWFlag(cs.exclude_moving_parts, reflect_state_widgets=[q_exclude_moving_parts_label])
 
+        q_x_offset_label = QLabelPopupInfo(label=L('@QFaceAligner.x_offset'))
+        q_x_offset       = QSpinBoxCSWNumber(cs.x_offset, reflect_state_widgets=[q_x_offset_label])
+
+        q_y_offset_label = QLabelPopupInfo(label=L('@QFaceAligner.y_offset'))
+        q_y_offset       = QSpinBoxCSWNumber(cs.y_offset, reflect_state_widgets=[q_y_offset_label])
+
         grid_l = lib_qt.QXGridLayout(spacing=5)
         row = 0
         grid_l.addWidget(q_face_coverage_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
@@ -35,6 +41,10 @@ class QFaceAligner(QBackendPanel):
         grid_l.addWidget(q_exclude_moving_parts_label, row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
         grid_l.addWidget(q_exclude_moving_parts, row, 1, alignment=Qt.AlignmentFlag.AlignLeft )
         row += 1
+        grid_l.addLayout( lib_qt.QXVBoxLayout([q_x_offset_label, q_y_offset_label]), row, 0, alignment=Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter  )
+        grid_l.addLayout( lib_qt.QXHBoxLayout([q_x_offset, q_y_offset]), row, 1, alignment=Qt.AlignmentFlag.AlignLeft )
+        row += 1
+
         super().__init__(backend, L('@QFaceAligner.module_title'),
                          layout=lib_qt.QXVBoxLayout([grid_l]))
 

localization/localization.py

@@ -337,6 +337,16 @@ class Localization:
                 'ru-RU' : 'Улучшить стабилизацию исключением лицевых точек\nдвижущихся частей лица, таких как рот и других.',
                 'zh-CN' : '通过排除面部移动部分（例如嘴巴和其他你懂的）的特征点来提高稳定性。'},
 
+    'QFaceAligner.x_offset':{
+                'en-US' : 'X offset',
+                'ru-RU' : 'Смещение по X',
+                'zh-CN' : 'X方向偏移'},
+
+    'QFaceAligner.y_offset':{
+                'en-US' : 'Y offset',
+                'ru-RU' : 'Смещение по Y',
+                'zh-CN' : 'Y方向偏移'},
+
     'QFaceMarker.module_title':{
                 'en-US' : 'Face marker',
                 'ru-RU' : 'Маркер лица',
@@ -681,7 +691,7 @@ class Localization:
                 'en-US' : 'Swapped face',
                 'ru-RU' : 'Заменённое лицо',
                 'zh-CN' : '换后的脸'},
-                
+
     'StreamOutput.SourceType.MERGED_FRAME':{
                 'en-US' : 'Merged frame',
                 'ru-RU' : 'Склеенный кадр',

xlib/facemeta/FaceULandmarks.py

@@ -93,18 +93,17 @@ class FaceULandmarks:
         return FaceULandmarks.create(type=self._type, ulmrks=ulmrks)
 
 
-    def calc_cut(self, w_h, coverage : float, output_size : int, exclude_moving_parts : bool):
+    def calc_cut(self, w_h, coverage : float, output_size : int, exclude_moving_parts : bool, x_offset : float = 0, y_offset : float = 0):
         """
         Calculates affine mat for face cut.
 
+
         returns
              mat,       matrix to transform img space to face_image space
              uni_mat    matrix to transform uniform img space to uniform face_image space
         """
 
         lmrks = (self._ulmrks * w_h ).astype(np.float32)
-
-        ulmrks_count = self.get_count()
         type = self._type
 
         # estimate landmarks transform from global space to local aligned space with bounds [0..1]
@@ -116,7 +115,7 @@ class FaceULandmarks:
             if exclude_moving_parts:
                 src_lmrks = np.delete(src_lmrks, landmarks_468_moving_parts_indexes, 0)
                 dst_lmrks = np.delete(dst_lmrks, landmarks_468_moving_parts_indexes, 0)
-            
+
             mat = Affine2DMat.umeyama(src_lmrks, dst_lmrks)
         else:
             raise NotImplementedError()
@@ -136,10 +135,10 @@ class FaceULandmarks:
         mod = (1.0 / scale)* ( npla.norm(g_p[0]-g_p[2])*( coverage * 0.5) )
 
         # adjust vertical offset to cover more forehead
-        vec = (g_p[0]-g_p[3]).astype(np.float32)
+        h_vec = (g_p[1]-g_p[0]).astype(np.float32)
-        vec_len = npla.norm(vec)
+        v_vec = (g_p[3]-g_p[0]).astype(np.float32)
-        vec /= vec_len
+
-        g_c += vec*vec_len*0.08
+        g_c += h_vec*x_offset + v_vec*(y_offset-0.08)
 
         l_t = np.array( [ g_c - tb_diag_vec*mod,
                           g_c + bt_diag_vec*mod,
@@ -152,7 +151,7 @@ class FaceULandmarks:
         return mat, uni_mat
 
 
-    def cut(self, img : np.ndarray, coverage : float, output_size : int, exclude_moving_parts=False) -> Tuple[Affine2DMat, Affine2DUniMat]:
+    def cut(self, img : np.ndarray, coverage : float, output_size : int, exclude_moving_parts=False, x_offset : float = 0, y_offset : float = 0) -> Tuple[Affine2DMat, Affine2DUniMat]:
         """
         Cut the face to square of output_size from img using landmarks with given parameters
 
@@ -163,15 +162,18 @@ class FaceULandmarks:
             coverage        float
 
             output_size     int
-            
+
             exclude_moving_parts(False)     exclude moving parts of the face, such as eyebrows and jaw
 
+            v_offset
+            h_offset    float   uniform h/v offset
+
         returns face_image,
                 uni_mat         uniform affine matrix to transform uniform img space to uniform face_image space
         """
         h,w = img.shape[0:2]
 
-        mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts)
+        mat, uni_mat = self.calc_cut( (w,h), coverage, output_size, exclude_moving_parts, x_offset=x_offset, y_offset=y_offset)
 
         face_image = cv2.warpAffine(img, mat, (output_size, output_size), cv2.INTER_CUBIC )
         return face_image, uni_mat
@@ -1198,7 +1200,7 @@ uni_landmarks_468 = np.array(
 
 # for i in sel:
 #     selected[i] = True
-    
+
 
 
 # select_holding = False
@@ -1206,27 +1208,27 @@ uni_landmarks_468 = np.array(
 # def onMouse(event, x, y, flags, _):
 #     global select_holding
 #     global unselect_holding
-    
+
-#     if event == cv2.EVENT_LBUTTONDOWN: 
+#     if event == cv2.EVENT_LBUTTONDOWN:
 #         select_holding = True
-#     elif event == cv2.EVENT_LBUTTONUP: 
+#     elif event == cv2.EVENT_LBUTTONUP:
 #         select_holding = False
-#     elif event == cv2.EVENT_RBUTTONDOWN: 
+#     elif event == cv2.EVENT_RBUTTONDOWN:
 #         unselect_holding = True
-#     elif event == cv2.EVENT_RBUTTONUP: 
+#     elif event == cv2.EVENT_RBUTTONUP:
-#         unselect_holding = False    
+#         unselect_holding = False
 #     elif event == cv2.EVENT_MBUTTONDOWN:
-        
+
 #         print([ i for i, x in enumerate(selected) if x == True ])
-        
+
 #     pt_idx = np.argsort( np.linalg.norm(pts - [x,y], axis=1) )[0]
-        
+
 #     if select_holding:
 #         selected[pt_idx] = True
 #     if unselect_holding:
 #         selected[pt_idx] = False
-   
+
-    
+
 # cv2.namedWindow(wnd_name)
 # cv2.setMouseCallback(wnd_name, onMouse)
 
@@ -1236,9 +1238,9 @@ uni_landmarks_468 = np.array(
 #             color = (255,0,0)
 #         else:
 #             color = (255,255,255)
-        
+
 #         cv2.circle(img, (x,y), 1, color, 1 )
-        
+
 #     cv2.imshow(wnd_name,img)
 #     cv2.waitKey(5)
 
@@ -1250,29 +1252,29 @@ uni_landmarks_468 = np.array(
 # def proc1(ev = multiprocessing.Event()):
 #     while True:
 #         ev.wait(timeout=0.001)
-    
+
 # # def proc2(obj : ClassWithEvent):
 # #     print('before wait')
 # #     obj.ev.wait(timeout=0.005)
 # #     print('after wait')
-    
+
 # if __name__ == '__main__':
-    
+
 
 #     multiprocessing.set_start_method('spawn', force=True)
-    
+
 #     ev = multiprocessing.Event()
 #     ev.set()
 
 #     p = multiprocessing.Process(target=proc1, args=(ev,), daemon=True)
-    
+
 #     threading.Thread(target=lambda: p.start(), daemon=True).start()
 #     time.sleep(1.0)
 
 #     p.terminate()
 #     p.join()
 #     del p
-    
+
 #     # p = multiprocessing.Process(target=proc2, args=(obj,), daemon=True)
 #     # threading.Thread(target=lambda: p.start(), daemon=True).start()
 #     # time.sleep(1.0)