diff --git a/apps/DeepFaceLive/backend/BackendBase.py b/apps/DeepFaceLive/backend/BackendBase.py index 1bd8a08..37a00d2 100644 --- a/apps/DeepFaceLive/backend/BackendBase.py +++ b/apps/DeepFaceLive/backend/BackendBase.py @@ -8,7 +8,7 @@ from xlib import time as lib_time from xlib.mp import csw as lib_csw from xlib.python.EventListener import EventListener -from xlib.facemeta import FRect, FLandmarks2D, FPose +from xlib.face import FRect, FLandmarks2D, FPose class BackendFaceSwapInfo: def __init__(self): diff --git a/apps/DeepFaceLive/backend/FaceDetector.py b/apps/DeepFaceLive/backend/FaceDetector.py index 9a3e36c..9dc4378 100644 --- a/apps/DeepFaceLive/backend/FaceDetector.py +++ b/apps/DeepFaceLive/backend/FaceDetector.py @@ -4,7 +4,7 @@ from enum import IntEnum import numpy as np from modelhub import onnx as onnx_models from xlib import os as lib_os -from xlib.facemeta import FRect +from xlib.face import FRect from xlib.image import ImageProcessor from xlib.mp import csw as lib_csw from xlib.python import all_is_not_None diff --git a/apps/DeepFaceLive/backend/FaceMarker.py b/apps/DeepFaceLive/backend/FaceMarker.py index 9395d01..8e7d5db 100644 --- a/apps/DeepFaceLive/backend/FaceMarker.py +++ b/apps/DeepFaceLive/backend/FaceMarker.py @@ -5,7 +5,7 @@ from modelhub import onnx as onnx_models from modelhub import cv as cv_models from xlib import os as lib_os -from xlib.facemeta import ELandmarks2D, FLandmarks2D, FPose +from xlib.face import ELandmarks2D, FLandmarks2D, FPose from xlib.image import ImageProcessor from xlib.mp import csw as lib_csw diff --git a/apps/trainers/FaceAligner/FaceAlignerTrainer.py b/apps/trainers/FaceAligner/FaceAlignerTrainer.py index df1c70e..4c739d0 100644 --- a/apps/trainers/FaceAligner/FaceAlignerTrainer.py +++ b/apps/trainers/FaceAligner/FaceAlignerTrainer.py @@ -1,18 +1,18 @@ -from xlib import facemeta as lib_fm +from xlib import face as lib_face from xlib import time as lib_time class FaceAlignerTrainer: def __init__(self, faceset_path): - #fs = self._fs = lib_fm.Faceset(faceset_path) - fs = lib_fm.Faceset(faceset_path) + #fs = self._fs = lib_face.Faceset(faceset_path) + fs = lib_face.Faceset(faceset_path) #fs.close() with lib_time.timeit(): for x in fs.iter_UImage(): x.get_image() - #fs = lib_fm.Faceset(faceset_path) - #fs.add_UFaceMark( [ lib_fm.UFaceMark() for _ in range(1000)] ) + #fs = lib_face.Faceset(faceset_path) + #fs.add_UFaceMark( [ lib_face.UFaceMark() for _ in range(1000)] ) import code code.interact(local=dict(globals(), **locals())) diff --git a/scripts/dev.py b/scripts/dev.py index 4b18163..f38209e 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -2,7 +2,7 @@ from pathlib import Path import numpy as np from xlib import console as lib_con -from xlib import facemeta as lib_fm +from xlib import face as lib_face from xlib import path as lib_path from xlib.file import SplittedFile from xlib import cv as lib_cv @@ -54,7 +54,7 @@ def extract_facesynthetics_dataset(input_dir): input_path = Path(input_dir) faceset_path = input_path.parent / f'{input_path.name}.dfs' - # fs = lib_fm.Faceset(output_dbpath) + # fs = lib_face.Faceset(output_dbpath) # for ufm in fs.iter_UFaceMark(): # uimg = fs.get_UImage_by_uuid( ufm.get_UImage_uuid() ) # img = uimg.get_image() @@ -64,7 +64,7 @@ def extract_facesynthetics_dataset(input_dir): filepaths = lib_path.get_files_paths(input_path)[:100] #TODO - fs = lib_fm.Faceset(faceset_path) + fs = lib_face.Faceset(faceset_path) fs.clear_db() @@ -91,13 +91,13 @@ def extract_facesynthetics_dataset(input_dir): lmrks = np.array(lmrks[:68], np.float32) / (H,W) - flmrks = lib_fm.FLandmarks2D.create(lib_fm.ELandmarks2D.L68, lmrks) + flmrks = lib_face.FLandmarks2D.create(lib_face.ELandmarks2D.L68, lmrks) - uimg = lib_fm.UImage() + uimg = lib_face.UImage() uimg.assign_image(img) uimg.set_name(image_filepath.stem) - ufm = lib_fm.UFaceMark() + ufm = lib_face.UFaceMark() ufm.set_UImage_uuid(uimg.get_uuid()) ufm.set_FRect(flmrks.get_FRect()) ufm.add_FLandmarks2D(flmrks) @@ -148,10 +148,10 @@ def extract_facesynthetics_dataset(input_dir): # # from xlib import math as lib_math -# fs1 = lib_fm.Faceset(r'D:\\1.dfs') +# fs1 = lib_face.Faceset(r'D:\\1.dfs') # fs1.clear_db() -# uimg = lib_fm.UImage() +# uimg = lib_face.UImage() # uimg.assign_image( np.random.uniform(0, 255, size=(128,128,1) ).astype(np.uint8) ) # fs1.add_UImage(uimg, format='jp2', quality=30) @@ -161,23 +161,23 @@ def extract_facesynthetics_dataset(input_dir): # #fs1.add_UImage(uimg, format='jp2', quality=30) -# up = lib_fm.UPerson() +# up = lib_face.UPerson() # up.set_name('Man') # up.set_age(13) # fs1.add_UPerson(up) -# ufm = lib_fm.UFaceMark() +# ufm = lib_face.UFaceMark() # ufm.set_UPerson_uuid(up.get_uuid()) # ufm.set_UImage_uuid(uimg.get_uuid()) -# ufm.add_mask_info( lib_fm.EMaskType.UNDEFINED, uimg.get_uuid(), lib_math.Affine2DUniMat.identity() ) +# ufm.add_mask_info( lib_face.EMaskType.UNDEFINED, uimg.get_uuid(), lib_math.Affine2DUniMat.identity() ) # fs1.add_UFaceMark(ufm) # fs1.close() -# fs = lib_fm.Faceset(r'D:\\1.dfs') +# fs = lib_face.Faceset(r'D:\\1.dfs') # for uperson in fs.iter_UPerson(): # print(uperson) @@ -193,10 +193,10 @@ def extract_facesynthetics_dataset(input_dir): # code.interact(local=dict(globals(), **locals())) -# uimg2 = lib_fm.UImage() +# uimg2 = lib_face.UImage() # uimg2.assign_image( np.random.uniform(0, 255, size=(128,128,1) ).astype(np.uint8) ) -# ufm = lib_fm.UFaceMark() +# ufm = lib_face.UFaceMark() # ufm.set_UImage_uuid(uimg.get_uuid()) diff --git a/xlib/facemeta/ELandmarks2D.py b/xlib/face/ELandmarks2D.py similarity index 100% rename from xlib/facemeta/ELandmarks2D.py rename to xlib/face/ELandmarks2D.py diff --git a/xlib/facemeta/EMaskType.py b/xlib/face/EMaskType.py similarity index 100% rename from xlib/facemeta/EMaskType.py rename to xlib/face/EMaskType.py diff --git a/xlib/facemeta/FLandmarks2D.py b/xlib/face/FLandmarks2D.py similarity index 99% rename from xlib/facemeta/FLandmarks2D.py rename to xlib/face/FLandmarks2D.py index 8e4f1d0..5dda90a 100644 --- a/xlib/facemeta/FLandmarks2D.py +++ b/xlib/face/FLandmarks2D.py @@ -103,7 +103,7 @@ class FLandmarks2D: return FRect.from_ltrb( (l,t,r,b) ) def calc_cut(self, h_w, coverage : float, output_size : int, - exclude_moving_parts : bool, + exclude_moving_parts : bool = False, head_yaw : float = None, x_offset : float = 0, y_offset : float = 0): """ diff --git a/xlib/facemeta/FMask.py b/xlib/face/FMask.py similarity index 100% rename from xlib/facemeta/FMask.py rename to xlib/face/FMask.py diff --git a/xlib/facemeta/FPose.py b/xlib/face/FPose.py similarity index 100% rename from xlib/facemeta/FPose.py rename to xlib/face/FPose.py diff --git a/xlib/facemeta/FRect.py b/xlib/face/FRect.py similarity index 100% rename from xlib/facemeta/FRect.py rename to xlib/face/FRect.py diff --git a/xlib/face/FaceWarper.py b/xlib/face/FaceWarper.py new file mode 100644 index 0000000..b0b7e95 --- /dev/null +++ b/xlib/face/FaceWarper.py @@ -0,0 +1,143 @@ +from typing import Iterable, Tuple, Union + +import cv2 +import numpy as np + +from ..math import Affine2DMat, Affine2DUniMat + +class FaceWarper: + def __init__(self, + img_to_face_uni_mat : Affine2DUniMat, + + align_rot_deg : Union[None, float, Tuple[float, float] ] = [-15,15], + align_scale : Union[None, float, Tuple[float, float] ] = [-0.15, 0.15], + align_tx : Union[None, float, Tuple[float, float] ] = [-0.05, 0.05], + align_ty : Union[None, float, Tuple[float, float] ] = [-0.05, 0.05], + + rw_grid_cell_count : Union[None, int, Tuple[int, int] ] = [3,7], + rw_grid_rot_deg : Union[None, float, Tuple[float, float] ] = [-180,180], + rw_grid_scale : Union[None, float, Tuple[float, float] ] = [-0.25, 0.25], + rw_grid_tx : Union[None, float, Tuple[float, float] ] = [-0.25, 0.25], + rw_grid_ty : Union[None, float, Tuple[float, float] ] = [-0.25, 0.25], + + rnd_state : np.random.RandomState = None, + ): + """ + Max quality one-pass face augmentation via geometric transformations with provided values. + + img_to_face_uni_mat Affine2DUniMat + + Affine2DUniMat given from FLandmarks2D.calc_cut + it is an uniform affineMat to transform original image to aligned face + + align_* rw_grid_* + + exact augmentation parameters or range for random generation. + + + """ + self._img_to_face_uni_mat = img_to_face_uni_mat + self._face_to_img_uni_mat = img_to_face_uni_mat.invert() + + if rnd_state is None: + rnd_state = np.random + self._rnd_state_state = rnd_state.get_state() + + self._align_rot_deg = rnd_state.uniform(*align_rot_deg) if isinstance(align_rot_deg, Iterable) else align_rot_deg + self._align_scale = rnd_state.uniform(*align_scale) if isinstance(align_scale, Iterable) else align_scale + self._align_tx = rnd_state.uniform(*align_tx) if isinstance(align_tx, Iterable) else align_tx + self._align_ty = rnd_state.uniform(*align_ty) if isinstance(align_ty, Iterable) else align_ty + self._rw_grid_cell_count = rnd_state.randint(*rw_grid_cell_count) if isinstance(rw_grid_cell_count, Iterable) else rw_grid_cell_count + self._rw_grid_rot_deg = rnd_state.uniform(*rw_grid_rot_deg) if isinstance(rw_grid_rot_deg, Iterable) else rw_grid_rot_deg + self._rw_grid_scale = rnd_state.uniform(*rw_grid_scale) if isinstance(rw_grid_scale, Iterable) else rw_grid_scale + self._rw_grid_tx = rnd_state.uniform(*rw_grid_tx) if isinstance(rw_grid_tx, Iterable) else rw_grid_tx + self._rw_grid_ty = rnd_state.uniform(*rw_grid_ty) if isinstance(rw_grid_ty, Iterable) else rw_grid_ty + + self._cached = {} + + def transform(self, img : np.ndarray, out_res : int, random_warp : bool = True) -> np.ndarray: + """ + transform an image. Subsequent calls will output the same result for any img shape and out_res. + + img np.ndarray (HWC) + + out_res int + + random_warp(True) bool + """ + H,W = img.shape[:2] + + key = (H,W,random_warp) + data = self._cached.get(key, None) + if data is None: + rnd_state = np.random.RandomState() + rnd_state.set_state( self._rnd_state_state ) + + image_grid, face_mask = self._cached[key] = self._gen(H,W, random_warp, out_res, rnd_state=rnd_state ) + + new_img = cv2.remap(img, image_grid, None, interpolation=cv2.INTER_LANCZOS4) + new_img *= face_mask + return new_img + + def _gen(self, H, W, random_warp, out_res, rnd_state): + + image_grid = np.stack(np.meshgrid(np.linspace(0., 1.0, H, dtype=np.float32), + np.linspace(0., 1.0, W, dtype=np.float32)), -1) + + if random_warp: + # make a random face_warp_grid in the space of the face + face_warp_grid = FaceWarper._gen_random_warp_uni_grid_diff(out_res, self._rw_grid_cell_count, 0.12, rnd_state) + + # make a randomly transformable mat to transform face_warp_grid from face to image + face_warp_grid_mat = (self._face_to_img_uni_mat * + Affine2DUniMat.from_transformation(0.5, 0.5, self._rw_grid_rot_deg, 1.0+self._rw_grid_scale, self._rw_grid_tx, self._rw_grid_ty) + ) + + # warp face_warp_grid to the space of image and merge with image_grid + image_grid += cv2.warpAffine(face_warp_grid, face_warp_grid_mat.to_exact_mat(out_res,out_res, W, H), (W,H), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + + # scale uniform grid from to image size + image_grid *= (H-1, W-1) + + # apply random transormations for align mat + img_to_face_rnd_mat = (self._face_to_img_uni_mat * Affine2DMat.from_transformation(0.5, 0.5, self._align_rot_deg, 1.0+self._align_scale, self._align_tx, self._align_ty) + ).invert().to_exact_mat(W,H,out_res,out_res) + + # warp image_grid to face space + image_grid = cv2.warpAffine(image_grid, img_to_face_rnd_mat, (out_res,out_res), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE ) + + # One-pass remap from original image to aligned face with all transformations + #new_img = cv2.remap(img, image_grid, None, interpolation=cv2.INTER_LANCZOS4) + + # make mask to refine image-boundary visible in face space + face_mask = cv2.warpAffine( np.ones( (H,W), dtype=np.uint8), img_to_face_rnd_mat, (out_res,out_res), flags=cv2.INTER_NEAREST)[...,None] + + return image_grid, face_mask + + def _gen_random_warp_uni_grid_diff(size: int, cell_count, cell_mod, rnd_state) -> np.ndarray: + """ + generates square uniform random warp + grid of shape (size, size, 2) (x,y) + + cell_count(3) 3+ + + cell_mod (0.12) [ 0 .. 0.24 ] + """ + cell_count = max(3, cell_count) + cell_mod = np.clip(cell_mod, 0, 0.24) + cell_size = 1.0 / (cell_count-1) + + grid = np.zeros( (cell_count,cell_count, 2), dtype=np.float32 ) + + grid[1:-1,1:-1, 0:2] += rnd_state.uniform (low=-cell_size*cell_mod, high=cell_size*cell_mod, size=(cell_count-2, cell_count-2, 2) ) + grid = cv2.resize(grid, (size, size), interpolation=cv2.INTER_CUBIC ).astype(np.float32) + + # Linear dump border cells to zero + border_size = size // cell_count + dumper = np.linspace(0, 1, border_size, dtype=np.float32) + grid[:border_size, :,:] *= dumper[:,None,None] + grid[-border_size:,:,:] *= dumper[::-1,None,None] + grid[:,:border_size ,:] *= dumper[None,:,None] + grid[:,-border_size:,:] *= dumper[None,::-1,None] + + return grid diff --git a/xlib/facemeta/Faceset.py b/xlib/face/Faceset.py similarity index 94% rename from xlib/facemeta/Faceset.py rename to xlib/face/Faceset.py index c690495..62406eb 100644 --- a/xlib/facemeta/Faceset.py +++ b/xlib/face/Faceset.py @@ -34,9 +34,11 @@ class Faceset: cur.execute('BEGIN IMMEDIATE') if not self._is_table_exists('FacesetInfo'): self.recreate(shrink=False, _transaction=False) - cur.execute('COMMIT') - self.shrink() - + cur.execute('COMMIT') + self.shrink() + else: + cur.execute('END') + def __del__(self): self.close() @@ -128,7 +130,15 @@ class Faceset: def get_all_UFaceMark(self) -> List[UFaceMark]: return [ pickle.loads(pickled_bytes) for pickled_bytes, in self._cur.execute('SELECT pickled_bytes FROM UFaceMark').fetchall() ] - + + def get_UFaceMark_by_uuid(self, uuid : bytes) -> Union[UFaceMark, None]: + c = self._cur.execute('SELECT * FROM UFaceMark WHERE uuid=?', [uuid]) + db_row = c.fetchone() + if db_row is None: + return None + + return self._UFaceMark_from_db_row(db_row) + def iter_UFaceMark(self) -> Generator[UFaceMark, None, None]: """ returns Generator of UFaceMark @@ -246,9 +256,12 @@ class Faceset: cur.execute('COMMIT') def get_UImage_count(self) -> int: return self._cur.execute('SELECT COUNT(*) FROM UImage').fetchone()[0] - def get_UImage_by_uuid(self, uuid : bytes) -> Union[UImage, None]: + def get_UImage_by_uuid(self, uuid : Union[bytes, None]) -> Union[UImage, None]: """ """ + if uuid is None: + return None + db_row = self._cur.execute('SELECT * FROM UImage where uuid=?', [uuid]).fetchone() if db_row is None: return None diff --git a/xlib/facemeta/UFaceMark.py b/xlib/face/UFaceMark.py similarity index 100% rename from xlib/facemeta/UFaceMark.py rename to xlib/face/UFaceMark.py diff --git a/xlib/facemeta/UImage.py b/xlib/face/UImage.py similarity index 100% rename from xlib/facemeta/UImage.py rename to xlib/face/UImage.py diff --git a/xlib/facemeta/UPerson.py b/xlib/face/UPerson.py similarity index 100% rename from xlib/facemeta/UPerson.py rename to xlib/face/UPerson.py diff --git a/xlib/facemeta/__init__.py b/xlib/face/__init__.py similarity index 77% rename from xlib/facemeta/__init__.py rename to xlib/face/__init__.py index f73c9a9..38097a9 100644 --- a/xlib/facemeta/__init__.py +++ b/xlib/face/__init__.py @@ -1,10 +1,20 @@ """ -facemeta library +Facelib. -Contains classes for effectively storing, manage, and transfering all face related data. +Contains classes for effectively storing, manage, transfering and processing all face related data. -All classes are picklable and expandable. -All classes have noneable members accessed via get/set. No properties. +##### + +Faceset + .List[UImage] + .List[UFaceMark] + .List[UPerson] + +FaceWarper A class for face augmentation with geometric transformations. + +##### META CLASSES + +F* U* classes are picklable and expandable, have noneable members accessed via get/set. No properties. E-classes are enums. U-classes are unique, have uuid and can be saved in Faceset. @@ -41,21 +51,16 @@ UFaceMark - face mark info referencing UImage from which the face was detected .FPose .List[ (EMaskType, FImage_uuid, uni_mat) ] - list of FMask and AffineMat to transform mask image space to UFaceMark image space - -Faceset - .List[UImage] - .List[UFaceMark] - .List[UPerson] """ - from .ELandmarks2D import ELandmarks2D from .EMaskType import EMaskType from .Faceset import Faceset -from .UImage import UImage +from .FaceWarper import FaceWarper from .FLandmarks2D import FLandmarks2D -from .UFaceMark import UFaceMark from .FMask import FMask -from .UPerson import UPerson from .FPose import FPose from .FRect import FRect +from .UFaceMark import UFaceMark +from .UImage import UImage +from .UPerson import UPerson