github/DeepFaceLab
1
0
Fork 0
mirror of https://github.com/iperov/DeepFaceLab.git synced 2025-07-06 13:02:15 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

removing default yaw_value from DFLIMG files,

Browse source 
added better pitch/yaw estimator from 68 landmarks,
improving face yaw accuracy for sorting and trainers,
added sort by face-pitch
This commit is contained in:
iperov 2019-02-12 21:31:37 +04:00
parent 535041f7bb
commit 06fe1314d8
13 changed files with 182 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

4
doc/doc_sort_tool.md
View file

@ -8,6 +8,10 @@
`hist-blur` sort by blur in groups of similar content
`face-pitch` sort by face pitch direction
`face-yaw` sort by face yaw direction
`brightness`
`hue`

93
facelib/LandmarksProcessor.py
View file

@ -2,6 +2,7 @@ import colorsys
import cv2
import numpy as np
from enum import IntEnum
import mathlib
from mathlib.umeyama import umeyama
from utils import image_utils
from facelib import FaceType
@ -36,6 +37,77 @@ landmarks_68_pt = { "mouth": (48,68),
"nose": (27, 36), # missed one point
"jaw": (0, 17) }
landmarks_68_3D = np.array( [
[-73.393523 , -29.801432 , 47.667532 ],
[-72.775014 , -10.949766 , 45.909403 ],
[-70.533638 , 7.929818 , 44.842580 ],
[-66.850058 , 26.074280 , 43.141114 ],
[-59.790187 , 42.564390 , 38.635298 ],
[-48.368973 , 56.481080 , 30.750622 ],
[-34.121101 , 67.246992 , 18.456453 ],
[-17.875411 , 75.056892 , 3.609035 ],
[0.098749 , 77.061286 , -0.881698 ],
[17.477031 , 74.758448 , 5.181201 ],
[32.648966 , 66.929021 , 19.176563 ],
[46.372358 , 56.311389 , 30.770570 ],
[57.343480 , 42.419126 , 37.628629 ],
[64.388482 , 25.455880 , 40.886309 ],
[68.212038 , 6.990805 , 42.281449 ],
[70.486405 , -11.666193 , 44.142567 ],
[71.375822 , -30.365191 , 47.140426 ],
[-61.119406 , -49.361602 , 14.254422 ],
[-51.287588 , -58.769795 , 7.268147 ],
[-37.804800 , -61.996155 , 0.442051 ],
[-24.022754 , -61.033399 , -6.606501 ],
[-11.635713 , -56.686759 , -11.967398 ],
[12.056636 , -57.391033 , -12.051204 ],
[25.106256 , -61.902186 , -7.315098 ],
[38.338588 , -62.777713 , -1.022953 ],
[51.191007 , -59.302347 , 5.349435 ],
[60.053851 , -50.190255 , 11.615746 ],
[0.653940 , -42.193790 , -13.380835 ],
[0.804809 , -30.993721 , -21.150853 ],
[0.992204 , -19.944596 , -29.284036 ],
[1.226783 , -8.414541 , -36.948060 ],
[-14.772472 , 2.598255 , -20.132003 ],
[-7.180239 , 4.751589 , -23.536684 ],
[0.555920 , 6.562900 , -25.944448 ],
[8.272499 , 4.661005 , -23.695741 ],
[15.214351 , 2.643046 , -20.858157 ],
[-46.047290 , -37.471411 , 7.037989 ],
[-37.674688 , -42.730510 , 3.021217 ],
[-27.883856 , -42.711517 , 1.353629 ],
[-19.648268 , -36.754742 , -0.111088 ],
[-28.272965 , -35.134493 , -0.147273 ],
[-38.082418 , -34.919043 , 1.476612 ],
[19.265868 , -37.032306 , -0.665746 ],
[27.894191 , -43.342445 , 0.247660 ],
[37.437529 , -43.110822 , 1.696435 ],
[45.170805 , -38.086515 , 4.894163 ],
[38.196454 , -35.532024 , 0.282961 ],
[28.764989 , -35.484289 , -1.172675 ],
[-28.916267 , 28.612716 , -2.240310 ],
[-17.533194 , 22.172187 , -15.934335 ],
[-6.684590 , 19.029051 , -22.611355 ],
[0.381001 , 20.721118 , -23.748437 ],
[8.375443 , 19.035460 , -22.721995 ],
[18.876618 , 22.394109 , -15.610679 ],
[28.794412 , 28.079924 , -3.217393 ],
[19.057574 , 36.298248 , -14.987997 ],
[8.956375 , 39.634575 , -22.554245 ],
[0.381549 , 40.395647 , -23.591626 ],
[-7.428895 , 39.836405 , -22.406106 ],
[-18.160634 , 36.677899 , -15.121907 ],
[-24.377490 , 28.677771 , -4.785684 ],
[-6.897633 , 25.475976 , -20.893742 ],
[0.340663 , 26.014269 , -22.220479 ],
[8.444722 , 25.326198 , -21.025520 ],
[24.474473 , 28.323008 , -5.712776 ],
[8.449166 , 30.596216 , -20.671489 ],
[0.205322 , 31.408738 , -21.903670 ],
[-7.198266 , 30.844876 , -20.328022 ] ], dtype=np.float32)
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
@ -214,6 +286,7 @@ def calc_face_pitch(landmarks):
t = ( (landmarks[6][1]-landmarks[8][1]) + (landmarks[10][1]-landmarks[8][1]) ) / 2.0
b = landmarks[8][1]
return float(b-t)
def calc_face_yaw(landmarks):
if not isinstance(landmarks, np.ndarray):
landmarks = np.array (landmarks)
@ -221,3 +294,23 @@ def calc_face_yaw(landmarks):
r = ( (landmarks[16][0]-landmarks[27][0]) + (landmarks[15][0]-landmarks[28][0]) + (landmarks[14][0]-landmarks[29][0]) ) / 3.0
return float(r-l)
#returns pitch,yaw [-1...+1]
def estimate_pitch_yaw(aligned_256px_landmarks):
shape = (256,256)
focal_length = shape[1]
camera_center = (shape[1] / 2, shape[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, camera_center[0]],
[0, focal_length, camera_center[1]],
[0, 0, 1]], dtype=np.float32)
(_, rotation_vector, translation_vector) = cv2.solvePnP(
landmarks_68_3D,
aligned_256px_landmarks.astype(np.float32),
camera_matrix,
np.zeros((4, 1)) )
pitch, yaw, _ = mathlib.rotationMatrixToEulerAngles( cv2.Rodrigues(rotation_vector)[0] )
pitch = np.clip ( pitch*1.25, -1.0, 1.0 )
yaw = np.clip ( yaw*1.25, -1.0, 1.0 )
return pitch, yaw

2
main.py
View file

@ -61,7 +61,7 @@ if __name__ == "__main__":
sort_parser = subparsers.add_parser( "sort", help="Sort faces in a directory.")
sort_parser.add_argument('--input-dir', required=True, action=fixPathAction, dest="input_dir", help="Input directory. A directory containing the files you wish to process.")
sort_parser.add_argument('--by', required=True, dest="sort_by_method", choices=("blur", "face", "face-dissim", "face-yaw", "hist", "hist-dissim", "brightness", "hue", "black", "origname", "final", "test"), help="Method of sorting. 'origname' sort by original filename to recover original sequence." )
sort_parser.add_argument('--by', required=True, dest="sort_by_method", choices=("blur", "face", "face-dissim", "face-yaw", "face-pitch", "hist", "hist-dissim", "brightness", "hue", "black", "origname", "final", "test"), help="Method of sorting. 'origname' sort by original filename to recover original sequence." )
sort_parser.set_defaults (func=process_sort)
def process_util(arguments):

2
mainscripts/Extractor.py
View file

@ -334,8 +334,6 @@ class ExtractSubprocessor(SubprocessorBase):
DFLJPG.embed_data(output_file, face_type = FaceType.toString(self.face_type),
landmarks = face_image_landmarks.tolist(),
yaw_value = LandmarksProcessor.calc_face_yaw (face_image_landmarks),
pitch_value = LandmarksProcessor.calc_face_pitch (face_image_landmarks),
source_filename = filename_path.name,
source_rect= rect,
source_landmarks = image_landmarks.tolist()

34
mainscripts/Sorter.py
View file

@ -237,7 +237,32 @@ def sort_by_face_yaw(input_path):
print ("%s is not a dfl image file" % (filepath.name) )
continue
img_list.append( [str(filepath), np.array( dflimg.get_yaw_value() ) ] )
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() )
img_list.append( [str(filepath), yaw ] )
print ("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(1), reverse=True)
return img_list
def sort_by_face_pitch(input_path):
print ("Sorting by face pitch...")
img_list = []
for filepath in tqdm( Path_utils.get_image_paths(input_path), desc="Loading", ascii=True):
filepath = Path(filepath)
if filepath.suffix == '.png':
dflimg = DFLPNG.load( str(filepath), print_on_no_embedded_data=True )
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load ( str(filepath), print_on_no_embedded_data=True )
else:
print ("%s is not a dfl image file" % (filepath.name) )
continue
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() )
img_list.append( [str(filepath), pitch ] )
print ("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(1), reverse=True)
@ -543,12 +568,14 @@ class FinalLoaderSubprocessor(SubprocessorBase):
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
gray_masked = ( gray * LandmarksProcessor.get_image_hull_mask (bgr.shape, dflimg.get_landmarks() )[:,:,0] ).astype(np.uint8)
sharpness = estimate_sharpness(gray_masked)
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() )
hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
except Exception as e:
print (e)
return [ 1, [str(filepath)] ]
return [ 0, [str(filepath), sharpness, hist, dflimg.get_yaw_value() ] ]
return [ 0, [str(filepath), sharpness, hist, yaw ] ]
#override
@ -577,7 +604,7 @@ def sort_final(input_path):
grads = 128
imgs_per_grad = 15
grads_space = np.linspace (-255,255,grads)
grads_space = np.linspace (-1.0,1.0,grads)
yaws_sample_list = [None]*grads
for g in tqdm ( range(grads), desc="Sort by yaw", ascii=True ):
@ -732,6 +759,7 @@ def main (input_path, sort_by_method):
elif sort_by_method == 'face': img_list = sort_by_face (input_path)
elif sort_by_method == 'face-dissim': img_list = sort_by_face_dissim (input_path)
elif sort_by_method == 'face-yaw': img_list = sort_by_face_yaw (input_path)
elif sort_by_method == 'face-pitch': img_list = sort_by_face_pitch (input_path)
elif sort_by_method == 'hist': img_list = sort_by_hist (input_path)
elif sort_by_method == 'hist-dissim': img_list = sort_by_hist_dissim (input_path)
elif sort_by_method == 'brightness': img_list = sort_by_brightness (input_path)

2
mainscripts/Util.py
View file

@ -34,8 +34,6 @@ def convert_png_to_jpg_file (filepath):
DFLJPG.embed_data( new_filepath,
face_type=dfl_dict.get('face_type', None),
landmarks=dfl_dict.get('landmarks', None),
yaw_value=dfl_dict.get('yaw_value', None),
pitch_value=dfl_dict.get('pitch_value', None),
source_filename=dfl_dict.get('source_filename', None),
source_rect=dfl_dict.get('source_rect', None),
source_landmarks=dfl_dict.get('source_landmarks', None) )

15
mathlib/__init__.py
View file

@ -1,3 +1,5 @@
import numpy as np
import math
from .umeyama import umeyama
def get_power_of_two(x):
@ -5,3 +7,16 @@ def get_power_of_two(x):
while (1 << i) < x:
i += 1
return i
def rotationMatrixToEulerAngles(R) :
sy = math.sqrt(R[0,0] * R[0,0] + R[1,0] * R[1,0])
singular = sy < 1e-6
if not singular :
x = math.atan2(R[2,1] , R[2,2])
y = math.atan2(-R[2,0], sy)
z = math.atan2(R[1,0], R[0,0])
else :
x = math.atan2(-R[1,2], R[1,1])
y = math.atan2(-R[2,0], sy)
z = 0
return np.array([x, y, z])

1
nnlib/nnlib.py
View file

@ -85,6 +85,7 @@ tanh = keras.layers.Activation('tanh')
sigmoid = keras.layers.Activation('sigmoid')
Dropout = keras.layers.Dropout
Lambda = keras.layers.Lambda
Add = keras.layers.Add
Concatenate = keras.layers.Concatenate

6
samples/Sample.py
View file

@ -16,23 +16,25 @@ class SampleType(IntEnum):
QTY = 5
class Sample(object):
def __init__(self, sample_type=None, filename=None, face_type=None, shape=None, landmarks=None, yaw=None, mirror=None, close_target_list=None):
def __init__(self, sample_type=None, filename=None, face_type=None, shape=None, landmarks=None, pitch=None, yaw=None, mirror=None, close_target_list=None):
self.sample_type = sample_type if sample_type is not None else SampleType.IMAGE
self.filename = filename
self.face_type = face_type
self.shape = shape
self.landmarks = np.array(landmarks) if landmarks is not None else None
self.pitch = pitch
self.yaw = yaw
self.mirror = mirror
self.close_target_list = close_target_list
def copy_and_set(self, sample_type=None, filename=None, face_type=None, shape=None, landmarks=None, yaw=None, mirror=None, close_target_list=None):
def copy_and_set(self, sample_type=None, filename=None, face_type=None, shape=None, landmarks=None, pitch=None, yaw=None, mirror=None, close_target_list=None):
return Sample(
sample_type=sample_type if sample_type is not None else self.sample_type,
filename=filename if filename is not None else self.filename,
face_type=face_type if face_type is not None else self.face_type,
shape=shape if shape is not None else self.shape,
landmarks=landmarks if landmarks is not None else self.landmarks.copy(),
pitch=pitch if pitch is not None else self.pitch,
yaw=yaw if yaw is not None else self.yaw,
mirror=mirror if mirror is not None else self.mirror,
close_target_list=close_target_list if close_target_list is not None else self.close_target_list)

23
samples/SampleGeneratorFace.py
View file

@ -4,6 +4,7 @@ import random
import cv2
import multiprocessing
from utils import iter_utils
from facelib import LandmarksProcessor
from samples import SampleType
from samples import SampleProcessor
@ -18,11 +19,13 @@ output_sample_types = [
]
'''
class SampleGeneratorFace(SampleGeneratorBase):
def __init__ (self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, with_close_to_self=False, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, generators_count=2, **kwargs):
def __init__ (self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, with_close_to_self=False, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, add_pitch=False, add_yaw=False, generators_count=2, **kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
self.add_pitch = add_pitch
self.add_yaw = add_yaw
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
@ -136,12 +139,28 @@ class SampleGeneratorFace(SampleGeneratorBase):
batches = [ [] for _ in range(len(x)) ]
if self.add_sample_idx:
batches += [ [] ]
i_sample_idx = len(batches)-1
if self.add_pitch:
batches += [ [] ]
i_pitch = len(batches)-1
if self.add_yaw:
batches += [ [] ]
i_yaw = len(batches)-1
for i in range(len(x)):
batches[i].append ( x[i] )
if self.add_sample_idx:
batches[-1].append (idx)
batches[i_sample_idx].append (idx)
if self.add_pitch or self.add_yaw:
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw (sample.landmarks)
if self.add_pitch:
batches[i_pitch].append (pitch)
if self.add_yaw:
batches[i_yaw].append (yaw)
break
yield [ np.array(batch) for batch in batches]

7
samples/SampleLoader.py
View file

@ -68,11 +68,14 @@ class SampleLoader:
print ("%s is not a dfl image file required for training" % (s_filename_path.name) )
continue
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() )
sample_list.append( s.copy_and_set(sample_type=SampleType.FACE,
face_type=FaceType.fromString (dflimg.get_face_type()),
shape=dflimg.get_shape(),
landmarks=dflimg.get_landmarks(),
yaw=dflimg.get_yaw_value()) )
pitch=pitch,
yaw=yaw) )
except:
print ("Unable to load %s , error: %s" % (str(s_filename_path), traceback.format_exc() ) )
@ -114,7 +117,7 @@ class SampleLoader:
@staticmethod
def upgradeToFaceYawSortedSamples( samples ):
lowest_yaw, highest_yaw = -256, +256
lowest_yaw, highest_yaw = -1.0, 1.0
gradations = 64
diff_rot_per_grad = abs(highest_yaw-lowest_yaw) / gradations

6
utils/DFLJPG.py
View file

@ -152,8 +152,6 @@ class DFLJPG(object):
@staticmethod
def embed_data(filename, face_type=None,
landmarks=None,
yaw_value=None,
pitch_value=None,
source_filename=None,
source_rect=None,
source_landmarks=None
@ -163,8 +161,6 @@ class DFLJPG(object):
inst.setDFLDictData ({
'face_type': face_type,
'landmarks': landmarks,
'yaw_value': yaw_value,
'pitch_value': pitch_value,
'source_filename': source_filename,
'source_rect': source_rect,
'source_landmarks': source_landmarks
@ -226,8 +222,6 @@ class DFLJPG(object):
def get_face_type(self): return self.dfl_dict['face_type']
def get_landmarks(self): return np.array ( self.dfl_dict['landmarks'] )
def get_yaw_value(self): return self.dfl_dict['yaw_value']
def get_pitch_value(self): return self.dfl_dict['pitch_value']
def get_source_filename(self): return self.dfl_dict['source_filename']
def get_source_rect(self): return self.dfl_dict['source_rect']
def get_source_landmarks(self): return np.array ( self.dfl_dict['source_landmarks'] )

10
utils/DFLPNG.py
View file

@ -267,8 +267,6 @@ class DFLPNG(object):
@staticmethod
def embed_data(filename, face_type=None,
landmarks=None,
yaw_value=None,
pitch_value=None,
source_filename=None,
source_rect=None,
source_landmarks=None
@ -278,8 +276,6 @@ class DFLPNG(object):
inst.setDFLDictData ({
'face_type': face_type,
'landmarks': landmarks,
'yaw_value': yaw_value,
'pitch_value': pitch_value,
'source_filename': source_filename,
'source_rect': source_rect,
'source_landmarks': source_landmarks
@ -334,12 +330,6 @@ class DFLPNG(object):
def get_landmarks(self):
return np.array ( self.fcwp_dict['landmarks'] )
def get_yaw_value(self):
return self.fcwp_dict['yaw_value']
def get_pitch_value(self):
return self.fcwp_dict['pitch_value']
def get_source_filename(self):
return self.fcwp_dict['source_filename']