github/DeepFaceLab
1
0
Fork 0
mirror of https://github.com/iperov/DeepFaceLab.git synced 2025-08-14 02:37:00 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Converter:

Browse source 
Session is now saved to the model folder.

blur and erode ranges are increased to -400+400

hist-match-bw is now replaced with seamless2 mode.

Added 'ebs' color transfer mode (works only on Windows).

FANSEG model (used in FAN-x mask modes) is retrained with new model configuration
and now produces better precision and less jitter
This commit is contained in:
Colombo 2019-09-07 13:57:42 +04:00
commit 7ed38a8097
29 changed files with 768 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
facelib/FANSeg_256_full_face.h5
View file

Binary file not shown.

69
facelib/FANSegmentator.py
View file

@ -47,10 +47,30 @@ class FANSegmentator(object):
self.model.get_layer (s).set_weights ( d[s] )
except:
io.log_err("Unable to load VGG11 pretrained weights from vgg11_enc_weights.npy")
conv_weights_list = []
for layer in self.model.layers:
if 'CA.' in layer.name:
conv_weights_list += [layer.weights[0]] #Conv2D kernel_weights
CAInitializerMP ( conv_weights_list )
if training:
#self.model.compile(loss='mse', optimizer=Adam(tf_cpu_mode=2))
self.model.compile(loss='binary_crossentropy', optimizer=Adam(tf_cpu_mode=2) )
inp_t = Input ( (resolution, resolution, 3) )
real_t = Input ( (resolution, resolution, 1) )
out_t = self.model(inp_t)
#loss = K.mean(10*K.square(out_t-real_t))
loss = K.mean(10*K.binary_crossentropy(real_t,out_t) )
out_t_diff1 = out_t[:, 1:, :, :] - out_t[:, :-1, :, :]
out_t_diff2 = out_t[:, :, 1:, :] - out_t[:, :, :-1, :]
total_var_loss = K.mean( 0.1*K.abs(out_t_diff1), axis=[1, 2, 3] ) + K.mean( 0.1*K.abs(out_t_diff2), axis=[1, 2, 3] )
opt = Adam(lr=0.0001, beta_1=0.5, beta_2=0.999, tf_cpu_mode=2)
self.train_func = K.function ( [inp_t, real_t], [K.mean(loss)], opt.get_updates( [loss,total_var_loss], self.model.trainable_weights) )
def __enter__(self):
return self
@ -61,8 +81,9 @@ class FANSegmentator(object):
def save_weights(self):
self.model.save_weights (str(self.weights_path))
def train_on_batch(self, inp, outp):
return self.model.train_on_batch(inp, outp)
def train(self, inp, real):
loss, = self.train_func ([inp, real])
return loss
def extract (self, input_image, is_input_tanh=False):
input_shape_len = len(input_image.shape)
@ -78,62 +99,62 @@ class FANSegmentator(object):
return result
@staticmethod
def BuildModel ( resolution, ngf=64, norm='', act='lrelu'):
def BuildModel ( resolution, ngf=64):
exec( nnlib.import_all(), locals(), globals() )
inp = Input ( (resolution,resolution,3) )
x = inp
x = FANSegmentator.Flow(ngf=ngf, norm=norm, act=act)(x)
x = FANSegmentator.Flow(ngf=ngf)(x)
model = Model(inp,x)
return model
@staticmethod
def Flow(ngf=64, num_downs=4, norm='', act='lrelu'):
def Flow(ngf=64):
exec( nnlib.import_all(), locals(), globals() )
def func(input):
x = input
x0 = x = Conv2D(ngf, kernel_size=3, strides=1, padding='same', activation='relu', name='features.0')(x)
x = MaxPooling2D()(x)
x = BlurPool(filt_size=3)(x) #x = MaxPooling2D()(x)
x1 = x = Conv2D(ngf*2, kernel_size=3, strides=1, padding='same', activation='relu', name='features.3')(x)
x = MaxPooling2D()(x)
x = BlurPool(filt_size=3)(x)
x = Conv2D(ngf*4, kernel_size=3, strides=1, padding='same', activation='relu', name='features.6')(x)
x2 = x = Conv2D(ngf*4, kernel_size=3, strides=1, padding='same', activation='relu', name='features.8')(x)
x = MaxPooling2D()(x)
x = BlurPool(filt_size=3)(x)
x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same', activation='relu', name='features.11')(x)
x3 = x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same', activation='relu', name='features.13')(x)
x = MaxPooling2D()(x)
x = BlurPool(filt_size=3)(x)
x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same', activation='relu', name='features.16')(x)
x4 = x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same', activation='relu', name='features.18')(x)
x = MaxPooling2D()(x)
x = BlurPool(filt_size=3)(x)
x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same')(x)
x = Conv2D(ngf*8, kernel_size=3, strides=1, padding='same', name='CA.1')(x)
x = Conv2DTranspose (ngf*4, 3, strides=2, padding='same', activation='relu') (x)
x = Conv2DTranspose (ngf*4, 3, strides=2, padding='same', activation='relu', name='CA.2') (x)
x = Concatenate(axis=3)([ x, x4])
x = Conv2D (ngf*8, 3, strides=1, padding='same', activation='relu') (x)
x = Conv2D (ngf*8, 3, strides=1, padding='same', activation='relu', name='CA.3') (x)
x = Conv2DTranspose (ngf*4, 3, strides=2, padding='same', activation='relu') (x)
x = Conv2DTranspose (ngf*4, 3, strides=2, padding='same', activation='relu', name='CA.4') (x)
x = Concatenate(axis=3)([ x, x3])
x = Conv2D (ngf*8, 3, strides=1, padding='same', activation='relu') (x)
x = Conv2D (ngf*8, 3, strides=1, padding='same', activation='relu', name='CA.5') (x)
x = Conv2DTranspose (ngf*2, 3, strides=2, padding='same', activation='relu') (x)
x = Conv2DTranspose (ngf*2, 3, strides=2, padding='same', activation='relu', name='CA.6') (x)
x = Concatenate(axis=3)([ x, x2])
x = Conv2D (ngf*4, 3, strides=1, padding='same', activation='relu') (x)
x = Conv2D (ngf*4, 3, strides=1, padding='same', activation='relu', name='CA.7') (x)
x = Conv2DTranspose (ngf, 3, strides=2, padding='same', activation='relu') (x)
x = Conv2DTranspose (ngf, 3, strides=2, padding='same', activation='relu', name='CA.8') (x)
x = Concatenate(axis=3)([ x, x1])
x = Conv2D (ngf*2, 3, strides=1, padding='same', activation='relu') (x)
x = Conv2D (ngf*2, 3, strides=1, padding='same', activation='relu', name='CA.9') (x)
x = Conv2DTranspose (ngf // 2, 3, strides=2, padding='same', activation='relu') (x)
x = Conv2DTranspose (ngf // 2, 3, strides=2, padding='same', activation='relu', name='CA.10') (x)
x = Concatenate(axis=3)([ x, x0])
x = Conv2D (ngf, 3, strides=1, padding='same', activation='relu') (x)
x = Conv2D (ngf, 3, strides=1, padding='same', activation='relu', name='CA.11') (x)
return Conv2D(1, 3, strides=1, padding='same', activation='sigmoid')(x)
return Conv2D(1, 3, strides=1, padding='same', activation='sigmoid', name='CA.12')(x)
return func

222
facelib/LandmarksProcessor.py
View file

@ -249,52 +249,224 @@ def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
return mat
def get_image_hull_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0, ie_polys=None):
if len(image_landmarks) != 68:
raise Exception('get_image_hull_mask works only with 68 landmarks')
int_lmrks = np.array(image_landmarks.copy(), dtype=np.int)
hull_mask = np.zeros(image_shape[0:2]+(1,),dtype=np.float32)
def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0):
if len(lmrks) != 68:
raise Exception('works only with 68 landmarks')
lmrks = np.array( lmrks.copy(), dtype=np.int )
# #nose
ml_pnt = (int_lmrks[36] + int_lmrks[0]) // 2
mr_pnt = (int_lmrks[16] + int_lmrks[45]) // 2
ml_pnt = (lmrks[36] + lmrks[0]) // 2
mr_pnt = (lmrks[16] + lmrks[45]) // 2
# mid points between the mid points and eye
ql_pnt = (int_lmrks[36] + ml_pnt) // 2
qr_pnt = (int_lmrks[45] + mr_pnt) // 2
ql_pnt = (lmrks[36] + ml_pnt) // 2
qr_pnt = (lmrks[45] + mr_pnt) // 2
# Top of the eye arrays
bot_l = np.array((ql_pnt, int_lmrks[36], int_lmrks[37], int_lmrks[38], int_lmrks[39]))
bot_r = np.array((int_lmrks[42], int_lmrks[43], int_lmrks[44], int_lmrks[45], qr_pnt))
bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39]))
bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt))
# Eyebrow arrays
top_l = int_lmrks[17:22]
top_r = int_lmrks[22:27]
top_l = lmrks[17:22]
top_r = lmrks[22:27]
# Adjust eyebrow arrays
int_lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l)
int_lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r)
lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l)
lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r)
return lmrks
r_jaw = (int_lmrks[0:9], int_lmrks[17:18])
l_jaw = (int_lmrks[8:17], int_lmrks[26:27])
r_cheek = (int_lmrks[17:20], int_lmrks[8:9])
l_cheek = (int_lmrks[24:27], int_lmrks[8:9])
nose_ridge = (int_lmrks[19:25], int_lmrks[8:9],)
r_eye = (int_lmrks[17:22], int_lmrks[27:28], int_lmrks[31:36], int_lmrks[8:9])
l_eye = (int_lmrks[22:27], int_lmrks[27:28], int_lmrks[31:36], int_lmrks[8:9])
nose = (int_lmrks[27:31], int_lmrks[31:36])
def get_image_hull_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0, ie_polys=None, color=(1,) ):
hull_mask = np.zeros(image_shape[0:2]+( len(color),),dtype=np.float32)
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
r_jaw = (lmrks[0:9], lmrks[17:18])
l_jaw = (lmrks[8:17], lmrks[26:27])
r_cheek = (lmrks[17:20], lmrks[8:9])
l_cheek = (lmrks[24:27], lmrks[8:9])
nose_ridge = (lmrks[19:25], lmrks[8:9],)
r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9])
l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9])
nose = (lmrks[27:31], lmrks[31:36])
parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
for item in parts:
merged = np.concatenate(item)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), 1)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), color )
if ie_polys is not None:
ie_polys.overlay_mask(hull_mask)
return hull_mask
def alpha_to_color (img_alpha, color):
if len(img_alpha.shape) == 2:
img_alpha = img_alpha[...,None]
h,w,c = img_alpha.shape
result = np.zeros( (h,w, len(color) ), dtype=np.float32 )
result[:,:] = color
return result * img_alpha
def get_cmask (image_shape, lmrks, eyebrows_expand_mod=1.0):
h,w,c = image_shape
hull = get_image_hull_mask (image_shape, lmrks, eyebrows_expand_mod, color=(1,) )
result = np.zeros( (h,w,3), dtype=np.float32 )
def process(w,h, data ):
d = {}
cur_lc = 0
all_lines = []
for s, pts_loop_ar in data:
lines = []
for pts, loop in pts_loop_ar:
pts_len = len(pts)
lines.append ( [ [ pts[i], pts[(i+1) % pts_len ] ] for i in range(pts_len - (0 if loop else 1) ) ] )
lines = np.concatenate (lines)
lc = lines.shape[0]
all_lines.append(lines)
d[s] = cur_lc, cur_lc+lc
cur_lc += lc
all_lines = np.concatenate (all_lines, 0)
#calculate signed distance for all points and lines
line_count = all_lines.shape[0]
pts_count = w*h
all_lines = np.repeat ( all_lines[None,...], pts_count, axis=0 ).reshape ( (pts_count*line_count,2,2) )
pts = np.empty( (h,w,line_count,2), dtype=np.float32 )
pts[...,1] = np.arange(h)[:,None,None]
pts[...,0] = np.arange(w)[:,None]
pts = pts.reshape ( (h*w*line_count, -1) )
a = all_lines[:,0,:]
b = all_lines[:,1,:]
pa = pts-a
ba = b-a
ph = np.clip ( np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1 )
dists = npla.norm ( pa - ba*ph[...,None], axis=1).reshape ( (h,w,line_count) )
def get_dists(name, thickness=0):
s,e = d[name]
result = dists[...,s:e]
if thickness != 0:
result = np.abs(result)-thickness
return np.min (result, axis=-1)
return get_dists
l_eye = lmrks[42:48]
r_eye = lmrks[36:42]
l_brow = lmrks[22:27]
r_brow = lmrks[17:22]
mouth = lmrks[48:60]
up_nose = np.concatenate( (lmrks[27:31], lmrks[33:34]) )
down_nose = lmrks[31:36]
nose = np.concatenate ( (up_nose, down_nose) )
gdf = process ( w,h,
(
('eyes', ((l_eye, True), (r_eye, True)) ),
('brows', ((l_brow, False), (r_brow,False)) ),
('up_nose', ((up_nose, False),) ),
('down_nose', ((down_nose, False),) ),
('mouth', ((mouth, True),) ),
)
)
#import code
#code.interact(local=dict(globals(), **locals()))
eyes_fall_dist = w // 32
eyes_thickness = max( w // 64, 1 )
brows_fall_dist = w // 32
brows_thickness = max( w // 256, 1 )
nose_fall_dist = w / 12
nose_thickness = max( w // 96, 1 )
mouth_fall_dist = w // 32
mouth_thickness = max( w // 64, 1 )
eyes_mask = gdf('eyes',eyes_thickness)
eyes_mask = 1-np.clip( eyes_mask/ eyes_fall_dist, 0, 1)
#eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
#eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
brows_mask = gdf('brows', brows_thickness)
brows_mask = 1-np.clip( brows_mask / brows_fall_dist, 0, 1)
#brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1)
mouth_mask = gdf('mouth', mouth_thickness)
mouth_mask = 1-np.clip( mouth_mask / mouth_fall_dist, 0, 1)
#mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1)
def blend(a,b,k):
x = np.clip ( 0.5+0.5*(b-a)/k, 0.0, 1.0 )
return (a-b)*x+b - k*x*(1.0-x)
#nose_mask = (a-b)*x+b - k*x*(1.0-x)
#nose_mask = np.minimum (up_nose_mask , down_nose_mask )
#nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
nose_mask = blend ( gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness*3 )
nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
up_nose_mask = gdf('up_nose', nose_thickness)
up_nose_mask = 1-np.clip( up_nose_mask / nose_fall_dist, 0, 1)
#up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1)
down_nose_mask = gdf('down_nose', nose_thickness)
down_nose_mask = 1-np.clip( down_nose_mask / nose_fall_dist, 0, 1)
#down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1)
#nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 )
#nose_mask /= np.max(nose_mask)
#nose_mask = np.maximum (up_nose_mask , down_nose_mask )
#nose_mask = down_nose_mask
#nose_mask = np.zeros_like(nose_mask)
eyes_mask = eyes_mask * (1-mouth_mask)
nose_mask = nose_mask * (1-eyes_mask)
hull_mask = hull[...,0].copy()
hull_mask = hull_mask * (1-eyes_mask) * (1-brows_mask) * (1-nose_mask) * (1-mouth_mask)
#eyes_mask = eyes_mask * (1-nose_mask)
mouth_mask= mouth_mask * (1-nose_mask)
brows_mask = brows_mask * (1-nose_mask)* (1-eyes_mask )
hull_mask = alpha_to_color(hull_mask, (0,1,0) )
eyes_mask = alpha_to_color(eyes_mask, (1,0,0) )
brows_mask = alpha_to_color(brows_mask, (0,0,1) )
nose_mask = alpha_to_color(nose_mask, (0,1,1) )
mouth_mask = alpha_to_color(mouth_mask, (0,0,1) )
#nose_mask = np.maximum( up_nose_mask, down_nose_mask )
result = hull_mask + mouth_mask+ nose_mask + brows_mask + eyes_mask
result *= hull
#result = np.clip (result, 0, 1)
return result
def get_image_eye_mask (image_shape, image_landmarks):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')