From 2dabecaa3616a956d97ddc7a7a0c620d22e8b26d Mon Sep 17 00:00:00 2001 From: Cioscos Date: Thu, 14 Oct 2021 20:54:34 +0200 Subject: [PATCH] Added last 5 Iperov commits - DFLJPG increase max chunk size for xseg data - dev_misc: code to extract microsoft/FaceSynthetics dataset - SAEHD: better training for bg_style_power - Change in c SAEHD variant - fix save in trainer --- DFLIMG/DFLJPG.py | 2 +- core/leras/archis/DeepFakeArchi.py | 2 +- mainscripts/Trainer.py | 8 ++- mainscripts/dev_misc.py | 96 +++++++++++++++++++++++++++++- models/Model_SAEHD/Model.py | 29 ++++----- 5 files changed, 112 insertions(+), 25 deletions(-) diff --git a/DFLIMG/DFLJPG.py b/DFLIMG/DFLJPG.py index 291db71..614cebc 100644 --- a/DFLIMG/DFLJPG.py +++ b/DFLIMG/DFLJPG.py @@ -308,7 +308,7 @@ class DFLJPG(object): mask_a = imagelib.normalize_channels(mask_a, 1) img_data = np.clip( mask_a*255, 0, 255 ).astype(np.uint8) - data_max_len = 8192 + data_max_len = 50000 ret, buf = cv2.imencode('.png', img_data) diff --git a/core/leras/archis/DeepFakeArchi.py b/core/leras/archis/DeepFakeArchi.py index e630624..3896aa5 100644 --- a/core/leras/archis/DeepFakeArchi.py +++ b/core/leras/archis/DeepFakeArchi.py @@ -23,7 +23,7 @@ class DeepFakeArchi(nn.ArchiBase): if 'c' in opts: def act(x, alpha=0.1): - return tf.nn.relu(x) + return x*tf.cos(x) else: def act(x, alpha=0.1): return tf.nn.leaky_relu(x, alpha) diff --git a/mainscripts/Trainer.py b/mainscripts/Trainer.py index 1f86f9d..8f3e4fe 100644 --- a/mainscripts/Trainer.py +++ b/mainscripts/Trainer.py @@ -184,8 +184,12 @@ def trainerThread (s2c, c2s, e, is_reached_goal = True io.log_info('You can use preview now.') - if not is_reached_goal and (time.time() - last_save_time) >= save_interval_min * 60: - last_save_time += save_interval_min * 60 + need_save = False + while time.time() - last_save_time >= save_interval_min*60: + last_save_time += save_interval_min*60 + need_save = True + + if not is_reached_goal and need_save: model_save() send_preview() diff --git a/mainscripts/dev_misc.py b/mainscripts/dev_misc.py index a71bcdf..93a4359 100644 --- a/mainscripts/dev_misc.py +++ b/mainscripts/dev_misc.py @@ -13,7 +13,6 @@ from core.joblib import Subprocessor from core.leras import nn from DFLIMG import * from facelib import FaceType, LandmarksProcessor - from . import Extractor, Sorter from .Extractor import ExtractSubprocessor @@ -359,7 +358,7 @@ def extract_umd_csv(input_file_csv, -def dev_test(input_dir): +def dev_test1(input_dir): # LaPa dataset image_size = 1024 @@ -500,3 +499,96 @@ def dev_segmented_trash(input_dir): except: io.log_info ('fail to trashing %s' % (src.name) ) + + +def dev_test(input_dir): + """ + extract FaceSynthetics dataset https://github.com/microsoft/FaceSynthetics + + BACKGROUND = 0 + SKIN = 1 + NOSE = 2 + RIGHT_EYE = 3 + LEFT_EYE = 4 + RIGHT_BROW = 5 + LEFT_BROW = 6 + RIGHT_EAR = 7 + LEFT_EAR = 8 + MOUTH_INTERIOR = 9 + TOP_LIP = 10 + BOTTOM_LIP = 11 + NECK = 12 + HAIR = 13 + BEARD = 14 + CLOTHING = 15 + GLASSES = 16 + HEADWEAR = 17 + FACEWEAR = 18 + IGNORE = 255 + """ + + + image_size = 1024 + face_type = FaceType.WHOLE_FACE + + input_path = Path(input_dir) + + + + output_path = input_path.parent / f'{input_path.name}_out' + if output_path.exists(): + output_images_paths = pathex.get_image_paths(output_path) + if len(output_images_paths) != 0: + io.input(f"\n WARNING !!! \n {output_path} contains files! \n They will be deleted. \n Press enter to continue.\n") + for filename in output_images_paths: + Path(filename).unlink() + output_path.mkdir(parents=True, exist_ok=True) + + data = [] + + for filepath in io.progress_bar_generator(pathex.get_paths(input_path), "Processing"): + if filepath.suffix == '.txt': + + image_filepath = filepath.parent / f'{filepath.name.split("_")[0]}.png' + if not image_filepath.exists(): + print(f'{image_filepath} does not exist, skipping') + + lmrks = [] + for lmrk_line in filepath.read_text().split('\n'): + if len(lmrk_line) == 0: + continue + + x, y = lmrk_line.split(' ') + x, y = float(x), float(y) + + lmrks.append( (x,y) ) + + lmrks = np.array(lmrks[:68], np.float32) + rect = LandmarksProcessor.get_rect_from_landmarks(lmrks) + data += [ ExtractSubprocessor.Data(filepath=image_filepath, rects=[rect], landmarks=[ lmrks ] ) ] + + if len(data) > 0: + io.log_info ("Performing 3rd pass...") + data = ExtractSubprocessor (data, 'final', image_size, 95, face_type, final_output_path=output_path, device_config=nn.DeviceConfig.CPU()).run() + + for filename in io.progress_bar_generator(pathex.get_image_paths (output_path), "Processing"): + filepath = Path(filename) + + dflimg = DFLJPG.load(filepath) + + src_filename = dflimg.get_source_filename() + image_to_face_mat = dflimg.get_image_to_face_mat() + + seg_filepath = input_path / ( Path(src_filename).stem + '_seg.png') + if not seg_filepath.exists(): + raise ValueError(f'{seg_filepath} does not exist') + + seg = cv2_imread(seg_filepath) + seg_inds = np.isin(seg, [1,2,3,4,5,6,9,10,11]) + seg[~seg_inds] = 0 + seg[seg_inds] = 1 + seg = seg.astype(np.float32) + seg = cv2.warpAffine(seg, image_to_face_mat, (image_size, image_size), cv2.INTER_LANCZOS4) + dflimg.set_xseg_mask(seg) + dflimg.save() + \ No newline at end of file diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index ae28076..042b037 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -261,7 +261,6 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... random_src_flip = self.random_src_flip if not self.pretrain else True random_dst_flip = self.random_dst_flip if not self.pretrain else True blur_out_mask = self.options['blur_out_mask'] - learn_dst_bg = False#True if self.pretrain: self.options_show_override['gan_power'] = 0.0 @@ -451,7 +450,6 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_pred_src_src, gpu_pred_src_srcm = self.decoder(gpu_src_code) gpu_pred_dst_dst, gpu_pred_dst_dstm = self.decoder(gpu_dst_code) - gpu_pred_dst_dst_no_code_grad, _ = self.decoder(tf.stop_gradient(gpu_dst_code)) gpu_pred_src_dst, gpu_pred_src_dstm = self.decoder(gpu_src_dst_code) gpu_pred_src_dst_no_code_grad, _ = self.decoder(tf.stop_gradient(gpu_src_dst_code)) @@ -478,19 +476,16 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_target_srcm_anti_blur = 1.0-gpu_target_srcm_blur gpu_target_dstm_blur = nn.gaussian_blur(gpu_target_dstm, max(1, resolution // 32) ) - gpu_target_dstm_style_blur = gpu_target_dstm_blur #default style mask is 0.5 on boundary - gpu_target_dstm_style_anti_blur = 1.0 - gpu_target_dstm_style_blur gpu_target_dstm_blur = tf.clip_by_value(gpu_target_dstm_blur, 0, 0.5) * 2 - gpu_target_dstm_anti_blur = 1.0-gpu_target_dstm_blur - gpu_target_dst_masked = gpu_target_dst*gpu_target_dstm_blur - gpu_target_dst_style_masked = gpu_target_dst*gpu_target_dstm_style_blur - gpu_target_dst_style_anti_masked = gpu_target_dst*gpu_target_dstm_style_anti_blur + gpu_style_mask_blur = nn.gaussian_blur(gpu_pred_src_dstm*gpu_pred_dst_dstm, max(1, resolution // 32) ) + gpu_style_mask_blur = tf.stop_gradient(tf.clip_by_value(gpu_target_srcm_blur, 0, 1.0)) + gpu_style_mask_anti_blur = 1.0 - gpu_style_mask_blur + + gpu_target_dst_masked = gpu_target_dst*gpu_target_dstm_blur gpu_target_src_anti_masked = gpu_target_src*gpu_target_srcm_anti_blur - gpu_target_dst_anti_masked = gpu_target_dst*gpu_target_dstm_anti_blur gpu_pred_src_src_anti_masked = gpu_pred_src_src*gpu_target_srcm_anti_blur - gpu_pred_dst_dst_anti_masked = gpu_pred_dst_dst*gpu_target_dstm_anti_blur gpu_target_src_masked_opt = gpu_target_src*gpu_target_srcm_blur if masked_training else gpu_target_src gpu_target_dst_masked_opt = gpu_target_dst_masked if masked_training else gpu_target_dst @@ -498,9 +493,6 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_pred_src_src_anti_masked = gpu_pred_src_src*(1.0-gpu_target_srcm_blur) gpu_pred_dst_dst_masked_opt = gpu_pred_dst_dst*gpu_target_dstm_blur if masked_training else gpu_pred_dst_dst - gpu_psd_target_dst_style_masked = gpu_pred_src_dst*gpu_target_dstm_style_blur - gpu_psd_target_dst_style_anti_masked = gpu_pred_src_dst*gpu_target_dstm_style_anti_blur - if self.options['loss_function'] == 'MS-SSIM': gpu_src_loss = 10 * nn.MsSsim(bs_per_gpu, input_ch, resolution)(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0) gpu_src_loss += tf.reduce_mean ( 10*tf.square ( gpu_target_src_masked_opt - gpu_pred_src_src_masked_opt ), axis=[1,2,3]) @@ -545,12 +537,14 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... face_style_power = self.options['face_style_power'] / 100.0 if face_style_power != 0 and not self.pretrain: gpu_src_loss += nn.style_loss(gpu_pred_src_dst_no_code_grad*tf.stop_gradient(gpu_pred_src_dstm), tf.stop_gradient(gpu_pred_dst_dst*gpu_pred_dst_dstm), gaussian_blur_radius=resolution//8, loss_weight=10000*face_style_power) - #gpu_src_loss += nn.style_loss(gpu_psd_target_dst_style_masked, gpu_target_dst_style_masked, gaussian_blur_radius=resolution//16, loss_weight=10000*face_style_power) bg_style_power = self.options['bg_style_power'] / 100.0 if bg_style_power != 0 and not self.pretrain: - gpu_src_loss += tf.reduce_mean( (10*bg_style_power)*nn.dssim( gpu_psd_target_dst_style_anti_masked, gpu_target_dst_style_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1]) - gpu_src_loss += tf.reduce_mean( (10*bg_style_power)*tf.square(gpu_psd_target_dst_style_anti_masked - gpu_target_dst_style_anti_masked), axis=[1,2,3] ) + gpu_target_dst_style_anti_masked = gpu_target_dst*gpu_style_mask_anti_blur + gpu_psd_style_anti_masked = gpu_pred_src_dst*gpu_style_mask_anti_blur + + gpu_src_loss += tf.reduce_mean( (10*bg_style_power)*nn.dssim( gpu_psd_style_anti_masked, gpu_target_dst_style_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1]) + gpu_src_loss += tf.reduce_mean( (10*bg_style_power)*tf.square(gpu_psd_style_anti_masked - gpu_target_dst_style_anti_masked), axis=[1,2,3] ) if self.options['loss_function'] == 'MS-SSIM': gpu_dst_loss = 10 * nn.MsSsim(bs_per_gpu, input_ch, resolution)(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0) @@ -598,9 +592,6 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_G_loss = gpu_src_loss + gpu_dst_loss - if learn_dst_bg and masked_training and 'liae' in archi_type: - gpu_G_loss += tf.reduce_mean( tf.square(gpu_pred_dst_dst_no_code_grad*gpu_target_dstm_anti_blur-gpu_target_dst_anti_masked),axis=[1,2,3] ) - def DLoss(labels,logits): return tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits), axis=[1,2,3])