Merge branch 'master' into feature/ms-ssim-loss-2

# Conflicts:
#	models/Model_SAEHD/Model.py

CHANGELOG.md

@@ -10,6 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [Freezeable layers (encoder/decoder/etc.)](https://github.com/faceshiftlabs/DeepFaceLab/tree/feature/freezable-weights)
 - [GAN stability improvements](https://github.com/faceshiftlabs/DeepFaceLab/tree/feature/gan-updates)
 
+## [1.3.0] - 2020-03-20
+### Added
+- [Background Power training option](doc/features/background-power/README.md)
+
+## [1.2.1] - 2020-03-20
+### Fixed
+- Fixes bug with `fs-aug` color mode.
+
 ## [1.2.0] - 2020-03-17
 ### Added
 - [Random color training option](doc/features/random-color/README.md)
@@ -45,7 +53,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Reset stale master branch to [seranus/DeepFaceLab](https://github.com/seranus/DeepFaceLab), 
   21 commits ahead of [iperov/DeepFaceLab](https://github.com/iperov/DeepFaceLab) ([compare](https://github.com/iperov/DeepFaceLab/compare/4818183...seranus:3f5ae05))
 
-[Unreleased]: https://github.com/olivierlacan/keep-a-changelog/compare/v1.2.0...HEAD
+[Unreleased]: https://github.com/olivierlacan/keep-a-changelog/compare/v1.3.0...HEAD
+[1.3.0]: https://github.com/faceshiftlabs/DeepFaceLab/compare/v1.2.1...v1.3.0
+[1.2.1]: https://github.com/faceshiftlabs/DeepFaceLab/compare/v1.2.0...v1.2.1
 [1.2.0]: https://github.com/faceshiftlabs/DeepFaceLab/compare/v1.1.5...v1.2.0
 [1.1.5]: https://github.com/faceshiftlabs/DeepFaceLab/compare/v1.1.4...v1.1.5
 [1.1.4]: https://github.com/faceshiftlabs/DeepFaceLab/compare/v1.1.3...v1.1.4

core/imagelib/color_transfer.py

@@ -1,7 +1,7 @@
 import cv2
 import numpy as np
 from numpy import linalg as npla
-from random import random, shuffle, choice
+import random
 from scipy.stats import special_ortho_group
 import scipy as sp
 
@@ -371,12 +371,12 @@ def color_transfer(ct_mode, img_src, img_trg):
 
 
 # imported from faceswap
-def color_augmentation(img):
+def color_augmentation(img, seed=None):
     """ Color adjust RGB image """
     face = img
     face = np.clip(face*255.0, 0, 255).astype(np.uint8)
-    face = random_clahe(face)
+    face = random_clahe(face, seed)
-    face = random_lab(face)
+    face = random_lab(face, seed)
     img[:, :, :3] = face
     return (face / 255.0).astype(np.float32)
 
@@ -400,13 +400,14 @@ def random_lab_rotation(image, seed=None):
     return image
 
 
-def random_lab(image):
+def random_lab(image, seed=None):
     """ Perform random color/lightness adjustment in L*a*b* colorspace """
+    random.seed(seed)
     amount_l = 30 / 100
     amount_ab = 8 / 100
-    randoms = [(random() * amount_l * 2) - amount_l,  # L adjust
+    randoms = [(random.random() * amount_l * 2) - amount_l,  # L adjust
-                (random() * amount_ab * 2) - amount_ab,  # A adjust
+                (random.random() * amount_ab * 2) - amount_ab,  # A adjust
-                (random() * amount_ab * 2) - amount_ab]  # B adjust
+                (random.random() * amount_ab * 2) - amount_ab]  # B adjust
     image = cv2.cvtColor(  # pylint:disable=no-member
     image, cv2.COLOR_BGR2LAB).astype("float32") / 255.0  # pylint:disable=no-member
 
@@ -419,15 +420,16 @@ def random_lab(image):
                         cv2.COLOR_LAB2BGR)  # pylint:disable=no-member
     return image
 
-def random_clahe(image):
+def random_clahe(image, seed=None):
     """ Randomly perform Contrast Limited Adaptive Histogram Equalization """
-    contrast_random = random()
+    random.seed(seed)
+    contrast_random = random.random()
     if contrast_random > 50 / 100:
         return image
 
     # base_contrast = image.shape[0] // 128
     base_contrast = 1 # testing because it breaks on small sizes
-    grid_base = random() * 4
+    grid_base = random.random() * 4
     contrast_adjustment = int(grid_base * (base_contrast / 2))
     grid_size = base_contrast + contrast_adjustment
 

doc/features/background-power/README.md

@@ -0,0 +1,32 @@
+# Background Power option
+
+Allows you to train the model to include the background, which may help with areas around the mask.
+Unlike **Background Style Power**, this does not use any additional VRAM, and does not require lowering the batch size.
+
+- [DESCRIPTION](#description)
+- [USAGE](#usage)
+- [DIFFERENCE WITH BACKGROUND STYLE POWER](#difference-with-background-style-power)
+
+*Examples trained with background power `0.3`:*
+
+![](example.jpeg)![](example2.jpeg)
+
+## DESCRIPTION
+
+Applies the same loss calculation used for the area *inside* the mask, to the area *outside* the mask, multiplied with
+the chosen background power value.
+
+E.g. (simplified): Source Loss = Masked area image difference + Background Power * Non-masked area image difference
+
+## USAGE
+
+`[0.0] Background power ( 0.0..1.0 ?:help ) : 0.3`
+
+## DIFFERENCE WITH BACKGROUND STYLE POWER
+
+**Background Style Power** applies a loss to the source by comparing the background of the dest to that of the
+predicted src/dest (5th column). This operation requires additional VRAM, due to the face that the predicted src/dest
+outputs are not normally used in training (other then being viewable in the preview window).
+
+**Background Power** does *not* use the src/dest images whatsoever, instead comparing the background of the predicted
+source to that of the original source, and the same for the background of the dest images.

models/Model_SAEHD/Model.py

@@ -56,6 +56,7 @@ class SAEHDModel(ModelBase):
         default_ms_ssim_loss       = self.options['ms_ssim_loss']       = self.load_or_def_option('ms_ssim_loss', False)
 
         default_random_warp        = self.options['random_warp']        = self.load_or_def_option('random_warp', True)
+        default_background_power   = self.options['background_power']   = self.load_or_def_option('background_power', 0.0)
         default_true_face_power    = self.options['true_face_power']    = self.load_or_def_option('true_face_power', 0.0)
         default_face_style_power   = self.options['face_style_power']   = self.load_or_def_option('face_style_power', 0.0)
         default_bg_style_power     = self.options['bg_style_power']     = self.load_or_def_option('bg_style_power', 0.0)
@@ -168,6 +169,8 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
             else:
                 self.options['true_face_power'] = 0.0
 
+            self.options['background_power'] = np.clip ( io.input_number("Background power", default_background_power, add_info="0.0..1.0", help_message="Learn the area outside of the mask. Helps smooth out area near the mask boundaries. Can be used at any time"), 0.0, 1.0 )
+
             self.options['face_style_power'] = np.clip ( io.input_number("Face style power", default_face_style_power, add_info="0.0..100.0", help_message="Learn the color of the predicted face to be the same as dst inside mask. If you want to use this option with 'whole_face' you have to use XSeg trained mask. Warning: Enable it only after 10k iters, when predicted face is clear enough to start learn style. Start from 0.001 value and check history changes. Enabling this option increases the chance of model collapse."), 0.0, 100.0 )
             self.options['bg_style_power'] = np.clip ( io.input_number("Background style power", default_bg_style_power, add_info="0.0..100.0", help_message="Learn the area outside mask of the predicted face to be the same as dst. If you want to use this option with 'whole_face' you have to use XSeg trained mask. For whole_face you have to use XSeg trained mask. This can make face more like dst. Enabling this option increases the chance of model collapse. Typical value is 2.0"), 0.0, 100.0 )
 
@@ -417,12 +420,15 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
                     gpu_target_dst_style_anti_masked = gpu_target_dst*(1.0 - gpu_target_dstm_style_blur)
 
                     gpu_target_src_anti_masked = gpu_target_src*(1.0-gpu_target_srcm_blur)
+                    gpu_target_dst_anti_masked = gpu_target_dst_style_anti_masked
+
                     gpu_target_src_masked_opt  = gpu_target_src*gpu_target_srcm_blur if masked_training else gpu_target_src
                     gpu_target_dst_masked_opt  = gpu_target_dst_masked if masked_training else gpu_target_dst
 
                     gpu_pred_src_src_masked_opt = gpu_pred_src_src*gpu_target_srcm_blur if masked_training else gpu_pred_src_src
                     gpu_pred_src_src_anti_masked = gpu_pred_src_src*(1.0-gpu_target_srcm_blur)
                     gpu_pred_dst_dst_masked_opt = gpu_pred_dst_dst*gpu_target_dstm_blur if masked_training else gpu_pred_dst_dst
+                    gpu_pred_dst_dst_anti_masked = gpu_pred_dst_dst*(1.0-gpu_target_dstm_blur)
 
                     gpu_psd_target_dst_style_masked = gpu_pred_src_dst*gpu_target_dstm_style_blur
                     gpu_psd_target_dst_style_anti_masked = gpu_pred_src_dst*(1.0 - gpu_target_dstm_style_blur)
@@ -430,12 +436,12 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
                     if self.options['ms_ssim_loss']:
                         gpu_src_loss = 10 * nn.MsSsim(resolution)(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0)
                     else:
-                        if resolution < 256:
+                    if resolution < 256:
-                            gpu_src_loss =  tf.reduce_mean ( 10*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                        gpu_src_loss =  tf.reduce_mean ( 10*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
-                        else:
+                    else:
-                            gpu_src_loss =  tf.reduce_mean ( 5*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                        gpu_src_loss =  tf.reduce_mean ( 5*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
-                            gpu_src_loss += tf.reduce_mean ( 5*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/23.2)), axis=[1])
+                        gpu_src_loss += tf.reduce_mean ( 5*nn.dssim(gpu_target_src_masked_opt, gpu_pred_src_src_masked_opt, max_val=1.0, filter_size=int(resolution/23.2)), axis=[1])
-                        gpu_src_loss += tf.reduce_mean ( 10*tf.square ( gpu_target_src_masked_opt - gpu_pred_src_src_masked_opt ), axis=[1,2,3])
+                    gpu_src_loss += tf.reduce_mean ( 10*tf.square ( gpu_target_src_masked_opt - gpu_pred_src_src_masked_opt ), axis=[1,2,3])
 
                     if eyes_prio or mouth_prio:
                         if eyes_prio and mouth_prio:
@@ -448,12 +454,21 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
                         if self.options['ms_ssim_loss']:
                             gpu_src_loss += 300 * nn.MsSsim(resolution, kernel_size=5)(gpu_target_src*gpu_target_part_mask, gpu_pred_src_src*gpu_target_part_mask, max_val=1.0)
                         else:
-                            gpu_src_loss += tf.reduce_mean ( 300*tf.abs ( gpu_target_src*gpu_target_part_mask - gpu_pred_src_src*gpu_target_part_mask ), axis=[1,2,3])
+                        gpu_src_loss += tf.reduce_mean ( 300*tf.abs ( gpu_target_src*gpu_target_part_mask - gpu_pred_src_src*gpu_target_part_mask ), axis=[1,2,3])
 
                     if self.options['ms_ssim_loss']:
                         gpu_src_loss += 10 * nn.MsSsim(resolution)(gpu_target_srcm, gpu_pred_src_srcm, max_val=1.0)
                     else:
-                        gpu_src_loss += tf.reduce_mean ( 10*tf.square( gpu_target_srcm - gpu_pred_src_srcm ),axis=[1,2,3] )
+                    gpu_src_loss += tf.reduce_mean ( 10*tf.square( gpu_target_srcm - gpu_pred_src_srcm ),axis=[1,2,3] )
+
+                    if self.options['background_power'] > 0:
+                        bg_factor = self.options['background_power']
+                        if resolution < 256:
+                            gpu_src_loss +=  bg_factor * tf.reduce_mean ( 10*nn.dssim(gpu_target_src_anti_masked, gpu_pred_src_src_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                        else:
+                            gpu_src_loss +=  bg_factor * tf.reduce_mean ( 5*nn.dssim(gpu_target_src_anti_masked, gpu_pred_src_src_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                            gpu_src_loss += bg_factor * tf.reduce_mean ( 5*nn.dssim(gpu_target_src_anti_masked, gpu_pred_src_src_anti_masked, max_val=1.0, filter_size=int(resolution/23.2)), axis=[1])
+                        gpu_src_loss += bg_factor * tf.reduce_mean ( 10*tf.square ( gpu_target_src_anti_masked - gpu_pred_src_src_anti_masked ), axis=[1,2,3])
 
                     face_style_power = self.options['face_style_power'] / 100.0
                     if face_style_power != 0 and not self.pretrain:
@@ -467,12 +482,12 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
                     if self.options['ms_ssim_loss']:
                         gpu_dst_loss = 10 * nn.MsSsim(resolution)(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0)
                     else:
-                        if resolution < 256:
+                    if resolution < 256:
-                            gpu_dst_loss = tf.reduce_mean ( 10*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/11.6) ), axis=[1])
+                        gpu_dst_loss = tf.reduce_mean ( 10*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/11.6) ), axis=[1])
-                        else:
+                    else:
-                            gpu_dst_loss = tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/11.6) ), axis=[1])
+                        gpu_dst_loss = tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/11.6) ), axis=[1])
-                            gpu_dst_loss += tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/23.2) ), axis=[1])
+                        gpu_dst_loss += tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_masked_opt, gpu_pred_dst_dst_masked_opt, max_val=1.0, filter_size=int(resolution/23.2) ), axis=[1])
-                        gpu_dst_loss += tf.reduce_mean ( 10*tf.square(  gpu_target_dst_masked_opt- gpu_pred_dst_dst_masked_opt ), axis=[1,2,3])
+                    gpu_dst_loss += tf.reduce_mean ( 10*tf.square(  gpu_target_dst_masked_opt- gpu_pred_dst_dst_masked_opt ), axis=[1,2,3])
 
 
                     if eyes_prio or mouth_prio:
@@ -486,12 +501,21 @@ Examples: df, liae, df-d, df-ud, liae-ud, ...
                         if self.options['ms_ssim_loss']:
                             gpu_dst_loss += 300 * nn.MsSsim(resolution, kernel_size=5)(gpu_target_dst*gpu_target_part_mask, gpu_pred_dst_dst*gpu_target_part_mask, max_val=1.0)
                         else:
-                            gpu_dst_loss += tf.reduce_mean ( 300*tf.abs ( gpu_target_dst*gpu_target_part_mask - gpu_pred_dst_dst*gpu_target_part_mask ), axis=[1,2,3])
+                        gpu_dst_loss += tf.reduce_mean ( 300*tf.abs ( gpu_target_dst*gpu_target_part_mask - gpu_pred_dst_dst*gpu_target_part_mask ), axis=[1,2,3])
+
+                    if self.options['background_power'] > 0:
+                        bg_factor = self.options['background_power']
+                        if resolution < 256:
+                            gpu_dst_loss +=  bg_factor * tf.reduce_mean ( 10*nn.dssim(gpu_target_dst_anti_masked, gpu_pred_dst_dst_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                        else:
+                            gpu_dst_loss +=  bg_factor * tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_anti_masked, gpu_pred_dst_dst_anti_masked, max_val=1.0, filter_size=int(resolution/11.6)), axis=[1])
+                            gpu_dst_loss += bg_factor * tf.reduce_mean ( 5*nn.dssim(gpu_target_dst_anti_masked, gpu_pred_dst_dst_anti_masked, max_val=1.0, filter_size=int(resolution/23.2)), axis=[1])
+                        gpu_dst_loss += bg_factor * tf.reduce_mean ( 10*tf.square ( gpu_target_dst_anti_masked - gpu_pred_dst_dst_anti_masked ), axis=[1,2,3])
 
                     if self.options['ms_ssim_loss']:
                         gpu_dst_loss += 10 * nn.MsSsim(resolution)(gpu_target_dstm, gpu_pred_dst_dstm, max_val=1.0)
                     else:
-                        gpu_dst_loss += tf.reduce_mean ( 10*tf.square( gpu_target_dstm - gpu_pred_dst_dstm ),axis=[1,2,3] )
+                    gpu_dst_loss += tf.reduce_mean ( 10*tf.square( gpu_target_dstm - gpu_pred_dst_dstm ),axis=[1,2,3] )
 
                     gpu_src_losses += [gpu_src_loss]
                     gpu_dst_losses += [gpu_dst_loss]

samplelib/SampleProcessor.py

@@ -207,7 +207,7 @@ class SampleProcessor(object):
                         # Apply random color transfer
                         if ct_mode is not None and ct_sample is not None or ct_mode == 'fs-aug':
                             if ct_mode == 'fs-aug':
-                                img = imagelib.color_augmentation(img)
+                                img = imagelib.color_augmentation(img, sample_rnd_seed)
                             else:
                                 if ct_sample_bgr is None:
                                     ct_sample_bgr = ct_sample.load_bgr()