converter:

fixed crashes

removed useless 'ebs' color transfer

changed keys for color degrade

added image degrade via denoise - same as denoise extracted data_dst.bat ,
but you can control this option directly in the interactive converter

added image degrade via bicubic downscale and upscale

SAEHD: default ae_dims for df now 256.

converters/ConvertMasked.py

@@ -8,10 +8,8 @@ from facelib import FaceType, LandmarksProcessor
 from interact import interact as io
 from utils.cv2_utils import *
 
-
 def ConvertMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, img_bgr_uint8, img_bgr, img_face_landmarks):
     img_size = img_bgr.shape[1], img_bgr.shape[0]
-
     img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr.shape, img_face_landmarks)
 
     if cfg.mode == 'original':
@@ -231,7 +229,7 @@ def ConvertMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, i
                 hist_match_2 = dst_face_bgr*hist_mask_a + white
                 hist_match_2[ hist_match_1 > 1.0 ] = 1.0
 
-                prd_face_bgr = imagelib.color_hist_match(hist_match_1, hist_match_2, cfg.hist_match_threshold )
+                prd_face_bgr = imagelib.color_hist_match(hist_match_1, hist_match_2, cfg.hist_match_threshold ).astype(dtype=np.float32)
 
             if cfg.mode == 'hist-match-bw':
                 prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
@@ -254,9 +252,7 @@ def ConvertMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, i
                     break
 
             if cfg.mode == 'seamless2':
-
                 face_seamless = imagelib.seamless_clone ( prd_face_bgr, dst_face_bgr, img_face_seamless_mask_a )
-
                 out_img = cv2.warpAffine( face_seamless, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
             else:
                 out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
@@ -279,6 +275,7 @@ def ConvertMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, i
                     else:
                         print ("Seamless fail: " + e_str)
             
+    
             out_img = img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa)
 
             out_face_bgr = cv2.warpAffine( out_img, face_mat, (output_size, output_size) )
@@ -322,6 +319,23 @@ def ConvertMaskedFace (predictor_func, predictor_input_shape, cfg, frame_info, i
             if cfg.blursharpen_amount != 0:
                 out_face_bgr = cfg.blursharpen_func ( out_face_bgr, cfg.sharpen_mode, 3, cfg.blursharpen_amount)
 
+
+            if cfg.image_denoise_power != 0:
+                n = cfg.image_denoise_power
+                while n > 0:
+                    img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
+                    if int(n / 100) != 0:
+                        img_bgr = img_bgr_denoised
+                    else:
+                        pass_power = (n % 100) / 100.0
+                        img_bgr = img_bgr*(1.0-pass_power)+img_bgr_denoised*pass_power
+                    n = max(n-10,0)
+
+            if cfg.bicubic_degrade_power != 0:
+                p = 1.0 - cfg.bicubic_degrade_power / 101.0
+                img_bgr_downscaled = cv2.resize (img_bgr, ( int(img_size[0]*p), int(img_size[1]*p ) ), cv2.INTER_CUBIC)
+                img_bgr = cv2.resize (img_bgr_downscaled, img_size, cv2.INTER_CUBIC)
+
             new_out = cv2.warpAffine( out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT )
             out_img =  np.clip( img_bgr*(1-img_face_mask_aaa) + (new_out*img_face_mask_aaa) , 0, 1.0 )
 

converters/ConverterConfig.py

@@ -14,7 +14,13 @@ class ConverterConfig(object):
     TYPE_IMAGE = 3
     TYPE_IMAGE_WITH_LANDMARKS = 4
 
-    def __init__(self, type=0):
+    def __init__(self, type=0,
+
+                       super_resolution_mode=0,
+                       sharpen_mode=0,
+                       blursharpen_amount=0,
+                       **kwargs
+                       ):
         self.type = type
 
         self.superres_func = None
@@ -30,9 +36,9 @@ class ConverterConfig(object):
         self.sharpen_dict = {0:"None", 1:'box', 2:'gaussian'}
 
         #default changeable params
-        self.super_resolution_mode = 0
-        self.sharpen_mode = 0
-        self.blursharpen_amount = 0
+        self.super_resolution_mode = super_resolution_mode
+        self.sharpen_mode = sharpen_mode
+        self.blursharpen_amount = blursharpen_amount
 
     def copy(self):
         return copy.copy(self)
@@ -65,6 +71,16 @@ class ConverterConfig(object):
         a = list( self.super_res_dict.keys() )
         self.super_resolution_mode = a[ (a.index(self.super_resolution_mode)+1) % len(a) ]
 
+    #overridable
+    def get_config(self):
+        d = self.__dict__.copy()
+        d.pop('type')
+        return d
+        return {'sharpen_mode':self.sharpen_mode,
+                'blursharpen_amount':self.blursharpen_amount,
+                'super_resolution_mode':self.super_resolution_mode
+                }
+
     #overridable
     def __eq__(self, other):
         #check equality of changeable params
@@ -115,9 +131,24 @@ class ConverterConfigMasked(ConverterConfig):
     def __init__(self, face_type=FaceType.FULL,
                        default_mode = 4,
                        clip_hborder_mask_per = 0,
+
+                       mode='overlay',
+                       masked_hist_match=True,
+                       hist_match_threshold = 238,
+                       mask_mode = 1,
+                       erode_mask_modifier = 0,
+                       blur_mask_modifier = 0,
+                       motion_blur_power = 0,
+                       output_face_scale = 0,
+                       color_transfer_mode = 0,
+                       image_denoise_power = 0,
+                       bicubic_degrade_power = 0,
+                       color_degrade_power = 0,
+                       export_mask_alpha = False,
+                       **kwargs
                        ):
 
-        super().__init__(type=ConverterConfig.TYPE_MASKED)
+        super().__init__(type=ConverterConfig.TYPE_MASKED, **kwargs)
 
         self.face_type = face_type
         if self.face_type not in [FaceType.HALF, FaceType.MID_FULL, FaceType.FULL ]:
@@ -127,17 +158,19 @@ class ConverterConfigMasked(ConverterConfig):
         self.clip_hborder_mask_per = clip_hborder_mask_per
 
         #default changeable params
-        self.mode = 'overlay'
-        self.masked_hist_match = True
-        self.hist_match_threshold = 238
-        self.mask_mode = 1
-        self.erode_mask_modifier = 0
-        self.blur_mask_modifier = 0
-        self.motion_blur_power = 0
-        self.output_face_scale = 0
-        self.color_transfer_mode = 0
-        self.color_degrade_power = 0
-        self.export_mask_alpha = False
+        self.mode = mode
+        self.masked_hist_match = masked_hist_match
+        self.hist_match_threshold = hist_match_threshold
+        self.mask_mode = mask_mode
+        self.erode_mask_modifier = erode_mask_modifier
+        self.blur_mask_modifier = blur_mask_modifier
+        self.motion_blur_power = motion_blur_power
+        self.output_face_scale = output_face_scale
+        self.color_transfer_mode = color_transfer_mode
+        self.image_denoise_power = image_denoise_power
+        self.bicubic_degrade_power = bicubic_degrade_power
+        self.color_degrade_power = color_degrade_power
+        self.export_mask_alpha = export_mask_alpha
 
     def copy(self):
         return copy.copy(self)
@@ -178,6 +211,12 @@ class ConverterConfigMasked(ConverterConfig):
     def add_color_degrade_power(self, diff):
         self.color_degrade_power = np.clip ( self.color_degrade_power+diff , 0, 100)
 
+    def add_image_denoise_power(self, diff):
+        self.image_denoise_power = np.clip ( self.image_denoise_power+diff, 0, 500)
+
+    def add_bicubic_degrade_power(self, diff):
+        self.bicubic_degrade_power = np.clip ( self.bicubic_degrade_power+diff, 0, 100)
+
     def toggle_export_mask_alpha(self):
         self.export_mask_alpha = not self.export_mask_alpha
 
@@ -227,6 +266,8 @@ class ConverterConfigMasked(ConverterConfig):
         super().ask_settings()
 
         if 'raw' not in self.mode:
+            self.image_denoise_power = np.clip ( io.input_int ("Choose image degrade by denoise power [0..500] (skip:%d) : " % (0), 0), 0, 500)
+            self.bicubic_degrade_power = np.clip ( io.input_int ("Choose image degrade by bicubic rescale power [0..100] (skip:%d) : " % (0), 0), 0, 100)
             self.color_degrade_power = np.clip (  io.input_int ("Degrade color power of final image [0..100] (skip:0) : ", 0), 0, 100)
             self.export_mask_alpha = io.input_bool("Export png with alpha channel of the mask? (y/n skip:n) : ", False)
 
@@ -246,6 +287,8 @@ class ConverterConfigMasked(ConverterConfig):
                    self.motion_blur_power == other.motion_blur_power and \
                    self.output_face_scale == other.output_face_scale and \
                    self.color_transfer_mode == other.color_transfer_mode and \
+                   self.image_denoise_power == other.image_denoise_power and \
+                   self.bicubic_degrade_power == other.bicubic_degrade_power and \
                    self.color_degrade_power == other.color_degrade_power and \
                    self.export_mask_alpha == other.export_mask_alpha
 
@@ -281,7 +324,9 @@ class ConverterConfigMasked(ConverterConfig):
         r += super().to_string(filename)
 
         if 'raw' not in self.mode:
-            r += (f"""color_degrade_power: {self.color_degrade_power}\n"""
+            r += (f"""image_denoise_power: {self.image_denoise_power}\n"""
+                  f"""bicubic_degrade_power: {self.bicubic_degrade_power}\n"""
+                  f"""color_degrade_power: {self.color_degrade_power}\n"""
                   f"""export_mask_alpha: {self.export_mask_alpha}\n""")
 
         r += "================"
@@ -291,12 +336,13 @@ class ConverterConfigMasked(ConverterConfig):
 
 class ConverterConfigFaceAvatar(ConverterConfig):
 
-    def __init__(self, temporal_face_count=0):
+    def __init__(self, temporal_face_count=0,
+                       add_source_image=False):
         super().__init__(type=ConverterConfig.TYPE_FACE_AVATAR)
         self.temporal_face_count = temporal_face_count
 
         #changeable params
-        self.add_source_image = False
+        self.add_source_image = add_source_image
 
     def copy(self):
         return copy.copy(self)

imagelib/color_transfer.py

@@ -33,7 +33,7 @@ def color_transfer_mkl(x0, x1):
     mx1 = np.mean(x1, axis=0)
 
     result = np.dot(x0-mx0, t) + mx1
-    return np.clip ( result.reshape ( (h,w,c) ), 0, 1)
+    return np.clip ( result.reshape ( (h,w,c) ).astype(x0.dtype), 0, 1)
     
 def color_transfer_idt(i0, i1, bins=256, n_rot=20):
     relaxation = 1 / n_rot
@@ -76,7 +76,7 @@ def color_transfer_idt(i0, i1, bins=256, n_rot=20):
         
         d0 = relaxation * np.linalg.solve(r, (d_r - d0r)) + d0
 
-    return np.clip ( d0.T.reshape ( (h,w,c) ), 0, 1)
+    return np.clip ( d0.T.reshape ( (h,w,c) ).astype(i0.dtype) , 0, 1)
 
 def laplacian_matrix(n, m):
     mat_D = scipy.sparse.lil_matrix((m, m))

mainscripts/Converter.py

@@ -104,8 +104,17 @@ class ConvertSubprocessor(Subprocessor):
                         img = cv2.addWeighted(img, 1.0 + (0.5 * amount), blur, -(0.5 * amount), 0)
                         return img
                 elif amount < 0:
-                    blur = cv2.GaussianBlur(img, (kernel_size, kernel_size) , 0)
-                    img = cv2.addWeighted(img, 1.0 - a / 50.0, blur, a /50.0, 0)  
+                    n = -amount
+                    while n > 0:
+
+                        img_blur = cv2.medianBlur(img, 5)
+                        if int(n / 10) != 0:
+                            img = img_blur
+                        else:
+                            pass_power = (n % 10) / 10.0
+                            img = img*(1.0-pass_power)+img_blur*pass_power
+                        n = max(n-10,0)
+
                     return img
                 return img
             self.blursharpen_func = blursharpen_func
@@ -276,10 +285,19 @@ class ConvertSubprocessor(Subprocessor):
 
             if frames_equal:
                 io.log_info ('Using saved session from ' + '/'.join (self.converter_session_filepath.parts[-2:]) )
+
+                for frame in s_frames:
+                    if frame.cfg is not None:
+                        #recreate ConverterConfig class using constructor with get_config() as dict params
+                        #so if any new param will be added, old converter session will work properly
+                        frame.cfg = frame.cfg.__class__( **frame.cfg.get_config() )
+
                 self.frames = s_frames
                 self.frames_idxs = s_frames_idxs
                 self.frames_done_idxs = s_frames_done_idxs
 
+
+
                 if self.model_iter != s_model_iter:
                     #model is more trained, recompute all frames
                     for frame in self.frames:
@@ -367,7 +385,7 @@ class ConvertSubprocessor(Subprocessor):
             io.log_info ("Session is saved to " + '/'.join (self.converter_session_filepath.parts[-2:]) )
 
     cfg_change_keys = ['`','1', '2', '3', '4', '5', '6', '7', '8', '9',
-                                 'q', 'a', 'w', 's', 'e', 'd', 'r', 'f', 't', 'g','y','h','u','j',
+                                 'q', 'a', 'w', 's', 'e', 'd', 'r', 'f', 'y','h','u','j','i','k','o','l','p', ';',':',#'t', 'g',
                                  'z', 'x', 'c', 'v', 'b','n'   ]
     #override
     def on_tick(self):
@@ -447,10 +465,6 @@ class ConvertSubprocessor(Subprocessor):
                                     cfg.add_motion_blur_power(1 if not shift_pressed else 5)
                                 elif chr_key == 'f':
                                     cfg.add_motion_blur_power(-1 if not shift_pressed else -5)
-                                elif chr_key == 't':
-                                    cfg.add_color_degrade_power(1 if not shift_pressed else 5)
-                                elif chr_key == 'g':
-                                    cfg.add_color_degrade_power(-1 if not shift_pressed else -5)
                                 elif chr_key == 'y':
                                     cfg.add_blursharpen_amount(1 if not shift_pressed else 5)
                                 elif chr_key == 'h':
@@ -459,6 +473,21 @@ class ConvertSubprocessor(Subprocessor):
                                     cfg.add_output_face_scale(1 if not shift_pressed else 5)
                                 elif chr_key == 'j':
                                     cfg.add_output_face_scale(-1 if not shift_pressed else -5)
+                                elif chr_key == 'i':
+                                    cfg.add_image_denoise_power(1 if not shift_pressed else 5)
+                                elif chr_key == 'k':
+                                    cfg.add_image_denoise_power(-1 if not shift_pressed else -5)
+                                elif chr_key == 'o':
+                                    cfg.add_bicubic_degrade_power(1 if not shift_pressed else 5)
+                                elif chr_key == 'l':
+                                    cfg.add_bicubic_degrade_power(-1 if not shift_pressed else -5)
+
+                                elif chr_key == 'p':
+                                    cfg.add_color_degrade_power(1 if not shift_pressed else 5)
+                                elif chr_key == ';':
+                                    cfg.add_color_degrade_power(-1)
+                                elif chr_key == ':':
+                                    cfg.add_color_degrade_power(-5)
 
                                 elif chr_key == 'z':
                                     cfg.toggle_masked_hist_match()

models/Model_SAEHD/Model.py

@@ -50,7 +50,7 @@ class SAEHDModel(ModelBase):
         else:
             self.options['archi'] = self.options.get('archi', default_archi)
 
-        default_ae_dims = 256 if 'liae' in self.options['archi'] else 512
+        default_ae_dims = 256
         default_ed_ch_dims = 21
 
         if is_first_run:

nnlib/nnlib.py

@@ -95,6 +95,7 @@ gaussian_blur = nnlib.gaussian_blur
 style_loss = nnlib.style_loss
 dssim = nnlib.dssim
 
+DenseMaxout = nnlib.DenseMaxout
 PixelShuffler = nnlib.PixelShuffler
 SubpixelUpscaler = nnlib.SubpixelUpscaler
 SubpixelDownscaler = nnlib.SubpixelDownscaler
@@ -912,6 +913,133 @@ NLayerDiscriminator = nnlib.NLayerDiscriminator
                 return dict(list(base_config.items()) + list(config.items()))
         nnlib.Adam = Adam
         
+        class DenseMaxout(keras.layers.Layer):
+            """A dense maxout layer.
+            A `MaxoutDense` layer takes the element-wise maximum of
+            `nb_feature` `Dense(input_dim, output_dim)` linear layers.
+            This allows the layer to learn a convex,
+            piecewise linear activation function over the inputs.
+            Note that this is a *linear* layer;
+            if you wish to apply activation function
+            (you shouldn't need to --they are universal function approximators),
+            an `Activation` layer must be added after.
+            # Arguments
+                output_dim: int > 0.
+                nb_feature: number of Dense layers to use internally.
+                init: name of initialization function for the weights of the layer
+                    (see [initializations](../initializations.md)),
+                    or alternatively, Theano function to use for weights
+                    initialization. This parameter is only relevant
+                    if you don't pass a `weights` argument.
+                weights: list of Numpy arrays to set as initial weights.
+                    The list should have 2 elements, of shape `(input_dim, output_dim)`
+                    and (output_dim,) for weights and biases respectively.
+                W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+                    (eg. L1 or L2 regularization), applied to the main weights matrix.
+                b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+                    applied to the bias.
+                activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+                    applied to the network output.
+                W_constraint: instance of the [constraints](../constraints.md) module
+                    (eg. maxnorm, nonneg), applied to the main weights matrix.
+                b_constraint: instance of the [constraints](../constraints.md) module,
+                    applied to the bias.
+                bias: whether to include a bias
+                    (i.e. make the layer affine rather than linear).
+                input_dim: dimensionality of the input (integer). This argument
+                    (or alternatively, the keyword argument `input_shape`)
+                    is required when using this layer as the first layer in a model.
+            # Input shape
+                2D tensor with shape: `(nb_samples, input_dim)`.
+            # Output shape
+                2D tensor with shape: `(nb_samples, output_dim)`.
+            # References
+                - [Maxout Networks](http://arxiv.org/abs/1302.4389)
+            """
+
+            def __init__(self, output_dim,
+                        nb_feature=4,
+                        kernel_initializer='glorot_uniform',
+                        weights=None,
+                        W_regularizer=None,
+                        b_regularizer=None,
+                        activity_regularizer=None,
+                        W_constraint=None,
+                        b_constraint=None,
+                        bias=True,
+                        input_dim=None,
+                        **kwargs):
+                self.output_dim = output_dim
+                self.nb_feature = nb_feature
+                self.kernel_initializer = keras.initializers.get(kernel_initializer)
+
+                self.W_regularizer = keras.regularizers.get(W_regularizer)
+                self.b_regularizer = keras.regularizers.get(b_regularizer)
+                self.activity_regularizer = keras.regularizers.get(activity_regularizer)
+
+                self.W_constraint = keras.constraints.get(W_constraint)
+                self.b_constraint = keras.constraints.get(b_constraint)
+
+                self.bias = bias
+                self.initial_weights = weights
+                self.input_spec = keras.layers.InputSpec(ndim=2)
+
+                self.input_dim = input_dim
+                if self.input_dim:
+                    kwargs['input_shape'] = (self.input_dim,)
+                super(DenseMaxout, self).__init__(**kwargs)
+
+            def build(self, input_shape):
+                input_dim = input_shape[1]
+                self.input_spec = keras.layers.InputSpec(dtype=K.floatx(),
+                                            shape=(None, input_dim))
+
+                self.W = self.add_weight(shape=(self.nb_feature, input_dim, self.output_dim),
+                                        initializer=self.kernel_initializer,
+                                        name='W',
+                                        regularizer=self.W_regularizer,
+                                        constraint=self.W_constraint)
+                if self.bias:
+                    self.b = self.add_weight(shape=(self.nb_feature, self.output_dim,),
+                                            initializer='zero',
+                                            name='b',
+                                            regularizer=self.b_regularizer,
+                                            constraint=self.b_constraint)
+                else:
+                    self.b = None
+
+                if self.initial_weights is not None:
+                    self.set_weights(self.initial_weights)
+                    del self.initial_weights
+                self.built = True
+
+            def compute_output_shape(self, input_shape):
+                assert input_shape and len(input_shape) == 2
+                return (input_shape[0], self.output_dim)
+
+            def call(self, x):
+                # no activation, this layer is only linear.
+                output = K.dot(x, self.W)
+                if self.bias:
+                    output += self.b
+                output = K.max(output, axis=1)
+                return output
+
+            def get_config(self):
+                config = {'output_dim': self.output_dim,
+                        'kernel_initializer': initializers.serialize(self.kernel_initializer),
+                        'nb_feature': self.nb_feature,
+                        'W_regularizer': regularizers.serialize(self.W_regularizer),
+                        'b_regularizer': regularizers.serialize(self.b_regularizer),
+                        'activity_regularizer': regularizers.serialize(self.activity_regularizer),
+                        'W_constraint': constraints.serialize(self.W_constraint),
+                        'b_constraint': constraints.serialize(self.b_constraint),
+                        'bias': self.bias,
+                        'input_dim': self.input_dim}
+                base_config = super(DenseMaxout, self).get_config()
+                return dict(list(base_config.items()) + list(config.items()))
+        nnlib.DenseMaxout = DenseMaxout
+        
         def CAInitializerMP( conv_weights_list ):
             #Convolution Aware Initialization https://arxiv.org/abs/1702.06295
             data = [ (i, K.int_shape(conv_weights)) for i, conv_weights in enumerate(conv_weights_list) ]