From 71ebf06c890f57b8b1c711faea48f228135fb092 Mon Sep 17 00:00:00 2001 From: Colombo Date: Sun, 15 Dec 2019 15:53:06 +0400 Subject: [PATCH] SAEHD,Quick96: improved model generalization, overall accuracy and sharpness by using new 'Learning rate dropout' technique from paper https://arxiv.org/abs/1912.00144 An example of a loss histogram where this function is enabled after the red arrow: https://i.imgur.com/3olskOd.jpg --- models/Model_Quick96/Model.py | 4 ++-- models/Model_SAEHD/Model.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/Model_Quick96/Model.py b/models/Model_Quick96/Model.py index 51d675b..e21957d 100644 --- a/models/Model_Quick96/Model.py +++ b/models/Model_Quick96/Model.py @@ -143,8 +143,8 @@ class Quick96Model(ModelBase): self.CA_conv_weights_list += [layer.weights[0]] #- is Conv2D kernel_weights if self.is_training_mode: - self.src_dst_opt = RMSprop(lr=2e-4) - self.src_dst_mask_opt = RMSprop(lr=2e-4) + self.src_dst_opt = RMSprop(lr=2e-4, lr_dropout=0.3) + self.src_dst_mask_opt = RMSprop(lr=2e-4, lr_dropout=0.3) target_src_masked = self.model.target_src*self.model.target_srcm target_dst_masked = self.model.target_dst*self.model.target_dstm diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 1ee851f..c1750b9 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -452,9 +452,9 @@ class SAEHDModel(ModelBase): psd_target_dst_anti_masked = self.model.pred_src_dst*(1.0 - target_dstm) if self.is_training_mode: - self.src_dst_opt = RMSprop(lr=5e-5, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) - self.src_dst_mask_opt = RMSprop(lr=5e-5, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) - self.D_opt = RMSprop(lr=5e-5, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) + self.src_dst_opt = RMSprop(lr=5e-5, lr_dropout=0.3, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) + self.src_dst_mask_opt = RMSprop(lr=5e-5, lr_dropout=0.3, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) + self.D_opt = RMSprop(lr=5e-5, lr_dropout=0.3, clipnorm=1.0 if self.options['clipgrad'] else 0.0, tf_cpu_mode=self.options['optimizer_mode']-1) src_loss = K.mean ( 10*dssim(kernel_size=int(resolution/11.6),max_value=1.0)( target_src_masked_opt, pred_src_src_masked_opt) ) src_loss += K.mean ( 10*K.square( target_src_masked_opt - pred_src_src_masked_opt ) )