From a366d28adeaf38aede24c657c9ef4becaa10d952 Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 16:36:52 -0700 Subject: [PATCH 01/21] debug shape --- models/Model_SAEHD/Model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 25d6228..b47e339 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -495,6 +495,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_pred_src_src_d, \ gpu_pred_src_src_d2 = self.D_src(gpu_pred_src_src_masked_opt) + print("Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.shape) + print("Shape (gpu_pred_src_src_d2): ", gpu_pred_src_src_d2.shape) + gpu_pred_src_src_d_ones = tf.ones_like (gpu_pred_src_src_d) gpu_pred_src_src_d_zeros = tf.zeros_like(gpu_pred_src_src_d) From 22cb3103d88e443e00ad37b099611941029e617a Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 16:43:03 -0700 Subject: [PATCH 02/21] more debugging --- models/Model_SAEHD/Model.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index b47e339..2bfed55 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -498,6 +498,19 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... print("Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.shape) print("Shape (gpu_pred_src_src_d2): ", gpu_pred_src_src_d2.shape) + print("Flattened Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.flatten().shape()) + + # def get_smooth_noisy_labels_like(label, tensor, smoothing=0.2, noise=0.05): + # new_labels = [] + # for label in labels: + # if np.random.random() < noise: + # label = 0 if label == 1 else 1 + # if label == 0: + # new_labels.append(np.random.uniform(0, 0+smoothing/2)) + # else: + # new_labels.append(np.random.uniform(1-smoothing/2, 1.0)) + # return new_labels + gpu_pred_src_src_d_ones = tf.ones_like (gpu_pred_src_src_d) gpu_pred_src_src_d_zeros = tf.zeros_like(gpu_pred_src_src_d) From 6092a77924f34762b8c4769b753d5a4136862664 Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 16:57:53 -0700 Subject: [PATCH 03/21] more debugging --- models/Model_SAEHD/Model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 2bfed55..5d48762 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -497,8 +497,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... print("Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.shape) print("Shape (gpu_pred_src_src_d2): ", gpu_pred_src_src_d2.shape) + for i in gpu_pred_src_src_d2.shape: + print("Shape (gpu_pred_src_src_d2) i: ", i) - print("Flattened Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.flatten().shape()) # def get_smooth_noisy_labels_like(label, tensor, smoothing=0.2, noise=0.05): # new_labels = [] From 0d5b67a35518c9f61cdd4fccd19089c0fc54642f Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 17:11:31 -0700 Subject: [PATCH 04/21] test --- models/Model_SAEHD/Model.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 5d48762..216f861 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -497,32 +497,30 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... print("Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.shape) print("Shape (gpu_pred_src_src_d2): ", gpu_pred_src_src_d2.shape) - for i in gpu_pred_src_src_d2.shape: - print("Shape (gpu_pred_src_src_d2) i: ", i) - # def get_smooth_noisy_labels_like(label, tensor, smoothing=0.2, noise=0.05): - # new_labels = [] - # for label in labels: - # if np.random.random() < noise: - # label = 0 if label == 1 else 1 - # if label == 0: - # new_labels.append(np.random.uniform(0, 0+smoothing/2)) - # else: - # new_labels.append(np.random.uniform(1-smoothing/2, 1.0)) - # return new_labels + def get_smooth_noisy_label(label, smoothing=0.2, noise=0.05): + if np.random.random() < noise: + label = 0 if label == 1 else 1 + if label == 0: + return lambda x: np.random.uniform(0, 0+smoothing/2) + else: + return lambda x: np.random.uniform(1-smoothing/2, 1.0) - gpu_pred_src_src_d_ones = tf.ones_like (gpu_pred_src_src_d) - gpu_pred_src_src_d_zeros = tf.zeros_like(gpu_pred_src_src_d) + gpu_pred_src_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) + gpu_pred_src_src_d_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) - gpu_pred_src_src_d2_ones = tf.ones_like (gpu_pred_src_src_d2) - gpu_pred_src_src_d2_zeros = tf.zeros_like(gpu_pred_src_src_d2) + print("Shape (gpu_pred_src_src_d_ones): ", gpu_pred_src_src_d.shape) + print("Shape (gpu_pred_src_src_d_zeros): ", gpu_pred_src_src_d2.shape) + + gpu_pred_src_src_d2_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d2) + gpu_pred_src_src_d2_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d2) gpu_target_src_d, \ gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) - gpu_target_src_d_ones = tf.ones_like(gpu_target_src_d) - gpu_target_src_d2_ones = tf.ones_like(gpu_target_src_d2) + gpu_target_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_target_src_d) + gpu_target_src_d2_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_target_src_d2) gpu_D_src_dst_loss = (DLoss(gpu_target_src_d_ones , gpu_target_src_d) + \ DLoss(gpu_pred_src_src_d_zeros , gpu_pred_src_src_d) ) * 0.5 + \ From cca2d86c3f2098afadc373d98b16a3d0805c2bde Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 17:17:38 -0700 Subject: [PATCH 05/21] test --- models/Model_SAEHD/Model.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 216f861..4fa6b72 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -495,8 +495,10 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_pred_src_src_d, \ gpu_pred_src_src_d2 = self.D_src(gpu_pred_src_src_masked_opt) - print("Shape (gpu_pred_src_src_d): ", gpu_pred_src_src_d.shape) - print("Shape (gpu_pred_src_src_d2): ", gpu_pred_src_src_d2.shape) + print("gpu_pred_src_src_d.shape: ", gpu_pred_src_src_d.shape) + print("gpu_pred_src_src_d2.shape: ", gpu_pred_src_src_d2.shape) + + print("gpu_pred_src_src_d.reshape([-1]).shape: ", gpu_pred_src_src_d.reshape([-1]).shape) def get_smooth_noisy_label(label, smoothing=0.2, noise=0.05): @@ -507,7 +509,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... else: return lambda x: np.random.uniform(1-smoothing/2, 1.0) - gpu_pred_src_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) + gpu_pred_src_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d.reshape([-1])).reshape(gpu_pred_src_src_d.shape) gpu_pred_src_src_d_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) print("Shape (gpu_pred_src_src_d_ones): ", gpu_pred_src_src_d.shape) From 08495a199e2821e4641cef3d3a71846d383e9c48 Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 17:19:26 -0700 Subject: [PATCH 06/21] test --- models/Model_SAEHD/Model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 4fa6b72..f879310 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -498,7 +498,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... print("gpu_pred_src_src_d.shape: ", gpu_pred_src_src_d.shape) print("gpu_pred_src_src_d2.shape: ", gpu_pred_src_src_d2.shape) - print("gpu_pred_src_src_d.reshape([-1]).shape: ", gpu_pred_src_src_d.reshape([-1]).shape) + print("tf.reshape(gpu_pred_src_src_d, [-1]).shape: ", tf.reshape(gpu_pred_src_src_d, [-1]).shape) def get_smooth_noisy_label(label, smoothing=0.2, noise=0.05): @@ -509,7 +509,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... else: return lambda x: np.random.uniform(1-smoothing/2, 1.0) - gpu_pred_src_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d.reshape([-1])).reshape(gpu_pred_src_src_d.shape) + gpu_pred_src_src_d_ones = tf.reshape(tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), tf.reshape(gpu_pred_src_src_d, [-1])), gpu_pred_src_src_d.shape) gpu_pred_src_src_d_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) print("Shape (gpu_pred_src_src_d_ones): ", gpu_pred_src_src_d.shape) From 91301d0516ac43816a386b542d0b58d68999b90e Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 19:16:48 -0700 Subject: [PATCH 07/21] update labels --- models/Model_SAEHD/Model.py | 39 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index f879310..991e805 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -498,31 +498,38 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... print("gpu_pred_src_src_d.shape: ", gpu_pred_src_src_d.shape) print("gpu_pred_src_src_d2.shape: ", gpu_pred_src_src_d2.shape) - print("tf.reshape(gpu_pred_src_src_d, [-1]).shape: ", tf.reshape(gpu_pred_src_src_d, [-1]).shape) + # print("tf.reshape(gpu_pred_src_src_d, [-1]).shape: ", tf.reshape(gpu_pred_src_src_d, [-1]).shape) - def get_smooth_noisy_label(label, smoothing=0.2, noise=0.05): - if np.random.random() < noise: - label = 0 if label == 1 else 1 - if label == 0: - return lambda x: np.random.uniform(0, 0+smoothing/2) - else: - return lambda x: np.random.uniform(1-smoothing/2, 1.0) + def get_smooth_noisy_labels(label, tensor, smoothing=0.2, noise=0.05): + labels = [] + num_labels = self.batch_size + for d in tensor.shape[1:]: + num_labels *= d + for _ in num_labels: + if np.random.random() < noise: + label = 0 if label == 1 else 1 + if label == 0: + label = np.random.uniform(0, 0+smoothing/2) + else: + label = np.random.uniform(1-smoothing/2, 1.0) + labels.append(label) + return tf.reshape(labels, tensor.shape) - gpu_pred_src_src_d_ones = tf.reshape(tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), tf.reshape(gpu_pred_src_src_d, [-1])), gpu_pred_src_src_d.shape) - gpu_pred_src_src_d_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d) + gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) + gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) - print("Shape (gpu_pred_src_src_d_ones): ", gpu_pred_src_src_d.shape) - print("Shape (gpu_pred_src_src_d_zeros): ", gpu_pred_src_src_d2.shape) + print("gpu_pred_src_src_d_ones.shape: ", gpu_pred_src_src_d_ones.shape) + print("gpu_pred_src_src_d_zeros.shape: ", gpu_pred_src_src_d_zeros.shape) - gpu_pred_src_src_d2_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_pred_src_src_d2) - gpu_pred_src_src_d2_zeros = tf.map_fn (get_smooth_noisy_label(0, smoothing=0.2, noise=0.05), gpu_pred_src_src_d2) + gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2, smoothing=0.2, noise=0.05) + gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2, smoothing=0.2, noise=0.05) gpu_target_src_d, \ gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) - gpu_target_src_d_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_target_src_d) - gpu_target_src_d2_ones = tf.map_fn (get_smooth_noisy_label(1, smoothing=0.2, noise=0.05), gpu_target_src_d2) + gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d, smoothing=0.2, noise=0.05) + gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2, smoothing=0.2, noise=0.05) gpu_D_src_dst_loss = (DLoss(gpu_target_src_d_ones , gpu_target_src_d) + \ DLoss(gpu_pred_src_src_d_zeros , gpu_pred_src_src_d) ) * 0.5 + \ From 4223ec79864767b129246be3072b439d1282bc17 Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 19:18:22 -0700 Subject: [PATCH 08/21] fix range --- models/Model_SAEHD/Model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 991e805..553e6e8 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -506,7 +506,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... num_labels = self.batch_size for d in tensor.shape[1:]: num_labels *= d - for _ in num_labels: + for _ in range(num_labels): if np.random.random() < noise: label = 0 if label == 1 else 1 if label == 0: From a89cf63e9f7adc1c380fc473ebaa6d9ced5a6a9c Mon Sep 17 00:00:00 2001 From: jh Date: Tue, 16 Mar 2021 19:20:24 -0700 Subject: [PATCH 09/21] fix reshape --- models/Model_SAEHD/Model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 553e6e8..7cd65a6 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -514,7 +514,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... else: label = np.random.uniform(1-smoothing/2, 1.0) labels.append(label) - return tf.reshape(labels, tensor.shape) + return tf.reshape(labels, (self.batch_size,) + tensor.shape[1:]) gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) From 3e7fb39cada1fce5a96a7743ebc4f5022959cc22 Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 08:26:00 -0700 Subject: [PATCH 10/21] Vectorize --- models/Model_SAEHD/Model.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 7cd65a6..95445ac 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -502,19 +502,15 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... def get_smooth_noisy_labels(label, tensor, smoothing=0.2, noise=0.05): - labels = [] num_labels = self.batch_size for d in tensor.shape[1:]: num_labels *= d - for _ in range(num_labels): - if np.random.random() < noise: - label = 0 if label == 1 else 1 - if label == 0: - label = np.random.uniform(0, 0+smoothing/2) - else: - label = np.random.uniform(1-smoothing/2, 1.0) - labels.append(label) - return tf.reshape(labels, (self.batch_size,) + tensor.shape[1:]) + probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) + x = tf.random.categorical(probs, num_labels) + x = tf.cast(x, tf.float32) + x = x * (1-smoothing) + (smoothing/x.shape[1]) + x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) + return x gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) From 3edcd9a173fc5bda2c62077a4246829551c402bb Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 08:30:39 -0700 Subject: [PATCH 11/21] fix error --- models/Model_SAEHD/Model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 95445ac..f8dfbe9 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -508,7 +508,8 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) - x = x * (1-smoothing) + (smoothing/x.shape[1]) + x = x * (1-smoothing) + (smoothing/num_labels) + # x = tf.math.scalar_mul(1-smoothing, x) + (smoothing/x.shape[1]) x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) return x From faf2a4a88544f79f5949d1c1c3cade6f81e1dfdd Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 08:34:00 -0700 Subject: [PATCH 12/21] fix error --- models/Model_SAEHD/Model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index f8dfbe9..3dfef64 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -508,8 +508,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) - x = x * (1-smoothing) + (smoothing/num_labels) - # x = tf.math.scalar_mul(1-smoothing, x) + (smoothing/x.shape[1]) + # x = x * (1-smoothing) + (smoothing/num_labels) + x = tf.math.scalar_mul(1-smoothing, x) + x = x + (smoothing/num_labels) x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) return x From 86c7618c071bfd3d0e70ef1c1f1a245f5ed5b75f Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 08:37:33 -0700 Subject: [PATCH 13/21] fix num_labels type --- models/Model_SAEHD/Model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 3dfef64..ff5584a 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -503,8 +503,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... def get_smooth_noisy_labels(label, tensor, smoothing=0.2, noise=0.05): num_labels = self.batch_size - for d in tensor.shape[1:]: + for d in tensor.get_shape().as_list()[1:]: num_labels *= d + # num_labels = int(num_labels) probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) From 7696a30d84133951614cde87f9abe30e631d12c4 Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 08:44:45 -0700 Subject: [PATCH 14/21] clean up --- models/Model_SAEHD/Model.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index ff5584a..81368b1 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -495,13 +495,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gpu_pred_src_src_d, \ gpu_pred_src_src_d2 = self.D_src(gpu_pred_src_src_masked_opt) - print("gpu_pred_src_src_d.shape: ", gpu_pred_src_src_d.shape) - print("gpu_pred_src_src_d2.shape: ", gpu_pred_src_src_d2.shape) - - # print("tf.reshape(gpu_pred_src_src_d, [-1]).shape: ", tf.reshape(gpu_pred_src_src_d, [-1]).shape) - - - def get_smooth_noisy_labels(label, tensor, smoothing=0.2, noise=0.05): + def get_smooth_noisy_labels(label, tensor, smoothing=0.1, noise=0.05): num_labels = self.batch_size for d in tensor.get_shape().as_list()[1:]: num_labels *= d @@ -515,20 +509,17 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) return x - gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) - gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=0.2, noise=0.05) + gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d) + gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d) - print("gpu_pred_src_src_d_ones.shape: ", gpu_pred_src_src_d_ones.shape) - print("gpu_pred_src_src_d_zeros.shape: ", gpu_pred_src_src_d_zeros.shape) - - gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2, smoothing=0.2, noise=0.05) - gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2, smoothing=0.2, noise=0.05) + gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2) + gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2) gpu_target_src_d, \ gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) - gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d, smoothing=0.2, noise=0.05) - gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2, smoothing=0.2, noise=0.05) + gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d) + gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2) gpu_D_src_dst_loss = (DLoss(gpu_target_src_d_ones , gpu_target_src_d) + \ DLoss(gpu_pred_src_src_d_zeros , gpu_pred_src_src_d) ) * 0.5 + \ From d2b44fc2ca9a94deaab9e084c5b1d300b19a7c0a Mon Sep 17 00:00:00 2001 From: jh Date: Wed, 17 Mar 2021 09:06:49 -0700 Subject: [PATCH 15/21] Add GAN options --- models/Model_SAEHD/Model.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 81368b1..28e14f1 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -139,6 +139,8 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... default_gan_power = self.options['gan_power'] = self.load_or_def_option('gan_power', 0.0) default_gan_patch_size = self.options['gan_patch_size'] = self.load_or_def_option('gan_patch_size', self.options['resolution'] // 8) default_gan_dims = self.options['gan_dims'] = self.load_or_def_option('gan_dims', 16) + default_gan_smoothing = self.options['gan_smoothing'] = self.load_or_def_option('gan_smoothing', 0.1) + default_gan_noise = self.options['gan_noise'] = self.load_or_def_option('gan_noise', 0.05) if self.is_first_run() or ask_override: self.options['models_opt_on_gpu'] = io.input_bool ("Place models and optimizer on GPU", default_models_opt_on_gpu, help_message="When you train on one GPU, by default model and optimizer weights are placed on GPU to accelerate the process. You can place they on CPU to free up extra VRAM, thus set bigger dimensions.") @@ -158,6 +160,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) self.options['gan_dims'] = gan_dims + self.options['gan_smoothing'] = np.clip ( io.input_number("GAN label smoothing", default_gan_smoothing, add_info="0 - 0.5", help_message="Uses soft labels with values slightly off from 0/1 for GAN, has a regularizing effect"), 0, 0.5) + self.options['gan_noise'] = np.clip ( io.input_number("GAN noisy labels", default_gan_noise, add_info="0 - 0.5", help_message="Marks some images with the wrong label, helps prevent collapse"), 0, 0.5) + if 'df' in self.options['archi']: self.options['true_face_power'] = np.clip ( io.input_number ("'True face' power.", default_true_face_power, add_info="0.0000 .. 1.0", help_message="Experimental option. Discriminates result face to be more like src face. Higher value - stronger discrimination. Typical value is 0.01 . Comparison - https://i.imgur.com/czScS9q.png"), 0.0, 1.0 ) else: @@ -499,27 +504,29 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... num_labels = self.batch_size for d in tensor.get_shape().as_list()[1:]: num_labels *= d - # num_labels = int(num_labels) + probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) - # x = x * (1-smoothing) + (smoothing/num_labels) x = tf.math.scalar_mul(1-smoothing, x) x = x + (smoothing/num_labels) x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) return x - gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d) - gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d) + smoothing = self.options['gan_smoothing'] + noise = self.options['gan_noise'] - gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2) - gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2) + gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=smoothing, noise=noise) + gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=smoothing, noise=noise) + + gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2, smoothing=smoothing, noise=noise) + gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2, smoothing=smoothing, noise=noise) gpu_target_src_d, \ gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) - gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d) - gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2) + gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d, smoothing=smoothing, noise=noise) + gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2, smoothing=smoothing, noise=noise) gpu_D_src_dst_loss = (DLoss(gpu_target_src_d_ones , gpu_target_src_d) + \ DLoss(gpu_pred_src_src_d_zeros , gpu_pred_src_src_d) ) * 0.5 + \ From 7bc5bf2b949ff62b04d077fb3dcde54270f3f6a1 Mon Sep 17 00:00:00 2001 From: jh Date: Sun, 21 Mar 2021 23:13:32 -0700 Subject: [PATCH 16/21] feature: adds GANv2 --- core/leras/models/PatchDiscriminator.py | 126 ++++++++++++++++++++++-- models/Model_SAEHD/Model.py | 23 ++++- 2 files changed, 138 insertions(+), 11 deletions(-) diff --git a/core/leras/models/PatchDiscriminator.py b/core/leras/models/PatchDiscriminator.py index 343e000..dcaf941 100644 --- a/core/leras/models/PatchDiscriminator.py +++ b/core/leras/models/PatchDiscriminator.py @@ -111,7 +111,7 @@ class UNetPatchDiscriminator(nn.ModelBase): for i in range(layers_count-1): st = 1 + (1 if val & (1 << i) !=0 else 0 ) layers.append ( [3, st ]) - sum_st += st + sum_st += st rf = self.calc_receptive_field_size(layers) @@ -131,7 +131,7 @@ class UNetPatchDiscriminator(nn.ModelBase): return s[q][2] def on_build(self, patch_size, in_ch, base_ch = 16): - + class ResidualBlock(nn.ModelBase): def on_build(self, ch, kernel_size=3 ): self.conv1 = nn.Conv2D( ch, ch, kernel_size=kernel_size, padding='SAME') @@ -152,7 +152,7 @@ class UNetPatchDiscriminator(nn.ModelBase): self.upres1 = [] self.upres2 = [] layers = self.find_archi(patch_size) - + level_chs = { i-1:v for i,v in enumerate([ min( base_ch * (2**i), 512 ) for i in range(len(layers)+1)]) } self.in_conv = nn.Conv2D( in_ch, level_chs[-1], kernel_size=1, padding='VALID') @@ -162,12 +162,12 @@ class UNetPatchDiscriminator(nn.ModelBase): self.res1.append ( ResidualBlock(level_chs[i]) ) self.res2.append ( ResidualBlock(level_chs[i]) ) - + self.upconvs.insert (0, nn.Conv2DTranspose( level_chs[i]*(2 if i != len(layers)-1 else 1), level_chs[i-1], kernel_size=kernel_size, strides=strides, padding='SAME') ) self.upres1.insert (0, ResidualBlock(level_chs[i-1]*2) ) self.upres2.insert (0, ResidualBlock(level_chs[i-1]*2) ) - + self.out_conv = nn.Conv2D( level_chs[-1]*2, 1, kernel_size=1, padding='VALID') self.center_out = nn.Conv2D( level_chs[len(layers)-1], 1, kernel_size=1, padding='VALID') @@ -183,7 +183,7 @@ class UNetPatchDiscriminator(nn.ModelBase): x = tf.nn.leaky_relu( conv(x), 0.2 ) x = res1(x) x = res2(x) - + center_out, x = self.center_out(x), tf.nn.leaky_relu( self.center_conv(x), 0.2 ) for i, (upconv, enc, upres1, upres2 ) in enumerate(zip(self.upconvs, encs, self.upres1, self.upres2)): @@ -195,3 +195,117 @@ class UNetPatchDiscriminator(nn.ModelBase): return center_out, self.out_conv(x) nn.UNetPatchDiscriminator = UNetPatchDiscriminator + +class UNetPatchDiscriminatorV2(nn.ModelBase): + """ + Inspired by https://arxiv.org/abs/2002.12655 "A U-Net Based Discriminator for Generative Adversarial Networks" + """ + def calc_receptive_field_size(self, layers): + """ + result the same as https://fomoro.com/research/article/receptive-field-calculatorindex.html + """ + rf = 0 + ts = 1 + for i, (k, s) in enumerate(layers): + if i == 0: + rf = k + else: + rf += (k-1)*ts + ts *= s + return rf + + def find_archi(self, target_patch_size, max_layers=6): + """ + Find the best configuration of layers using only 3x3 convs for target patch size + """ + s = {} + for layers_count in range(1,max_layers+1): + val = 1 << (layers_count-1) + while True: + val -= 1 + + layers = [] + sum_st = 0 + for i in range(layers_count-1): + st = 1 + (1 if val & (1 << i) !=0 else 0 ) + layers.append ( [3, st ]) + sum_st += st + layers.append ( [3, 2]) + sum_st += 2 + + rf = self.calc_receptive_field_size(layers) + + s_rf = s.get(rf, None) + if s_rf is None: + s[rf] = (layers_count, sum_st, layers) + else: + if layers_count < s_rf[0] or \ + ( layers_count == s_rf[0] and sum_st > s_rf[1] ): + s[rf] = (layers_count, sum_st, layers) + + if val == 0: + break + + x = sorted(list(s.keys())) + q=x[np.abs(np.array(x)-target_patch_size).argmin()] + return s[q][2] + + def on_build(self, patch_size, in_ch): + class ResidualBlock(nn.ModelBase): + def on_build(self, ch, kernel_size=3 ): + self.conv1 = nn.Conv2D( ch, ch, kernel_size=kernel_size, padding='SAME') + self.conv2 = nn.Conv2D( ch, ch, kernel_size=kernel_size, padding='SAME') + + def forward(self, inp): + x = self.conv1(inp) + x = tf.nn.leaky_relu(x, 0.2) + x = self.conv2(x) + x = tf.nn.leaky_relu(inp + x, 0.2) + return x + + prev_ch = in_ch + self.convs = [] + self.res = [] + self.upconvs = [] + self.upres = [] + layers = self.find_archi(patch_size) + base_ch = 16 + + level_chs = { i-1:v for i,v in enumerate([ min( base_ch * (2**i), 512 ) for i in range(len(layers)+1)]) } + + self.in_conv = nn.Conv2D( in_ch, level_chs[-1], kernel_size=1, padding='VALID') + + for i, (kernel_size, strides) in enumerate(layers): + self.convs.append ( nn.Conv2D( level_chs[i-1], level_chs[i], kernel_size=kernel_size, strides=strides, padding='SAME') ) + + self.res.append ( ResidualBlock(level_chs[i]) ) + + self.upconvs.insert (0, nn.Conv2DTranspose( level_chs[i]*(2 if i != len(layers)-1 else 1), level_chs[i-1], kernel_size=kernel_size, strides=strides, padding='SAME') ) + + self.upres.insert (0, ResidualBlock(level_chs[i-1]*2) ) + + self.out_conv = nn.Conv2D( level_chs[-1]*2, 1, kernel_size=1, padding='VALID') + + self.center_out = nn.Conv2D( level_chs[len(layers)-1], 1, kernel_size=1, padding='VALID') + self.center_conv = nn.Conv2D( level_chs[len(layers)-1], level_chs[len(layers)-1], kernel_size=1, padding='VALID') + + + def forward(self, x): + x = tf.nn.leaky_relu( self.in_conv(x), 0.1 ) + + encs = [] + for conv, res in zip(self.convs, self.res): + encs.insert(0, x) + x = tf.nn.leaky_relu( conv(x), 0.1 ) + x = res(x) + + center_out, x = self.center_out(x), self.center_conv(x) + + for i, (upconv, enc, upres) in enumerate(zip(self.upconvs, encs, self.upres)): + x = tf.nn.leaky_relu( upconv(x), 0.1 ) + x = tf.concat( [enc, x], axis=nn.conv2d_ch_axis) + x = upres(x) + + return center_out, self.out_conv(x) + +nn.UNetPatchDiscriminatorV2 = UNetPatchDiscriminatorV2 diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 476d0f0..afeff57 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -138,6 +138,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... self.options['uniform_yaw'] = io.input_bool ("Uniform yaw distribution of samples", default_uniform_yaw, help_message='Helps to fix blurry side faces due to small amount of them in the faceset.') + default_gan_version = self.options['gan_version'] = self.load_or_def_option('gan_version', 2) default_gan_power = self.options['gan_power'] = self.load_or_def_option('gan_power', 0.0) default_gan_patch_size = self.options['gan_patch_size'] = self.load_or_def_option('gan_patch_size', self.options['resolution'] // 8) default_gan_dims = self.options['gan_dims'] = self.load_or_def_option('gan_dims', 16) @@ -154,6 +155,9 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 1.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) if self.options['gan_power'] != 0.0: + gan_version = np.clip (io.input_int("GAN version", default_gan_version, add_info="2 or 3", help_message="Choose GAN version (v2: 7/16/2020, v3: 1/3/2021):"), 2, 3) + self.options['gan_version'] = gan_version + gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) self.options['gan_patch_size'] = gan_patch_size @@ -299,8 +303,12 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... if self.is_training: if gan_power != 0: - self.D_src = nn.UNetPatchDiscriminator(patch_size=self.options['gan_patch_size'], in_ch=input_ch, base_ch=self.options['gan_dims'], name="D_src") - self.model_filename_list += [ [self.D_src, 'GAN.npy'] ] + if self.options['gan_version'] == 2: + self.D_src = nn.UNetPatchDiscriminatorV2(patch_size=resolution//16, in_ch=input_ch, name="D_src") + self.model_filename_list += [ [self.D_src, 'D_src_v2.npy'] ] + else: + self.D_src = nn.UNetPatchDiscriminator(patch_size=self.options['gan_patch_size'], in_ch=input_ch, base_ch=self.options['gan_dims'], name="D_src") + self.model_filename_list += [ [self.D_src, 'GAN.npy'] ] # Initialize optimizers lr=5e-5 @@ -325,9 +333,14 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... self.model_filename_list += [ (self.D_code_opt, 'D_code_opt.npy') ] if gan_power != 0: - self.D_src_dst_opt = OptimizerClass(lr=lr, lr_dropout=lr_dropout, clipnorm=clipnorm, name='GAN_opt') - self.D_src_dst_opt.initialize_variables ( self.D_src.get_weights(), vars_on_cpu=optimizer_vars_on_cpu, lr_dropout_on_cpu=self.options['lr_dropout']=='cpu')#+self.D_src_x2.get_weights() - self.model_filename_list += [ (self.D_src_dst_opt, 'GAN_opt.npy') ] + if self.options['gan_version'] == 2: + self.D_src_dst_opt = OptimizerClass(lr=lr, lr_dropout=lr_dropout, clipnorm=clipnorm, name='D_src_dst_opt') + self.D_src_dst_opt.initialize_variables ( self.D_src.get_weights(), vars_on_cpu=optimizer_vars_on_cpu, lr_dropout_on_cpu=self.options['lr_dropout']=='cpu')#+self.D_src_x2.get_weights() + self.model_filename_list += [ (self.D_src_dst_opt, 'D_src_v2_opt.npy') ] + else: + self.D_src_dst_opt = OptimizerClass(lr=lr, lr_dropout=lr_dropout, clipnorm=clipnorm, name='GAN_opt') + self.D_src_dst_opt.initialize_variables ( self.D_src.get_weights(), vars_on_cpu=optimizer_vars_on_cpu, lr_dropout_on_cpu=self.options['lr_dropout']=='cpu')#+self.D_src_x2.get_weights() + self.model_filename_list += [ (self.D_src_dst_opt, 'GAN_opt.npy') ] if self.is_training: # Adjust batch size for multiple GPU From ba1a1b2b912cc543bc6e94fb970e872e1dd307c1 Mon Sep 17 00:00:00 2001 From: jh Date: Sun, 21 Mar 2021 23:46:13 -0700 Subject: [PATCH 17/21] fix: check gan version for certain gan settings --- models/Model_SAEHD/Model.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index afeff57..5ebc4e3 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -157,12 +157,13 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... if self.options['gan_power'] != 0.0: gan_version = np.clip (io.input_int("GAN version", default_gan_version, add_info="2 or 3", help_message="Choose GAN version (v2: 7/16/2020, v3: 1/3/2021):"), 2, 3) self.options['gan_version'] = gan_version + + if self.options['gan_version'] == 3: + gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) + self.options['gan_patch_size'] = gan_patch_size - gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) - self.options['gan_patch_size'] = gan_patch_size - - gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) - self.options['gan_dims'] = gan_dims + gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) + self.options['gan_dims'] = gan_dims if 'df' in self.options['archi']: self.options['true_face_power'] = np.clip ( io.input_number ("'True face' power.", default_true_face_power, add_info="0.0000 .. 1.0", help_message="Experimental option. Discriminates result face to be more like src face. Higher value - stronger discrimination. Typical value is 0.01 . Comparison - https://i.imgur.com/czScS9q.png"), 0.0, 1.0 ) From 4462e376215754bec063c7982dbf3293525ecd0b Mon Sep 17 00:00:00 2001 From: jh Date: Sun, 21 Mar 2021 23:50:38 -0700 Subject: [PATCH 18/21] fix gan power --- models/Model_SAEHD/Model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 5ebc4e3..f061710 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -152,12 +152,12 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... self.options['random_warp'] = io.input_bool ("Enable random warp of samples", default_random_warp, help_message="Random warp is required to generalize facial expressions of both faces. When the face is trained enough, you can disable it to get extra sharpness and reduce subpixel shake for less amount of iterations.") - self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 1.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) + self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 10.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) if self.options['gan_power'] != 0.0: gan_version = np.clip (io.input_int("GAN version", default_gan_version, add_info="2 or 3", help_message="Choose GAN version (v2: 7/16/2020, v3: 1/3/2021):"), 2, 3) self.options['gan_version'] = gan_version - + if self.options['gan_version'] == 3: gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) self.options['gan_patch_size'] = gan_patch_size From f989d0aac6b688dacffc7efada5ac7fe014f358b Mon Sep 17 00:00:00 2001 From: jh Date: Mon, 22 Mar 2021 08:54:15 -0700 Subject: [PATCH 19/21] feat: ask GAN version prior to power --- models/Model_SAEHD/Model.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index f061710..7d382d4 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -152,18 +152,19 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... self.options['random_warp'] = io.input_bool ("Enable random warp of samples", default_random_warp, help_message="Random warp is required to generalize facial expressions of both faces. When the face is trained enough, you can disable it to get extra sharpness and reduce subpixel shake for less amount of iterations.") - self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 10.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) + self.options['gan_version'] = np.clip (io.input_int("GAN version", default_gan_version, add_info="2 or 3", help_message="Choose GAN version (v2: 7/16/2020, v3: 1/3/2021):"), 2, 3) - if self.options['gan_power'] != 0.0: - gan_version = np.clip (io.input_int("GAN version", default_gan_version, add_info="2 or 3", help_message="Choose GAN version (v2: 7/16/2020, v3: 1/3/2021):"), 2, 3) - self.options['gan_version'] = gan_version + if self.options['gan_version'] == 2: + self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 10.0", help_message="Train the network in Generative Adversarial manner. Forces the neural network to learn small details of the face. Enable it only when the face is trained enough and don't disable. Typical value is 0.1"), 0.0, 10.0 ) + else: + self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 1.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) - if self.options['gan_version'] == 3: - gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) - self.options['gan_patch_size'] = gan_patch_size + if self.options['gan_power'] != 0.0 and self.options['gan_version'] == 3: + gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) + self.options['gan_patch_size'] = gan_patch_size - gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) - self.options['gan_dims'] = gan_dims + gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) + self.options['gan_dims'] = gan_dims if 'df' in self.options['archi']: self.options['true_face_power'] = np.clip ( io.input_number ("'True face' power.", default_true_face_power, add_info="0.0000 .. 1.0", help_message="Experimental option. Discriminates result face to be more like src face. Higher value - stronger discrimination. Typical value is 0.01 . Comparison - https://i.imgur.com/czScS9q.png"), 0.0, 1.0 ) From f3f28079728b56f788f81fee9ecab2bba6df9f39 Mon Sep 17 00:00:00 2001 From: jh Date: Mon, 22 Mar 2021 12:00:44 -0700 Subject: [PATCH 20/21] Fix gan options --- models/Model_SAEHD/Model.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index 791e051..f19cbfe 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -161,13 +161,14 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... else: self.options['gan_power'] = np.clip ( io.input_number ("GAN power", default_gan_power, add_info="0.0 .. 1.0", help_message="Forces the neural network to learn small details of the face. Enable it only when the face is trained enough with lr_dropout(on) and random_warp(off), and don't disable. The higher the value, the higher the chances of artifacts. Typical fine value is 0.1"), 0.0, 1.0 ) - if self.options['gan_power'] != 0.0 and self.options['gan_version'] == 3: - gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) - self.options['gan_patch_size'] = gan_patch_size - - gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) - self.options['gan_dims'] = gan_dims + if self.options['gan_power'] != 0.0: + if self.options['gan_version'] == 3: + gan_patch_size = np.clip ( io.input_int("GAN patch size", default_gan_patch_size, add_info="3-640", help_message="The higher patch size, the higher the quality, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is resolution / 8." ), 3, 640 ) + self.options['gan_patch_size'] = gan_patch_size + gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) + self.options['gan_dims'] = gan_dims + self.options['gan_smoothing'] = np.clip ( io.input_number("GAN label smoothing", default_gan_smoothing, add_info="0 - 0.5", help_message="Uses soft labels with values slightly off from 0/1 for GAN, has a regularizing effect"), 0, 0.5) self.options['gan_noise'] = np.clip ( io.input_number("GAN noisy labels", default_gan_noise, add_info="0 - 0.5", help_message="Marks some images with the wrong label, helps prevent collapse"), 0, 0.5) @@ -545,7 +546,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... num_labels = self.batch_size for d in tensor.get_shape().as_list()[1:]: num_labels *= d - + probs = tf.math.log([[noise, 1-noise]]) if label == 1 else tf.math.log([[1-noise, noise]]) x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) From c36eca36105623f16906739e5700bc1b05eae587 Mon Sep 17 00:00:00 2001 From: jh Date: Mon, 22 Mar 2021 15:22:28 -0700 Subject: [PATCH 21/21] feat: use one-sided smoothing (eg: positive/real labels 1.0 -> 0.9), and only apply to discriminator --- models/Model_SAEHD/Model.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/models/Model_SAEHD/Model.py b/models/Model_SAEHD/Model.py index f19cbfe..5522891 100644 --- a/models/Model_SAEHD/Model.py +++ b/models/Model_SAEHD/Model.py @@ -168,7 +168,7 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... gan_dims = np.clip ( io.input_int("GAN dimensions", default_gan_dims, add_info="4-64", help_message="The dimensions of the GAN network. The higher dimensions, the more VRAM is required. You can get sharper edges even at the lowest setting. Typical fine value is 16." ), 4, 64 ) self.options['gan_dims'] = gan_dims - + self.options['gan_smoothing'] = np.clip ( io.input_number("GAN label smoothing", default_gan_smoothing, add_info="0 - 0.5", help_message="Uses soft labels with values slightly off from 0/1 for GAN, has a regularizing effect"), 0, 0.5) self.options['gan_noise'] = np.clip ( io.input_number("GAN noisy labels", default_gan_noise, add_info="0 - 0.5", help_message="Marks some images with the wrong label, helps prevent collapse"), 0, 0.5) @@ -551,29 +551,28 @@ Examples: df, liae, df-d, df-ud, liae-ud, ... x = tf.random.categorical(probs, num_labels) x = tf.cast(x, tf.float32) x = tf.math.scalar_mul(1-smoothing, x) - x = x + (smoothing/num_labels) + # x = x + (smoothing/num_labels) x = tf.reshape(x, (self.batch_size,) + tensor.shape[1:]) return x smoothing = self.options['gan_smoothing'] noise = self.options['gan_noise'] - gpu_pred_src_src_d_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d, smoothing=smoothing, noise=noise) - gpu_pred_src_src_d_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=smoothing, noise=noise) + gpu_pred_src_src_d_ones = tf.ones_like(gpu_pred_src_src_d) + gpu_pred_src_src_d2_ones = tf.ones_like(gpu_pred_src_src_d2) - gpu_pred_src_src_d2_ones = get_smooth_noisy_labels(1, gpu_pred_src_src_d2, smoothing=smoothing, noise=noise) - gpu_pred_src_src_d2_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2, smoothing=smoothing, noise=noise) + gpu_pred_src_src_d_smooth_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d, smoothing=smoothing, noise=noise) + gpu_pred_src_src_d2_smooth_zeros = get_smooth_noisy_labels(0, gpu_pred_src_src_d2, smoothing=smoothing, noise=noise) - gpu_target_src_d, \ - gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) + gpu_target_src_d, gpu_target_src_d2 = self.D_src(gpu_target_src_masked_opt) - gpu_target_src_d_ones = get_smooth_noisy_labels(1, gpu_target_src_d, smoothing=smoothing, noise=noise) - gpu_target_src_d2_ones = get_smooth_noisy_labels(1, gpu_target_src_d2, smoothing=smoothing, noise=noise) + gpu_target_src_d_smooth_ones = get_smooth_noisy_labels(1, gpu_target_src_d, smoothing=smoothing, noise=noise) + gpu_target_src_d2_smooth_ones = get_smooth_noisy_labels(1, gpu_target_src_d2, smoothing=smoothing, noise=noise) - gpu_D_src_dst_loss = (DLoss(gpu_target_src_d_ones , gpu_target_src_d) + \ - DLoss(gpu_pred_src_src_d_zeros , gpu_pred_src_src_d) ) * 0.5 + \ - (DLoss(gpu_target_src_d2_ones , gpu_target_src_d2) + \ - DLoss(gpu_pred_src_src_d2_zeros , gpu_pred_src_src_d2) ) * 0.5 + gpu_D_src_dst_loss = DLoss(gpu_target_src_d_smooth_ones, gpu_target_src_d) \ + + DLoss(gpu_pred_src_src_d_smooth_zeros, gpu_pred_src_src_d) \ + + DLoss(gpu_target_src_d2_smooth_ones, gpu_target_src_d2) \ + + DLoss(gpu_pred_src_src_d2_smooth_zeros, gpu_pred_src_src_d2) gpu_D_src_dst_loss_gvs += [ nn.gradients (gpu_D_src_dst_loss, self.D_src.get_weights() ) ]#+self.D_src_x2.get_weights()