From c4e68ef539aaf77f3864d2fe5401ac6ddf3590db Mon Sep 17 00:00:00 2001 From: Auroir <36361058+Auroir@users.noreply.github.com> Date: Fri, 16 Aug 2019 07:35:27 -0700 Subject: [PATCH 1/6] Formatted Model Summary (#348) * Formatted Model Summary Aligns the model summary output using f-string formatting. The logic structure of the base class has not been changed, only the lines put into `model_summary_text`. Output width is calculated from keys & values and will scale to show a clean summary for any model/platform. GPU VRAM has been added as an output. Incorrect detection of VRAM is possible in broken environments and GPUs of different sizes can report the same name. Showing it here adds clarity for the user and for issue tickets. Concatenation changed from "\r\n" to "\n", CRLF end of lines for Windows are handled transparently so using it here caused extra blank lines in the summary txt file. **Examples:** Using CUDA + SAE-LIAE ``` ============= Model Summary ============== == == == Model name: SAE == == == == Current iteration: 16 == == == ==----------- Model Options ------------== == == == batch_size: 4 == == sort_by_yaw: False == == random_flip: True == == resolution: 128 == == face_type: f == == learn_mask: True == == optimizer_mode: 1 == == archi: liae == == ae_dims: 256 == == e_ch_dims: 42 == == d_ch_dims: 21 == == multiscale_decoder: False == == ca_weights: False == == pixel_loss: False == == face_style_power: 0.0 == == bg_style_power: 0.0 == == apply_random_ct: False == == clipgrad: False == == == ==------------- Running On -------------== == == == Device index: 0 == == Name: GeForce GTX 1080 == == VRAM: 8.00GB == == == ========================================== ``` Colab ``` ========== Model Summary ========== == == == Model name: SAE == == == == Current iteration: 39822 == == == ==-------- Model Options --------== == == == batch_size: 24 == == sort_by_yaw: True == == random_flip: False == == resolution: 128 == == face_type: f == == learn_mask: True == == optimizer_mode: 2 == == archi: liae == == ae_dims: 222 == == e_ch_dims: 34 == == d_ch_dims: 16 == == multiscale_decoder: True == == ca_weights: True == == pixel_loss: False == == face_style_power: 2.0 == == bg_style_power: 1.5 == == apply_random_ct: False == == clipgrad: True == == == ==--------- Running On ----------== == == == Device index: 0 == == Name: Tesla K80 == == VRAM: 11.00GB == == == =================================== ``` Using OpenCL + H128 ``` =========================== Model Summary =========================== == == == Model name: H128 == == == == Current iteration: 0 == == == ==------------------------- Model Options -------------------------== == == == batch_size: 4 == == sort_by_yaw: False == == random_flip: True == == lighter_ae: False == == pixel_loss: False == == == ==-------------------------- Running On ---------------------------== == == == Device index: 0 == == Name: Advanced Micro Devices, Inc. gfx900 (OpenCL) == == VRAM: 7.98GB == == == ===================================================================== ``` Using CPU (output trimmed) ``` ==------- Running On --------== == == == Using device: CPU == == == =============================== ``` multi_gpu support is retained (output trimmed) ``` ==------------- Running On -------------== == == == Using multi_gpu: True == == == == Device index: 1 == == Name: Geforce GTX 1080 == == VRAM: 8.00GB == == Device index: 2 == == Name: Geforce GTX 1080 == == VRAM: 8.00GB == == == ========================================== ``` Low VRAM warning (output trimmed) ``` ==------------- Running On -------------== == == == Device index: 0 == == Name: Geforce GTX 1050 == == VRAM: 2.00GB == == == ========================================== /!\ /!\ WARNING: /!\ You are using a GPU with 2GB or less VRAM. This may significantly reduce the quality of your result! /!\ If training does not start, close all programs and try again. /!\ Also you can disable Windows Aero Desktop to increase available VRAM. /!\ ``` * Fix indent --- models/ModelBase.py | 66 ++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/models/ModelBase.py b/models/ModelBase.py index f841ca6..e7ad843 100644 --- a/models/ModelBase.py +++ b/models/ModelBase.py @@ -231,36 +231,54 @@ class ModelBase(object): else: self.sample_for_preview = self.generate_next_sample() self.last_sample = self.sample_for_preview + + ###Generate text summary of model hyperparameters + #Find the longest key name and value string. Used as column widths. + width_name = max([len(k) for k in self.options.keys()] + [17]) + 1 # Single space buffer to left edge. Minimum of 17, the length of the longest static string used "Current iteration" + width_value = max([len(str(x)) for x in self.options.values()] + [len(str(self.iter)), len(self.get_model_name())]) + 1 # Single space buffer to right edge + if not self.device_config.cpu_only: #Check length of GPU names + width_value = max([len(nnlib.device.getDeviceName(idx))+1 for idx in self.device_config.gpu_idxs] + [width_value]) + width_total = width_name + width_value + 2 #Plus 2 for ": " + model_summary_text = [] - - model_summary_text += ["===== Model summary ====="] - model_summary_text += ["== Model name: " + self.get_model_name()] - model_summary_text += ["=="] - model_summary_text += ["== Current iteration: " + str(self.iter)] - model_summary_text += ["=="] - model_summary_text += ["== Model options:"] + model_summary_text += [f'=={" Model Summary ":=^{width_total}}=='] # Model/status summary + model_summary_text += [f'=={" "*width_total}=='] + model_summary_text += [f'=={"Model name": >{width_name}}: {self.get_model_name(): <{width_value}}=='] # Name + model_summary_text += [f'=={" "*width_total}=='] + model_summary_text += [f'=={"Current iteration": >{width_name}}: {str(self.iter): <{width_value}}=='] # Iter + model_summary_text += [f'=={" "*width_total}=='] + + model_summary_text += [f'=={" Model Options ":-^{width_total}}=='] # Model options + model_summary_text += [f'=={" "*width_total}=='] for key in self.options.keys(): - model_summary_text += ["== |== %s : %s" % (key, self.options[key])] - + model_summary_text += [f'=={key: >{width_name}}: {str(self.options[key]): <{width_value}}=='] # self.options key/value pairs + model_summary_text += [f'=={" "*width_total}=='] + + model_summary_text += [f'=={" Running On ":-^{width_total}}=='] # Training hardware info + model_summary_text += [f'=={" "*width_total}=='] if self.device_config.multi_gpu: - model_summary_text += ["== |== multi_gpu : True "] - - model_summary_text += ["== Running on:"] + model_summary_text += [f'=={"Using multi_gpu": >{width_name}}: {"True": <{width_value}}=='] # multi_gpu + model_summary_text += [f'=={" "*width_total}=='] if self.device_config.cpu_only: - model_summary_text += ["== |== [CPU]"] + model_summary_text += [f'=={"Using device": >{width_name}}: {"CPU": <{width_value}}=='] # cpu_only else: for idx in self.device_config.gpu_idxs: - model_summary_text += ["== |== [%d : %s]" % (idx, nnlib.device.getDeviceName(idx))] - - if not self.device_config.cpu_only and self.device_config.gpu_vram_gb[0] == 2: - model_summary_text += ["=="] - model_summary_text += ["== WARNING: You are using 2GB GPU. Result quality may be significantly decreased."] - model_summary_text += ["== If training does not start, close all programs and try again."] - model_summary_text += ["== Also you can disable Windows Aero Desktop to get extra free VRAM."] - model_summary_text += ["=="] - - model_summary_text += ["========================="] - model_summary_text = "\r\n".join (model_summary_text) + model_summary_text += [f'=={"Device index": >{width_name}}: {idx: <{width_value}}=='] # GPU hardware device index + model_summary_text += [f'=={"Name": >{width_name}}: {nnlib.device.getDeviceName(idx): <{width_value}}=='] # GPU name + vram_str = f'{nnlib.device.getDeviceVRAMTotalGb(idx):.2f}GB' # GPU VRAM - Formated as #.## (or ##.##) + model_summary_text += [f'=={"VRAM": >{width_name}}: {vram_str: <{width_value}}=='] + model_summary_text += [f'=={" "*width_total}=='] + model_summary_text += [f'=={"="*width_total}=='] + + if not self.device_config.cpu_only and self.device_config.gpu_vram_gb[0] <= 2: # Low VRAM warning + model_summary_text += ["/!\\"] + model_summary_text += ["/!\\ WARNING:"] + model_summary_text += ["/!\\ You are using a GPU with 2GB or less VRAM. This may significantly reduce the quality of your result!"] + model_summary_text += ["/!\\ If training does not start, close all programs and try again."] + model_summary_text += ["/!\\ Also you can disable Windows Aero Desktop to increase available VRAM."] + model_summary_text += ["/!\\"] + + model_summary_text = "\n".join (model_summary_text) self.model_summary_text = model_summary_text io.log_info(model_summary_text) From a906f24a4d89dd0895a13f04fb96c051d176c0f3 Mon Sep 17 00:00:00 2001 From: Auroir <36361058+Auroir@users.noreply.github.com> Date: Sat, 17 Aug 2019 00:20:10 -0700 Subject: [PATCH 2/6] Fix log_info/log_err printing with progress bars (#349) --- interact/interact.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/interact/interact.py b/interact/interact.py index c9b22f1..bda74af 100644 --- a/interact/interact.py +++ b/interact/interact.py @@ -33,6 +33,7 @@ class InteractBase(object): self.key_events = {} self.pg_bar = None self.focus_wnd_name = None + self.error_log_line_prefix = '/!\\ ' def is_support_windows(self): return False @@ -65,10 +66,22 @@ class InteractBase(object): raise NotImplemented def log_info(self, msg, end='\n'): + if self.pg_bar is not None: + try: # Attempt print before the pb + tqdm.write(msg) + return + except: + pass #Fallback to normal print upon failure print (msg, end=end) def log_err(self, msg, end='\n'): - print (msg, end=end) + if self.pg_bar is not None: + try: # Attempt print before the pb + tqdm.write(f'{self.error_log_line_prefix}{msg}') + return + except: + pass #Fallback to normal print upon failure + print (f'{self.error_log_line_prefix}{msg}', end=end) def named_window(self, wnd_name): if wnd_name not in self.named_windows: @@ -150,9 +163,12 @@ class InteractBase(object): else: print("progress_bar not set.") def progress_bar_generator(self, data, desc, leave=True): - for x in tqdm( data, desc=desc, leave=leave, ascii=True ): + self.pg_bar = tqdm( data, desc=desc, leave=leave, ascii=True ) + for x in self.pg_bar: yield x - + self.pg_bar.close() + self.pg_bar = None + def process_messages(self, sleep_time=0): self.on_process_messages(sleep_time) From e7562054d06e2482f274359d7dd9904d3c871c55 Mon Sep 17 00:00:00 2001 From: Auroir <36361058+Auroir@users.noreply.github.com> Date: Sat, 17 Aug 2019 21:01:44 -0700 Subject: [PATCH 3/6] Fix sorting pathlib objects (#353) Fixing an issue caused by attempting to sort Path objects. Directly using `<` is unsupported between these, so `sorted()` needs a key specified. "PurePath" objects support `>` while normal paths do not, causing the confusion. https://docs.python.org/3/library/pathlib.html --- utils/Path_utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/utils/Path_utils.py b/utils/Path_utils.py index d34e348..e753834 100644 --- a/utils/Path_utils.py +++ b/utils/Path_utils.py @@ -31,19 +31,18 @@ def get_image_unique_filestem_paths(dir_path, verbose_print_func=None): def get_file_paths(dir_path): dir_path = Path (dir_path) - result = [] if dir_path.exists(): - return [ x.path for x in list(scandir(str(dir_path))) if x.is_file() ] - return sorted(result) + return sorted([ x.path for x in list(scandir(str(dir_path))) if x.is_file() ]) + else: + return [] def get_all_dir_names (dir_path): dir_path = Path (dir_path) - result = [] if dir_path.exists(): - return [ x.name for x in list(scandir(str(dir_path))) if x.is_dir() ] - - return sorted(result) + return sorted([ x.name for x in list(scandir(str(dir_path))) if x.is_dir() ]) + else: + return [] def get_all_dir_names_startswith (dir_path, startswith): dir_path = Path (dir_path) @@ -61,7 +60,7 @@ def get_first_file_by_stem (dir_path, stem, exts=None): stem = stem.lower() if dir_path.exists(): - for x in sorted(list(scandir(str(dir_path)))): + for x in sorted(list(scandir(str(dir_path))), key=lambda x: x.name): if not x.is_file(): continue xp = Path(x.path) From a33ef50da5d3e576c61ba54806ce6480dddce48a Mon Sep 17 00:00:00 2001 From: Luke Barr Date: Mon, 19 Aug 2019 13:56:54 -0500 Subject: [PATCH 4/6] Long startup time on Linux (#356) --- mainscripts/Trainer.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mainscripts/Trainer.py b/mainscripts/Trainer.py index f170fcc..85a6d68 100644 --- a/mainscripts/Trainer.py +++ b/mainscripts/Trainer.py @@ -12,7 +12,7 @@ import cv2 import models from interact import interact as io -def trainerThread (s2c, c2s, args, device_args): +def trainerThread (s2c, c2s, e, args, device_args): while True: try: start_time = time.time() @@ -66,6 +66,7 @@ def trainerThread (s2c, c2s, args, device_args): else: previews = [( 'debug, press update for new', model.debug_one_iter())] c2s.put ( {'op':'show', 'previews': previews} ) + e.set() #Set the GUI Thread as Ready if model.is_first_run(): @@ -189,9 +190,12 @@ def main(args, device_args): s2c = queue.Queue() c2s = queue.Queue() - thread = threading.Thread(target=trainerThread, args=(s2c, c2s, args, device_args) ) + e = threading.Event() + thread = threading.Thread(target=trainerThread, args=(s2c, c2s, e, args, device_args) ) thread.start() + e.wait() #Wait for inital load to occur. + if no_preview: while True: if not c2s.empty(): @@ -291,7 +295,7 @@ def main(args, device_args): key_events = io.get_key_events(wnd_name) key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[-1] if len(key_events) > 0 else (0,0,False,False,False) - + if key == ord('\n') or key == ord('\r'): s2c.put ( {'op': 'close'} ) elif key == ord('s'): @@ -321,4 +325,4 @@ def main(args, device_args): except KeyboardInterrupt: s2c.put ( {'op': 'close'} ) - io.destroy_all_windows() + io.destroy_all_windows() \ No newline at end of file From fc5efb559d9df17a945c7233604f2454433a8468 Mon Sep 17 00:00:00 2001 From: iperov Date: Tue, 20 Aug 2019 17:10:00 +0400 Subject: [PATCH 5/6] upd readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5d4402f..f699578 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ bitcoin:31mPd6DxPCzbpCMZk4k1koWAbErSyqkAXr - ### Communication groups: +[mrdeepfakes (English)](https://mrdeepfakes.com) - the biggest SFW and NSFW community + (Chinese) QQ group 951138799 for ML/AI experts [deepfakes (Chinese)](https://deepfakescn.com) From 3f0bf2e9948e1752dd54aea9f5aeda1ef75a2735 Mon Sep 17 00:00:00 2001 From: iperov Date: Tue, 20 Aug 2019 17:11:37 +0400 Subject: [PATCH 6/6] upd readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f699578..5c73e4b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ bitcoin:31mPd6DxPCzbpCMZk4k1koWAbErSyqkAXr - ### Communication groups: -[mrdeepfakes (English)](https://mrdeepfakes.com) - the biggest SFW and NSFW community +[mrdeepfakes (English)](https://mrdeepfakes.com/forums/) - the biggest SFW and NSFW community (Chinese) QQ group 951138799 for ML/AI experts