From a65b557b0edf0b1d96cec7324adb966a115e05ec Mon Sep 17 00:00:00 2001 From: iperov Date: Thu, 7 Feb 2019 22:34:51 +0400 Subject: [PATCH] loss history with epoch now saves to preview history images --- mainscripts/Trainer.py | 68 ++++------------------------------------- models/ModelBase.py | 69 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 66 deletions(-) diff --git a/mainscripts/Trainer.py b/mainscripts/Trainer.py index 9a348f9..6334c5b 100644 --- a/mainscripts/Trainer.py +++ b/mainscripts/Trainer.py @@ -1,7 +1,7 @@ import sys import traceback import queue -import colorsys +import colorsys import time import numpy as np import itertools @@ -10,6 +10,7 @@ from pathlib import Path from utils import Path_utils from utils import image_utils import cv2 +import models def trainerThread (input_queue, output_queue, training_data_src_dir, training_data_dst_dir, model_path, model_name, save_interval_min=10, debug=False, **in_options): @@ -29,8 +30,7 @@ def trainerThread (input_queue, output_queue, training_data_src_dir, training_da if not model_path.exists(): model_path.mkdir(exist_ok=True) - - import models + model = models.import_model(model_name)( model_path, training_data_src_path=training_data_src_path, @@ -169,7 +169,6 @@ def previewThread (input_queue, output_queue): (h,w,c) = selected_preview_rgb.shape # HEAD - head_text_color = [0.8]*c head_lines = [ '[s]:save [enter]:exit', '[p]:update [space]:next preview', @@ -182,70 +181,15 @@ def previewThread (input_queue, output_queue): for i in range(0, len(head_lines)): t = i*head_line_height b = (i+1)*head_line_height - head[t:b, 0:w] += image_utils.get_text_image ( (w,head_line_height,c) , head_lines[i], color=head_text_color ) + head[t:b, 0:w] += image_utils.get_text_image ( (w,head_line_height,c) , head_lines[i], color=[0.8]*c ) final = head if loss_history is not None: # LOSS HISTORY - loss_history = np.array (loss_history) - - lh_height = 100 - lh_img = np.ones ( (lh_height,w,c) ) * 0.1 - loss_count = len(loss_history[0]) - lh_len = len(loss_history) - - l_per_col = lh_len / w - plist_max = [ [ max (0.0, loss_history[int(col*l_per_col)][p], - *[ loss_history[i_ab][p] - for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) - ] - ) - for p in range(loss_count) - ] - for col in range(w) - ] - - plist_min = [ [ min (plist_max[col][p], loss_history[int(col*l_per_col)][p], - *[ loss_history[i_ab][p] - for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) - ] - ) - for p in range(loss_count) - ] - for col in range(w) - ] - - plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2 - - for col in range(0, w): - for p in range(0,loss_count): - point_color = [1.0]*c - point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 ) - - ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) ) - ph_max = np.clip( ph_max, 0, lh_height-1 ) - - ph_min = int ( (plist_min[col][p] / plist_abs_max) * (lh_height-1) ) - ph_min = np.clip( ph_min, 0, lh_height-1 ) - - for ph in range(ph_min, ph_max+1): - lh_img[ (lh_height-ph-1), col ] = point_color - - lh_lines = 5 - lh_line_height = (lh_height-1)/lh_lines - for i in range(0,lh_lines+1): - lh_img[ int(i*lh_line_height), : ] = (0.8,)*c - - last_line_t = int((lh_lines-1)*lh_line_height) - last_line_b = int(lh_lines*lh_line_height) - - lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else '' - - lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image ( (w,last_line_b-last_line_t,c), lh_text, color=head_text_color ) - + lh_img = models.ModelBase.get_loss_history_preview(loss_history, epoch, w, c) final = np.concatenate ( [final, lh_img], axis=0 ) - + final = np.concatenate ( [final, selected_preview_rgb], axis=0 ) final = np.clip(final, 0, 1) diff --git a/models/ModelBase.py b/models/ModelBase.py index 216f2c3..c99d059 100644 --- a/models/ModelBase.py +++ b/models/ModelBase.py @@ -1,8 +1,8 @@ import os import time import inspect -import operator import pickle +import colorsys from pathlib import Path from utils import Path_utils from utils import std_utils @@ -320,8 +320,10 @@ class ModelBase(object): self.supress_std_once = False if self.write_preview_history: - if self.epoch % 10 == 0: - img = (self.get_static_preview() * 255).astype(np.uint8) + if self.epoch % 10 == 0: + preview = self.get_static_preview() + preview_lh = ModelBase.get_loss_history_preview(self.loss_history, self.epoch, preview.shape[1], preview.shape[2]) + img = (np.concatenate ( [preview_lh, preview], axis=0 ) * 255).astype(np.uint8) cv2_imwrite ( str (self.preview_history_path / ('%.6d.jpg' %( self.epoch) )), img ) self.epoch += 1 @@ -387,4 +389,63 @@ class ModelBase(object): break if self.batch_size == 0: - self.batch_size = d[ keys[-1] ] \ No newline at end of file + self.batch_size = d[ keys[-1] ] + + @staticmethod + def get_loss_history_preview(loss_history, epoch, w, c): + loss_history = np.array (loss_history.copy()) + + lh_height = 100 + lh_img = np.ones ( (lh_height,w,c) ) * 0.1 + loss_count = len(loss_history[0]) + lh_len = len(loss_history) + + l_per_col = lh_len / w + plist_max = [ [ max (0.0, loss_history[int(col*l_per_col)][p], + *[ loss_history[i_ab][p] + for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) + ] + ) + for p in range(loss_count) + ] + for col in range(w) + ] + + plist_min = [ [ min (plist_max[col][p], loss_history[int(col*l_per_col)][p], + *[ loss_history[i_ab][p] + for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) + ] + ) + for p in range(loss_count) + ] + for col in range(w) + ] + + plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2 + + for col in range(0, w): + for p in range(0,loss_count): + point_color = [1.0]*c + point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 ) + + ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) ) + ph_max = np.clip( ph_max, 0, lh_height-1 ) + + ph_min = int ( (plist_min[col][p] / plist_abs_max) * (lh_height-1) ) + ph_min = np.clip( ph_min, 0, lh_height-1 ) + + for ph in range(ph_min, ph_max+1): + lh_img[ (lh_height-ph-1), col ] = point_color + + lh_lines = 5 + lh_line_height = (lh_height-1)/lh_lines + for i in range(0,lh_lines+1): + lh_img[ int(i*lh_line_height), : ] = (0.8,)*c + + last_line_t = int((lh_lines-1)*lh_line_height) + last_line_b = int(lh_lines*lh_line_height) + + lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else '' + + lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image ( (w,last_line_b-last_line_t,c), lh_text, color=[0.8]*c ) + return lh_img \ No newline at end of file