mirror of
https://github.com/iperov/DeepFaceLab.git
synced 2025-07-06 21:12:07 -07:00
loss history with epoch now saves to preview history images
This commit is contained in:
iperov
parent
9545774c59
commit
a65b557b0e
2 changed files with 71 additions and 66 deletions
|
|
@ -10,6 +10,7 @@ from pathlib import Path
|
||||||
from utils import Path_utils
|
from utils import Path_utils
|
||||||
from utils import image_utils
|
from utils import image_utils
|
||||||
import cv2
|
import cv2
|
||||||
|
import models
|
||||||
|
|
||||||
def trainerThread (input_queue, output_queue, training_data_src_dir, training_data_dst_dir, model_path, model_name, save_interval_min=10, debug=False, **in_options):
|
def trainerThread (input_queue, output_queue, training_data_src_dir, training_data_dst_dir, model_path, model_name, save_interval_min=10, debug=False, **in_options):
|
||||||
|
|
||||||
|
|
@ -30,7 +31,6 @@ def trainerThread (input_queue, output_queue, training_data_src_dir, training_da
|
||||||
if not model_path.exists():
|
if not model_path.exists():
|
||||||
model_path.mkdir(exist_ok=True)
|
model_path.mkdir(exist_ok=True)
|
||||||
|
|
||||||
import models
|
|
||||||
model = models.import_model(model_name)(
|
model = models.import_model(model_name)(
|
||||||
model_path,
|
model_path,
|
||||||
training_data_src_path=training_data_src_path,
|
training_data_src_path=training_data_src_path,
|
||||||
|
|
@ -169,7 +169,6 @@ def previewThread (input_queue, output_queue):
|
||||||
(h,w,c) = selected_preview_rgb.shape
|
(h,w,c) = selected_preview_rgb.shape
|
||||||
|
|
||||||
# HEAD
|
# HEAD
|
||||||
head_text_color = [0.8]*c
|
|
||||||
head_lines = [
|
head_lines = [
|
||||||
'[s]:save [enter]:exit',
|
'[s]:save [enter]:exit',
|
||||||
'[p]:update [space]:next preview',
|
'[p]:update [space]:next preview',
|
||||||
|
|
@ -182,68 +181,13 @@ def previewThread (input_queue, output_queue):
|
||||||
for i in range(0, len(head_lines)):
|
for i in range(0, len(head_lines)):
|
||||||
t = i*head_line_height
|
t = i*head_line_height
|
||||||
b = (i+1)*head_line_height
|
b = (i+1)*head_line_height
|
||||||
head[t:b, 0:w] += image_utils.get_text_image ( (w,head_line_height,c) , head_lines[i], color=head_text_color )
|
head[t:b, 0:w] += image_utils.get_text_image ( (w,head_line_height,c) , head_lines[i], color=[0.8]*c )
|
||||||
|
|
||||||
final = head
|
final = head
|
||||||
|
|
||||||
if loss_history is not None:
|
if loss_history is not None:
|
||||||
# LOSS HISTORY
|
# LOSS HISTORY
|
||||||
loss_history = np.array (loss_history)
|
lh_img = models.ModelBase.get_loss_history_preview(loss_history, epoch, w, c)
|
||||||
|
|
||||||
lh_height = 100
|
|
||||||
lh_img = np.ones ( (lh_height,w,c) ) * 0.1
|
|
||||||
loss_count = len(loss_history[0])
|
|
||||||
lh_len = len(loss_history)
|
|
||||||
|
|
||||||
l_per_col = lh_len / w
|
|
||||||
plist_max = [ [ max (0.0, loss_history[int(col*l_per_col)][p],
|
|
||||||
*[ loss_history[i_ab][p]
|
|
||||||
for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) )
|
|
||||||
]
|
|
||||||
)
|
|
||||||
for p in range(loss_count)
|
|
||||||
]
|
|
||||||
for col in range(w)
|
|
||||||
]
|
|
||||||
|
|
||||||
plist_min = [ [ min (plist_max[col][p], loss_history[int(col*l_per_col)][p],
|
|
||||||
*[ loss_history[i_ab][p]
|
|
||||||
for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) )
|
|
||||||
]
|
|
||||||
)
|
|
||||||
for p in range(loss_count)
|
|
||||||
]
|
|
||||||
for col in range(w)
|
|
||||||
]
|
|
||||||
|
|
||||||
plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2
|
|
||||||
|
|
||||||
for col in range(0, w):
|
|
||||||
for p in range(0,loss_count):
|
|
||||||
point_color = [1.0]*c
|
|
||||||
point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 )
|
|
||||||
|
|
||||||
ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) )
|
|
||||||
ph_max = np.clip( ph_max, 0, lh_height-1 )
|
|
||||||
|
|
||||||
ph_min = int ( (plist_min[col][p] / plist_abs_max) * (lh_height-1) )
|
|
||||||
ph_min = np.clip( ph_min, 0, lh_height-1 )
|
|
||||||
|
|
||||||
for ph in range(ph_min, ph_max+1):
|
|
||||||
lh_img[ (lh_height-ph-1), col ] = point_color
|
|
||||||
|
|
||||||
lh_lines = 5
|
|
||||||
lh_line_height = (lh_height-1)/lh_lines
|
|
||||||
for i in range(0,lh_lines+1):
|
|
||||||
lh_img[ int(i*lh_line_height), : ] = (0.8,)*c
|
|
||||||
|
|
||||||
last_line_t = int((lh_lines-1)*lh_line_height)
|
|
||||||
last_line_b = int(lh_lines*lh_line_height)
|
|
||||||
|
|
||||||
lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else ''
|
|
||||||
|
|
||||||
lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image ( (w,last_line_b-last_line_t,c), lh_text, color=head_text_color )
|
|
||||||
|
|
||||||
final = np.concatenate ( [final, lh_img], axis=0 )
|
final = np.concatenate ( [final, lh_img], axis=0 )
|
||||||
|
|
||||||
final = np.concatenate ( [final, selected_preview_rgb], axis=0 )
|
final = np.concatenate ( [final, selected_preview_rgb], axis=0 )
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import inspect
|
import inspect
|
||||||
import operator
|
|
||||||
import pickle
|
import pickle
|
||||||
|
import colorsys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from utils import Path_utils
|
from utils import Path_utils
|
||||||
from utils import std_utils
|
from utils import std_utils
|
||||||
|
|
@ -321,7 +321,9 @@ class ModelBase(object):
|
||||||
|
|
||||||
if self.write_preview_history:
|
if self.write_preview_history:
|
||||||
if self.epoch % 10 == 0:
|
if self.epoch % 10 == 0:
|
||||||
img = (self.get_static_preview() * 255).astype(np.uint8)
|
preview = self.get_static_preview()
|
||||||
|
preview_lh = ModelBase.get_loss_history_preview(self.loss_history, self.epoch, preview.shape[1], preview.shape[2])
|
||||||
|
img = (np.concatenate ( [preview_lh, preview], axis=0 ) * 255).astype(np.uint8)
|
||||||
cv2_imwrite ( str (self.preview_history_path / ('%.6d.jpg' %( self.epoch) )), img )
|
cv2_imwrite ( str (self.preview_history_path / ('%.6d.jpg' %( self.epoch) )), img )
|
||||||
|
|
||||||
self.epoch += 1
|
self.epoch += 1
|
||||||
|
|
@ -388,3 +390,62 @@ class ModelBase(object):
|
||||||
|
|
||||||
if self.batch_size == 0:
|
if self.batch_size == 0:
|
||||||
self.batch_size = d[ keys[-1] ]
|
self.batch_size = d[ keys[-1] ]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_loss_history_preview(loss_history, epoch, w, c):
|
||||||
|
loss_history = np.array (loss_history.copy())
|
||||||
|
|
||||||
|
lh_height = 100
|
||||||
|
lh_img = np.ones ( (lh_height,w,c) ) * 0.1
|
||||||
|
loss_count = len(loss_history[0])
|
||||||
|
lh_len = len(loss_history)
|
||||||
|
|
||||||
|
l_per_col = lh_len / w
|
||||||
|
plist_max = [ [ max (0.0, loss_history[int(col*l_per_col)][p],
|
||||||
|
*[ loss_history[i_ab][p]
|
||||||
|
for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) )
|
||||||
|
]
|
||||||
|
)
|
||||||
|
for p in range(loss_count)
|
||||||
|
]
|
||||||
|
for col in range(w)
|
||||||
|
]
|
||||||
|
|
||||||
|
plist_min = [ [ min (plist_max[col][p], loss_history[int(col*l_per_col)][p],
|
||||||
|
*[ loss_history[i_ab][p]
|
||||||
|
for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) )
|
||||||
|
]
|
||||||
|
)
|
||||||
|
for p in range(loss_count)
|
||||||
|
]
|
||||||
|
for col in range(w)
|
||||||
|
]
|
||||||
|
|
||||||
|
plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2
|
||||||
|
|
||||||
|
for col in range(0, w):
|
||||||
|
for p in range(0,loss_count):
|
||||||
|
point_color = [1.0]*c
|
||||||
|
point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 )
|
||||||
|
|
||||||
|
ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) )
|
||||||
|
ph_max = np.clip( ph_max, 0, lh_height-1 )
|
||||||
|
|
||||||
|
ph_min = int ( (plist_min[col][p] / plist_abs_max) * (lh_height-1) )
|
||||||
|
ph_min = np.clip( ph_min, 0, lh_height-1 )
|
||||||
|
|
||||||
|
for ph in range(ph_min, ph_max+1):
|
||||||
|
lh_img[ (lh_height-ph-1), col ] = point_color
|
||||||
|
|
||||||
|
lh_lines = 5
|
||||||
|
lh_line_height = (lh_height-1)/lh_lines
|
||||||
|
for i in range(0,lh_lines+1):
|
||||||
|
lh_img[ int(i*lh_line_height), : ] = (0.8,)*c
|
||||||
|
|
||||||
|
last_line_t = int((lh_lines-1)*lh_line_height)
|
||||||
|
last_line_b = int(lh_lines*lh_line_height)
|
||||||
|
|
||||||
|
lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else ''
|
||||||
|
|
||||||
|
lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image ( (w,last_line_b-last_line_t,c), lh_text, color=[0.8]*c )
|
||||||
|
return lh_img
|
||||||
Loading…
Add table
Add a link
Reference in a new issue