github/DeepFaceLab
1
0
Fork 0
mirror of https://github.com/iperov/DeepFaceLab.git synced 2025-07-06 13:02:15 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix autobackup_hour

Browse source
This commit is contained in:
Colombo 2020-02-27 11:13:45 +04:00
parent 9860a38907
commit a5783df546
2 changed files with 16 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

4
mainscripts/Trainer.py
View file

@ -131,8 +131,7 @@ def trainerThread (s2c, c2s, e,
if shared_state['after_save']: if shared_state['after_save']:
shared_state['after_save'] = False shared_state['after_save'] = False
last_save_time = time.time()
mean_loss = np.mean ( loss_history[save_iter:iter], axis=0) mean_loss = np.mean ( loss_history[save_iter:iter], axis=0)
for loss_value in mean_loss: for loss_value in mean_loss:
@ -160,6 +159,7 @@ def trainerThread (s2c, c2s, e,
io.log_info ('You can use preview now.') io.log_info ('You can use preview now.')
if not is_reached_goal and (time.time() - last_save_time) >= save_interval_min*60: if not is_reached_goal and (time.time() - last_save_time) >= save_interval_min*60:
last_save_time += save_interval_min*60
model_save() model_save()
send_preview() send_preview()

31
models/ModelBase.py
View file

@ -173,18 +173,10 @@ class ModelBase(object):
self.on_initialize() self.on_initialize()
self.options['batch_size'] = self.batch_size self.options['batch_size'] = self.batch_size
if self.is_training: if self.is_training:
self.preview_history_path = self.saved_models_path / ( f'{self.get_model_name()}_history' ) self.preview_history_path = self.saved_models_path / ( f'{self.get_model_name()}_history' )
self.autobackups_path = self.saved_models_path / ( f'{self.get_model_name()}_autobackups' ) self.autobackups_path = self.saved_models_path / ( f'{self.get_model_name()}_autobackups' )
if self.autobackup_hour != 0:
self.autobackup_start_hour = int(time.time() // 3600)
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
if self.write_preview_history or io.is_colab(): if self.write_preview_history or io.is_colab():
if not self.preview_history_path.exists(): if not self.preview_history_path.exists():
self.preview_history_path.mkdir(exist_ok=True) self.preview_history_path.mkdir(exist_ok=True)
@ -201,6 +193,12 @@ class ModelBase(object):
raise ValueError('training data generator is not subclass of SampleGeneratorBase') raise ValueError('training data generator is not subclass of SampleGeneratorBase')
self.update_sample_for_preview(choose_preview_history=self.choose_preview_history) self.update_sample_for_preview(choose_preview_history=self.choose_preview_history)
if self.autobackup_hour != 0:
self.autobackup_start_time = time.time()
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
io.log_info( self.get_summary_text() ) io.log_info( self.get_summary_text() )
@ -256,12 +254,12 @@ class ModelBase(object):
def ask_override(self): def ask_override(self):
return self.is_training and self.iter != 0 and io.input_in_time ("Press enter in 2 seconds to override model settings.", 5 if io.is_colab() else 2 ) return self.is_training and self.iter != 0 and io.input_in_time ("Press enter in 2 seconds to override model settings.", 5 if io.is_colab() else 2 )
def ask_autobackup_hour(self): def ask_autobackup_hour(self, default_value):
default_autobackup_hour = self.options['autobackup_hour'] = self.load_or_def_option('autobackup_hour', 0) default_autobackup_hour = self.options['autobackup_hour'] = self.load_or_def_option('autobackup_hour', default_value)
self.options['autobackup_hour'] = io.input_int(f"Autobackup every N hour", default_autobackup_hour, add_info="0..24", help_message="Autobackup model files with preview every N hour. Latest backup located in model/<>_autobackups/01") self.options['autobackup_hour'] = io.input_int(f"Autobackup every N hour", default_autobackup_hour, add_info="0..24", help_message="Autobackup model files with preview every N hour. Latest backup located in model/<>_autobackups/01")
def ask_write_preview_history(self): def ask_write_preview_history(self, default_value=False):
default_write_preview_history = self.load_or_def_option('write_preview_history', False) default_write_preview_history = self.load_or_def_option('write_preview_history', default_value)
self.options['write_preview_history'] = io.input_bool(f"Write preview history", default_write_preview_history, help_message="Preview history will be writed to <ModelName>_history folder.") self.options['write_preview_history'] = io.input_bool(f"Write preview history", default_write_preview_history, help_message="Preview history will be writed to <ModelName>_history folder.")
if self.options['write_preview_history']: if self.options['write_preview_history']:
@ -270,8 +268,8 @@ class ModelBase(object):
elif io.is_colab(): elif io.is_colab():
self.choose_preview_history = io.input_bool("Randomly choose new image for preview history", False, help_message="Preview image history will stay stuck with old faces if you reuse the same model on different celebs. Choose no unless you are changing src/dst to a new person") self.choose_preview_history = io.input_bool("Randomly choose new image for preview history", False, help_message="Preview image history will stay stuck with old faces if you reuse the same model on different celebs. Choose no unless you are changing src/dst to a new person")
def ask_target_iter(self): def ask_target_iter(self, default_value=0):
default_target_iter = self.load_or_def_option('target_iter', 0) default_target_iter = self.load_or_def_option('target_iter', default_value)
self.options['target_iter'] = max(0, io.input_int("Target iteration", default_target_iter)) self.options['target_iter'] = max(0, io.input_int("Target iteration", default_target_iter))
def ask_random_flip(self): def ask_random_flip(self):
@ -359,11 +357,10 @@ class ModelBase(object):
pathex.write_bytes_safe (self.model_data_path, pickle.dumps(model_data) ) pathex.write_bytes_safe (self.model_data_path, pickle.dumps(model_data) )
if self.autobackup_hour != 0: if self.autobackup_hour != 0:
current_hour = int(time.time() // 3600) diff_hour = int ( (time.time() - self.autobackup_start_time) // 3600 )
diff_hour = current_hour - self.autobackup_start_hour
if diff_hour > 0 and diff_hour % self.autobackup_hour == 0: if diff_hour > 0 and diff_hour % self.autobackup_hour == 0:
self.autobackup_start_hour = current_hour self.autobackup_start_time += self.autobackup_hour*3600
self.create_backup() self.create_backup()
def create_backup(self): def create_backup(self):