github/DeepFaceLab
1
0
Fork 0
mirror of https://github.com/iperov/DeepFaceLab.git synced 2025-07-06 21:12:07 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

_

Browse source
This commit is contained in:
Colombo 2020-02-29 14:31:21 +04:00
parent 10ffca37c0
commit fcbfbdd560
1 changed files with 35 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

48
core/joblib/SubprocessorBase.py
View file

@ -26,14 +26,9 @@ class Subprocessor(object):
self.name = None self.name = None
self.host_dict = None self.host_dict = None
def close(self, terminate=False): def kill(self):
if terminate:
self.p.terminate() self.p.terminate()
self.p.join() self.p.join()
self.s2c.close()
self.c2s.close()
self.s2c = None
self.c2s = None
#overridable optional #overridable optional
def on_initialize(self, client_dict): def on_initialize(self, client_dict):
@ -60,9 +55,17 @@ class Subprocessor(object):
def progress_bar_inc(self, c): self.c2s.put ( {'op': 'progress_bar_inc' , 'c':c } ) def progress_bar_inc(self, c): self.c2s.put ( {'op': 'progress_bar_inc' , 'c':c } )
def _subprocess_run(self, client_dict, s2c, c2s): def _subprocess_run(self, client_dict, s2c, c2s):
import sys
if sys.platform != 'win32':
# fix for Linux , Ignoring :
# /usr/lib/python3.6/multiprocessing/semaphore_tracker.py:143:
# UserWarning: semaphore_tracker: There appear to be 1 leaked semaphores to clean up at shutdown
import warnings
warnings.simplefilter(action='ignore', category=UserWarning)
self.s2c = s2c
self.c2s = c2s self.c2s = c2s
data = None data = None
is_error = False
try: try:
self.on_initialize(client_dict) self.on_initialize(client_dict)
@ -83,25 +86,17 @@ class Subprocessor(object):
self.on_finalize() self.on_finalize()
c2s.put ( {'op': 'finalized'} ) c2s.put ( {'op': 'finalized'} )
return
except Subprocessor.SilenceException as e: except Subprocessor.SilenceException as e:
is_error = True pass
except Exception as e: except Exception as e:
is_error = True
if data is not None: if data is not None:
print ('Exception while process data [%s]: %s' % (self.get_data_name(data), traceback.format_exc()) ) print ('Exception while process data [%s]: %s' % (self.get_data_name(data), traceback.format_exc()) )
else: else:
print ('Exception: %s' % (traceback.format_exc()) ) print ('Exception: %s' % (traceback.format_exc()) )
if is_error:
c2s.put ( {'op': 'error', 'data' : data} ) c2s.put ( {'op': 'error', 'data' : data} )
s2c.close()
c2s.close()
self.c2s = None
import gc
gc.collect()
# disable pickling # disable pickling
def __getstate__(self): def __getstate__(self):
return dict() return dict()
@ -194,7 +189,7 @@ class Subprocessor(object):
elif op == 'log_err': elif op == 'log_err':
io.log_err(obj['msg']) io.log_err(obj['msg'])
elif op == 'error': elif op == 'error':
cli.close() cli.kill()
self.clis.remove(cli) self.clis.remove(cli)
break break
if cli.state == 0: if cli.state == 0:
@ -219,7 +214,7 @@ class Subprocessor(object):
elif op == 'log_err': elif op == 'log_err':
io.log_err(obj['msg']) io.log_err(obj['msg'])
elif op == 'error': elif op == 'error':
cli.close() cli.kill()
self.clis.remove(cli) self.clis.remove(cli)
break break
if all ([cli.state == 0 for cli in self.clis]): if all ([cli.state == 0 for cli in self.clis]):
@ -248,7 +243,8 @@ class Subprocessor(object):
#some error occured while process data, returning chunk to on_data_return #some error occured while process data, returning chunk to on_data_return
if 'data' in obj.keys(): if 'data' in obj.keys():
self.on_data_return (cli.host_dict, obj['data'] ) self.on_data_return (cli.host_dict, obj['data'] )
cli.close() #and killing process
cli.kill()
self.clis.remove(cli) self.clis.remove(cli)
elif op == 'log_info': elif op == 'log_info':
io.log_info(obj['msg']) io.log_info(obj['msg'])
@ -263,7 +259,7 @@ class Subprocessor(object):
#subprocess busy too long #subprocess busy too long
print ( '%s doesnt response, terminating it.' % (cli.name) ) print ( '%s doesnt response, terminating it.' % (cli.name) )
self.on_data_return (cli.host_dict, cli.sent_data ) self.on_data_return (cli.host_dict, cli.sent_data )
cli.close(kill=True) cli.kill()
self.clis.remove(cli) self.clis.remove(cli)
for cli in self.clis[:]: for cli in self.clis[:]:
@ -293,18 +289,20 @@ class Subprocessor(object):
while True: while True:
for cli in self.clis[:]: for cli in self.clis[:]:
if cli.state != 2: terminate_it = False
while not cli.c2s.empty(): while not cli.c2s.empty():
obj = cli.c2s.get() obj = cli.c2s.get()
obj_op = obj['op'] obj_op = obj['op']
if obj_op == 'finalized': if obj_op == 'finalized':
cli.state = 2 terminate_it = True
cli.close()
break break
if (time.time() - cli.sent_time) > 30: if (time.time() - cli.sent_time) > 30:
terminate_it = True
if terminate_it:
cli.state = 2 cli.state = 2
cli.close(kill=True) cli.kill()
if all ([cli.state == 2 for cli in self.clis]): if all ([cli.state == 2 for cli in self.clis]):
break break