From 71a242ccc1131b7f60cd10c41fcfc1d24914a55a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 18 Dec 2022 03:56:58 -0500 Subject: [PATCH] Refactor subprocess.Popen calls --- nzb2media/__init__.py | 194 ++++++++++---------------------- nzb2media/extractor/__init__.py | 11 +- nzb2media/scene_exceptions.py | 11 +- nzb2media/transcoder.py | 93 ++++++--------- nzb2media/user_scripts.py | 4 +- nzb2media/utils/processes.py | 6 +- nzb2media/version_check.py | 47 ++++---- 7 files changed, 127 insertions(+), 239 deletions(-) diff --git a/nzb2media/__init__.py b/nzb2media/__init__.py index 0dc0941f..f9a6bc7e 100644 --- a/nzb2media/__init__.py +++ b/nzb2media/__init__.py @@ -11,6 +11,7 @@ import subprocess import sys import time import typing +from subprocess import PIPE, DEVNULL from nzb2media import main_db from nzb2media import version_check @@ -35,6 +36,16 @@ except ImportError: sys.exit('Please install pywin32') +def which(name): + proc = subprocess.Popen(['which', name], stdout=PIPE) + try: + proc_out, proc_err = proc.communicate() + except Exception: + return '' + else: + return proc_out.strip().decode() + + def module_path(module=__file__): try: path = pathlib.Path(module.__file__) @@ -288,7 +299,7 @@ MOUNTED = None GETSUBS = False TRANSCODE = None CONCAT = None -FFMPEG_PATH = None +FFMPEG_PATH = '' SYS_PATH = None DUPLICATE = None IGNOREEXTENSIONS = [] @@ -532,40 +543,36 @@ def configure_remote_paths(): def configure_niceness(): global NICENESS - - with open(os.devnull, 'w') as devnull: + try: + proc = subprocess.Popen(['nice'], stdout=DEVNULL, stderr=DEVNULL) + proc.communicate() + niceness = CFG['Posix']['niceness'] + if ( + len(niceness.split(',')) > 1 + ): # Allow passing of absolute command, not just value. + NICENESS.extend(niceness.split(',')) + else: + NICENESS.extend(['nice', f'-n{int(niceness)}']) + except Exception: + pass + try: + proc = subprocess.Popen(['ionice'], stdout=DEVNULL, stderr=DEVNULL) + proc.communicate() try: - subprocess.Popen( - ['nice'], stdout=devnull, stderr=devnull, - ).communicate() - niceness = CFG['Posix']['niceness'] - if ( - len(niceness.split(',')) > 1 - ): # Allow passing of absolute command, not just value. - NICENESS.extend(niceness.split(',')) + ionice = CFG['Posix']['ionice_class'] + NICENESS.extend(['ionice', f'-c{int(ionice)}']) + except Exception: + pass + try: + if 'ionice' in NICENESS: + ionice = CFG['Posix']['ionice_classdata'] + NICENESS.extend([f'-n{int(ionice)}']) else: - NICENESS.extend(['nice', f'-n{int(niceness)}']) - except Exception: - pass - try: - subprocess.Popen( - ['ionice'], stdout=devnull, stderr=devnull, - ).communicate() - try: - ionice = CFG['Posix']['ionice_class'] - NICENESS.extend(['ionice', f'-c{int(ionice)}']) - except Exception: - pass - try: - if 'ionice' in NICENESS: - ionice = CFG['Posix']['ionice_classdata'] - NICENESS.extend([f'-n{int(ionice)}']) - else: - NICENESS.extend(['ionice', f'-n{int(ionice)}']) - except Exception: - pass + NICENESS.extend(['ionice', f'-n{int(ionice)}']) except Exception: pass + except Exception: + pass def configure_containers(): @@ -1413,123 +1420,36 @@ def configure_utility_locations(): else: if SYS_PATH: os.environ['PATH'] += ':' + SYS_PATH - try: - SEVENZIP = ( - subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass + + SEVENZIP = which('7z') or which('7zr') or which('7za') if not SEVENZIP: - try: - SEVENZIP = ( - subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass - if not SEVENZIP: - try: - SEVENZIP = ( - subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass - if not SEVENZIP: - SEVENZIP = None log.warning('Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!') - try: - PAR2CMD = ( - subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass + + PAR2CMD = which('par2') if not PAR2CMD: PAR2CMD = None log.warning('Failed to locate par2. Repair and rename using par files will not be possible!') - if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffmpeg')) or os.access( - os.path.join(FFMPEG_PATH, 'ffmpeg'), - os.X_OK, - ): - FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg') - elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avconv')) or os.access( - os.path.join(FFMPEG_PATH, 'avconv'), - os.X_OK, - ): - FFMPEG = os.path.join(FFMPEG_PATH, 'avconv') + + ffmpeg_bin = os.path.join(FFMPEG_PATH, 'ffmpeg') + avconv_bin = os.path.join(FFMPEG_PATH, 'avconv') + if os.path.isfile(ffmpeg_bin) or os.access(ffmpeg_bin, os.X_OK): + FFMPEG = ffmpeg_bin + elif os.path.isfile(avconv_bin) or os.access(avconv_bin, os.X_OK): + FFMPEG = avconv_bin else: - try: - FFMPEG = ( - subprocess.Popen( - ['which', 'ffmpeg'], stdout=subprocess.PIPE, - ) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass - if not FFMPEG: - try: - FFMPEG = ( - subprocess.Popen( - ['which', 'avconv'], stdout=subprocess.PIPE, - ) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass + FFMPEG = which('ffmpeg') or which('avconv') if not FFMPEG: FFMPEG = None log.warning('Failed to locate ffmpeg. Transcoding disabled!') log.warning('Install ffmpeg with x264 support to enable this feature ...') - - if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access( - os.path.join(FFMPEG_PATH, 'ffprobe'), - os.X_OK, - ): - FFPROBE = os.path.join(FFMPEG_PATH, 'ffprobe') - elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avprobe')) or os.access( - os.path.join(FFMPEG_PATH, 'avprobe'), - os.X_OK, - ): - FFPROBE = os.path.join(FFMPEG_PATH, 'avprobe') + ffprobe_bin = os.path.join(FFMPEG_PATH, 'ffprobe') + avprobe_bin = os.path.join(FFMPEG_PATH, 'avprobe') + if os.path.isfile(ffprobe_bin) or os.access(ffprobe_bin, os.X_OK): + FFPROBE = ffprobe_bin + elif os.path.isfile(avprobe_bin) or os.access(avprobe_bin, os.X_OK): + FFPROBE = avprobe_bin else: - try: - FFPROBE = ( - subprocess.Popen( - ['which', 'ffprobe'], stdout=subprocess.PIPE, - ) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass - if not FFPROBE: - try: - FFPROBE = ( - subprocess.Popen( - ['which', 'avprobe'], stdout=subprocess.PIPE, - ) - .communicate()[0] - .strip() - .decode() - ) - except Exception: - pass + FFPROBE = which('ffprobe') or which('avprobe') if not FFPROBE: FFPROBE = None if CHECK_MEDIA: diff --git a/nzb2media/extractor/__init__.py b/nzb2media/extractor/__init__.py index c2c25b70..1a3156c6 100644 --- a/nzb2media/extractor/__init__.py +++ b/nzb2media/extractor/__init__.py @@ -192,10 +192,9 @@ def extract(file_path, output_destination): cmd2 = cmd if 'gunzip' not in cmd: # gunzip doesn't support password cmd2.append('-p-') # don't prompt for password. - p = Popen( + res = Popen( cmd2, stdout=devnull, stderr=devnull, startupinfo=info, - ) # should extract files fine. - res = p.wait() + ).wait() # should extract files fine. if res == 0: # Both Linux and Windows return 0 for successful. log.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination}') success = 1 @@ -210,10 +209,10 @@ def extract(file_path, output_destination): # append password here. passcmd = f'-p{password}' cmd2.append(passcmd) - p = Popen( + proc = Popen( cmd2, stdout=devnull, stderr=devnull, startupinfo=info, - ) # should extract files fine. - res = p.wait() + ) + res = proc.wait() # should extract files fine. if (res >= 0 and platform == 'Windows') or res == 0: log.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination} using password: {password}') success = 1 diff --git a/nzb2media/scene_exceptions.py b/nzb2media/scene_exceptions.py index 8bc7b49f..ad909592 100644 --- a/nzb2media/scene_exceptions.py +++ b/nzb2media/scene_exceptions.py @@ -2,10 +2,10 @@ from __future__ import annotations import logging import os -import platform import re import shlex import subprocess +from subprocess import DEVNULL import nzb2media from nzb2media.utils.files import list_media_files @@ -212,10 +212,6 @@ def par2(dirname): if nzb2media.PAR2CMD and parfile: pwd = os.getcwd() # Get our Present Working Directory os.chdir(dirname) # set directory to run par on. - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') log.info(f'Running par2 on file {parfile}.') command = [nzb2media.PAR2CMD, 'r', parfile, '*'] cmd = '' @@ -223,9 +219,7 @@ def par2(dirname): cmd = f'{cmd} {item}' log.debug(f'calling command:{cmd}') try: - proc = subprocess.Popen( - command, stdout=bitbucket, stderr=bitbucket, - ) + proc = subprocess.Popen(command, stdout=DEVNULL, stderr=DEVNULL) proc.communicate() result = proc.returncode except Exception: @@ -233,7 +227,6 @@ def par2(dirname): if result == 0: log.info('par2 file processing succeeded') os.chdir(pwd) - bitbucket.close() # dict for custom groups diff --git a/nzb2media/transcoder.py b/nzb2media/transcoder.py index dbe8ba3a..a1f4ce29 100644 --- a/nzb2media/transcoder.py +++ b/nzb2media/transcoder.py @@ -11,6 +11,7 @@ import shutil import subprocess import sys import time +from subprocess import PIPE, DEVNULL from babelfish import Language @@ -100,22 +101,20 @@ def is_video_good(video: pathlib.Path, status, require_lan=None): return False -def zip_out(file, img, bitbucket): - procin = None +def zip_out(file, img): + proc = None if os.path.isfile(file): cmd = ['cat', file] else: cmd = [nzb2media.SEVENZIP, '-so', 'e', img, file] try: - procin = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=bitbucket, - ) + proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL) except Exception: log.error(f'Extracting [{file}] has failed') - return procin + return proc -def get_video_details(videofile, img=None, bitbucket=None): +def get_video_details(videofile, img=None): video_details = {} result = 1 file = videofile @@ -138,13 +137,11 @@ def get_video_details(videofile, img=None, bitbucket=None): ] print_cmd(command) if img: - procin = zip_out(file, img, bitbucket) - proc = subprocess.Popen( - command, stdout=subprocess.PIPE, stdin=procin.stdout, - ) + procin = zip_out(file, img) + proc = subprocess.Popen(command, stdout=PIPE, stdin=procin.stdout) procin.stdout.close() else: - proc = subprocess.Popen(command, stdout=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=PIPE) out, err = proc.communicate() result = proc.returncode video_details = json.loads(out.decode()) @@ -162,13 +159,11 @@ def get_video_details(videofile, img=None, bitbucket=None): ] print_cmd(command) if img: - procin = zip_out(file, img, bitbucket) - proc = subprocess.Popen( - command, stdout=subprocess.PIPE, stdin=procin.stdout, - ) + procin = zip_out(file, img) + proc = subprocess.Popen(command, stdout=PIPE, stdin=procin.stdout) procin.stdout.close() else: - proc = subprocess.Popen(command, stdout=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=PIPE) out, err = proc.communicate() result = proc.returncode video_details = json.loads(out.decode()) @@ -200,7 +195,7 @@ def check_vid_file(video_details, result): return False -def build_commands(file, new_dir, movie_name, bitbucket): +def build_commands(file, new_dir, movie_name): if isinstance(file, str): input_file = file if 'concat:' in file: @@ -228,16 +223,14 @@ def build_commands(file, new_dir, movie_name, bitbucket): new_file = [] rem_vid = [] for vid in data['files']: - video_details, result = get_video_details(vid, img, bitbucket) + video_details, result = get_video_details(vid, img) if not check_vid_file( video_details, result, ): # lets not transcode menu or other clips that don't have audio and video. rem_vid.append(vid) data['files'] = [f for f in data['files'] if f not in rem_vid] new_file = {img: {'name': data['name'], 'files': data['files']}} - video_details, result = get_video_details( - data['files'][0], img, bitbucket, - ) + video_details, result = get_video_details(data['files'][0], img) input_file = '-' file = '-' @@ -752,7 +745,7 @@ def get_subs(file): return subfiles -def extract_subs(file, newfile_path, bitbucket): +def extract_subs(file, newfile_path): video_details, result = get_video_details(file) if not video_details: return @@ -815,9 +808,9 @@ def extract_subs(file, newfile_path, bitbucket): result = 1 # set result to failed in case call fails. try: proc = subprocess.Popen( - command, stdout=bitbucket, stderr=bitbucket, + command, stdout=DEVNULL, stderr=DEVNULL, ) - out, err = proc.communicate() + proc_out, proc_error = proc.communicate() result = proc.returncode except Exception: log.error('Extracting subtitle has failed') @@ -832,7 +825,7 @@ def extract_subs(file, newfile_path, bitbucket): log.error('Extracting subtitles has failed') -def process_list(it, new_dir, bitbucket): +def process_list(it, new_dir): rem_list = [] new_list = [] combine = [] @@ -846,7 +839,7 @@ def process_list(it, new_dir, bitbucket): and ext not in nzb2media.IGNOREEXTENSIONS ): log.debug(f'Attempting to rip disk image: {item}') - new_list.extend(rip_iso(item, new_dir, bitbucket)) + new_list.extend(rip_iso(item, new_dir)) rem_list.append(item) elif ( re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', item) @@ -907,9 +900,7 @@ def process_list(it, new_dir, bitbucket): return it, rem_list, new_list, success -def mount_iso( - item, new_dir, bitbucket, -): # Currently only supports Linux Mount when permissions allow. +def mount_iso(item, new_dir): # Currently only supports Linux Mount when permissions allow. if platform.system() == 'Windows': log.error(f'No mounting options available under Windows for image file {item}') return [] @@ -917,7 +908,7 @@ def mount_iso( make_dir(mount_point) cmd = ['mount', '-o', 'loop', item, mount_point] print_cmd(cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL) out, err = proc.communicate() nzb2media.MOUNTED = ( mount_point # Allows us to verify this has been done and then cleanup. @@ -951,16 +942,15 @@ def mount_iso( return ['failure'] # If we got here, nothing matched our criteria -def rip_iso(item, new_dir, bitbucket): +def rip_iso(item, new_dir): new_files = [] failure_dir = 'failure' # Mount the ISO in your OS and call combineVTS. if not nzb2media.SEVENZIP: log.debug(f'No 7zip installed. Attempting to mount image file {item}') try: - new_files = mount_iso( - item, new_dir, bitbucket, - ) # Currently only works for Linux. + # Currently only works for Linux. + new_files = mount_iso(item, new_dir) except Exception: log.error(f'Failed to mount and extract from image file {item}') new_files = [failure_dir] @@ -969,7 +959,7 @@ def rip_iso(item, new_dir, bitbucket): try: log.debug(f'Attempting to extract .vob or .mts from image file {item}') print_cmd(cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL) out, err = proc.communicate() file_match_gen = ( re.match( @@ -1040,7 +1030,7 @@ def rip_iso(item, new_dir, bitbucket): new_files.append({item: {'name': name, 'files': combined}}) if not new_files: log.error(f'No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {item}') - new_files = mount_iso(item, new_dir, bitbucket) + new_files = mount_iso(item, new_dir) except Exception: log.error(f'Failed to extract from image file {item}') new_files = [failure_dir] @@ -1159,19 +1149,12 @@ def transcode_directory(dir_name): make_dir(new_dir) else: new_dir = dir_name - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') movie_name = os.path.splitext(os.path.split(dir_name)[1])[0] file_list = nzb2media.list_media_files( dir_name, media=True, audio=False, meta=False, archives=False, ) - file_list, rem_list, new_list, success = process_list( - file_list, new_dir, bitbucket, - ) + file_list, rem_list, new_list, success = process_list(file_list, new_dir) if not success: - bitbucket.close() return 1, dir_name for file in file_list: @@ -1180,12 +1163,12 @@ def transcode_directory(dir_name): and os.path.splitext(file)[1] in nzb2media.IGNOREEXTENSIONS ): continue - command, file = build_commands(file, new_dir, movie_name, bitbucket) + command, file = build_commands(file, new_dir, movie_name) newfile_path = command[-1] # transcoding files may remove the original file, so make sure to extract subtitles first if nzb2media.SEXTRACT and isinstance(file, str): - extract_subs(file, newfile_path, bitbucket) + extract_subs(file, newfile_path) try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) os.remove(newfile_path) @@ -1202,19 +1185,12 @@ def transcode_directory(dir_name): result = 1 # set result to failed in case call fails. try: if isinstance(file, str): - proc = subprocess.Popen( - command, stdout=bitbucket, stderr=subprocess.PIPE, - ) + proc = subprocess.Popen(command, stdout=DEVNULL, stderr=PIPE) else: img, data = next(file.items()) - proc = subprocess.Popen( - command, - stdout=bitbucket, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - ) + proc = subprocess.Popen(command, stdout=DEVNULL, stderr=PIPE, stdin=PIPE) for vob in data['files']: - procin = zip_out(vob, img, bitbucket) + procin = zip_out(vob, img) if procin: log.debug(f'Feeding in file: {vob} to Transcoder') shutil.copyfileobj(procin.stdout, proc.stdin) @@ -1258,7 +1234,7 @@ def transcode_directory(dir_name): time.sleep(5) # play it safe and avoid failing to unmount. cmd = ['umount', '-l', nzb2media.MOUNTED] print_cmd(cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL) out, err = proc.communicate() time.sleep(5) os.rmdir(nzb2media.MOUNTED) @@ -1278,5 +1254,4 @@ def transcode_directory(dir_name): not nzb2media.PROCESSOUTPUT and nzb2media.DUPLICATE ): # We postprocess the original files to CP/SB new_dir = dir_name - bitbucket.close() return final_result, new_dir diff --git a/nzb2media/user_scripts.py b/nzb2media/user_scripts.py index ddb057a0..d761ddd0 100644 --- a/nzb2media/user_scripts.py +++ b/nzb2media/user_scripts.py @@ -118,8 +118,8 @@ def external_script(output_destination, torrent_name, torrent_label, settings): cmd = f'{cmd} {item}' log.info(f'Running script {cmd} on file {file_path}.') try: - p = Popen(command) - res = p.wait() + proc = Popen(command) + res = proc.wait() if ( str(res) in nzb2media.USER_SCRIPT_SUCCESSCODES ): # Linux returns 0 for successful. diff --git a/nzb2media/utils/processes.py b/nzb2media/utils/processes.py index 1056f73f..7682deae 100644 --- a/nzb2media/utils/processes.py +++ b/nzb2media/utils/processes.py @@ -111,8 +111,8 @@ def restart(): if popen_list: popen_list += nzb2media.SYS_ARGV log.info(f'Restarting nzbToMedia with {popen_list}') - p = subprocess.Popen(popen_list, cwd=os.getcwd()) - p.wait() - status = p.returncode + proc = subprocess.Popen(popen_list, cwd=os.getcwd()) + proc.wait() + status = proc.returncode os._exit(status) diff --git a/nzb2media/version_check.py b/nzb2media/version_check.py index c96db2ca..be7efeb8 100644 --- a/nzb2media/version_check.py +++ b/nzb2media/version_check.py @@ -11,6 +11,7 @@ import stat import subprocess import tarfile import traceback +from subprocess import PIPE, STDOUT from urllib.request import urlretrieve import nzb2media @@ -161,52 +162,52 @@ class GitUpdateManager(UpdateManager): def _run_git(self, git_path, args): - output = None - err = None + proc_out = None + proc_err = None if not git_path: log.debug('No git specified, can\'t use git commands') - exit_status = 1 - return output, err, exit_status + proc_status = 1 + return proc_out, proc_err, proc_status cmd = f'{git_path} {args}' try: log.debug(f'Executing {cmd} with your shell in {nzb2media.APP_ROOT}') - p = subprocess.Popen( + proc = subprocess.Popen( cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stdin=PIPE, + stdout=PIPE, + stderr=STDOUT, shell=True, cwd=nzb2media.APP_ROOT, ) - output, err = p.communicate() - exit_status = p.returncode + proc_out, proc_err = proc.communicate() + proc_status = proc.returncode - output = output.decode('utf-8') + proc_out = proc_out.decode('utf-8') - if output: - output = output.strip() + if proc_out: + proc_out = proc_out.strip() if nzb2media.LOG_GIT: - log.debug(f'git output: {output}') + log.debug(f'git output: {proc_out}') except OSError: log.error(f'Command {cmd} didn\'t work') - exit_status = 1 + proc_status = 1 - exit_status = 128 if ('fatal:' in output) or err else exit_status - if exit_status == 0: + proc_status = 128 if ('fatal:' in proc_out) or proc_err else proc_status + if proc_status == 0: log.debug(f'{cmd} : returned successful') - exit_status = 0 - elif nzb2media.LOG_GIT and exit_status in (1, 128): - log.debug(f'{cmd} returned : {output}') + proc_status = 0 + elif nzb2media.LOG_GIT and proc_status in (1, 128): + log.debug(f'{cmd} returned : {proc_out}') else: if nzb2media.LOG_GIT: - log.debug(f'{cmd} returned : {output}, treat as error for now') - exit_status = 1 + log.debug(f'{cmd} returned : {proc_out}, treat as error for now') + proc_status = 1 - return output, err, exit_status + return proc_out, proc_err, proc_status def _find_installed_version(self): """