Refactor subprocess.Popen calls

This commit is contained in:
Labrys of Knossos 2022-12-18 03:56:58 -05:00
commit 71a242ccc1
7 changed files with 127 additions and 239 deletions

View file

@ -11,6 +11,7 @@ import subprocess
import sys
import time
import typing
from subprocess import PIPE, DEVNULL
from nzb2media import main_db
from nzb2media import version_check
@ -35,6 +36,16 @@ except ImportError:
sys.exit('Please install pywin32')
def which(name):
proc = subprocess.Popen(['which', name], stdout=PIPE)
try:
proc_out, proc_err = proc.communicate()
except Exception:
return ''
else:
return proc_out.strip().decode()
def module_path(module=__file__):
try:
path = pathlib.Path(module.__file__)
@ -288,7 +299,7 @@ MOUNTED = None
GETSUBS = False
TRANSCODE = None
CONCAT = None
FFMPEG_PATH = None
FFMPEG_PATH = ''
SYS_PATH = None
DUPLICATE = None
IGNOREEXTENSIONS = []
@ -532,40 +543,36 @@ def configure_remote_paths():
def configure_niceness():
global NICENESS
with open(os.devnull, 'w') as devnull:
try:
proc = subprocess.Popen(['nice'], stdout=DEVNULL, stderr=DEVNULL)
proc.communicate()
niceness = CFG['Posix']['niceness']
if (
len(niceness.split(',')) > 1
): # Allow passing of absolute command, not just value.
NICENESS.extend(niceness.split(','))
else:
NICENESS.extend(['nice', f'-n{int(niceness)}'])
except Exception:
pass
try:
proc = subprocess.Popen(['ionice'], stdout=DEVNULL, stderr=DEVNULL)
proc.communicate()
try:
subprocess.Popen(
['nice'], stdout=devnull, stderr=devnull,
).communicate()
niceness = CFG['Posix']['niceness']
if (
len(niceness.split(',')) > 1
): # Allow passing of absolute command, not just value.
NICENESS.extend(niceness.split(','))
ionice = CFG['Posix']['ionice_class']
NICENESS.extend(['ionice', f'-c{int(ionice)}'])
except Exception:
pass
try:
if 'ionice' in NICENESS:
ionice = CFG['Posix']['ionice_classdata']
NICENESS.extend([f'-n{int(ionice)}'])
else:
NICENESS.extend(['nice', f'-n{int(niceness)}'])
except Exception:
pass
try:
subprocess.Popen(
['ionice'], stdout=devnull, stderr=devnull,
).communicate()
try:
ionice = CFG['Posix']['ionice_class']
NICENESS.extend(['ionice', f'-c{int(ionice)}'])
except Exception:
pass
try:
if 'ionice' in NICENESS:
ionice = CFG['Posix']['ionice_classdata']
NICENESS.extend([f'-n{int(ionice)}'])
else:
NICENESS.extend(['ionice', f'-n{int(ionice)}'])
except Exception:
pass
NICENESS.extend(['ionice', f'-n{int(ionice)}'])
except Exception:
pass
except Exception:
pass
def configure_containers():
@ -1413,123 +1420,36 @@ def configure_utility_locations():
else:
if SYS_PATH:
os.environ['PATH'] += ':' + SYS_PATH
try:
SEVENZIP = (
subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
SEVENZIP = which('7z') or which('7zr') or which('7za')
if not SEVENZIP:
try:
SEVENZIP = (
subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
if not SEVENZIP:
try:
SEVENZIP = (
subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
if not SEVENZIP:
SEVENZIP = None
log.warning('Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!')
try:
PAR2CMD = (
subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
PAR2CMD = which('par2')
if not PAR2CMD:
PAR2CMD = None
log.warning('Failed to locate par2. Repair and rename using par files will not be possible!')
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffmpeg')) or os.access(
os.path.join(FFMPEG_PATH, 'ffmpeg'),
os.X_OK,
):
FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg')
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avconv')) or os.access(
os.path.join(FFMPEG_PATH, 'avconv'),
os.X_OK,
):
FFMPEG = os.path.join(FFMPEG_PATH, 'avconv')
ffmpeg_bin = os.path.join(FFMPEG_PATH, 'ffmpeg')
avconv_bin = os.path.join(FFMPEG_PATH, 'avconv')
if os.path.isfile(ffmpeg_bin) or os.access(ffmpeg_bin, os.X_OK):
FFMPEG = ffmpeg_bin
elif os.path.isfile(avconv_bin) or os.access(avconv_bin, os.X_OK):
FFMPEG = avconv_bin
else:
try:
FFMPEG = (
subprocess.Popen(
['which', 'ffmpeg'], stdout=subprocess.PIPE,
)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
if not FFMPEG:
try:
FFMPEG = (
subprocess.Popen(
['which', 'avconv'], stdout=subprocess.PIPE,
)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
FFMPEG = which('ffmpeg') or which('avconv')
if not FFMPEG:
FFMPEG = None
log.warning('Failed to locate ffmpeg. Transcoding disabled!')
log.warning('Install ffmpeg with x264 support to enable this feature ...')
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access(
os.path.join(FFMPEG_PATH, 'ffprobe'),
os.X_OK,
):
FFPROBE = os.path.join(FFMPEG_PATH, 'ffprobe')
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avprobe')) or os.access(
os.path.join(FFMPEG_PATH, 'avprobe'),
os.X_OK,
):
FFPROBE = os.path.join(FFMPEG_PATH, 'avprobe')
ffprobe_bin = os.path.join(FFMPEG_PATH, 'ffprobe')
avprobe_bin = os.path.join(FFMPEG_PATH, 'avprobe')
if os.path.isfile(ffprobe_bin) or os.access(ffprobe_bin, os.X_OK):
FFPROBE = ffprobe_bin
elif os.path.isfile(avprobe_bin) or os.access(avprobe_bin, os.X_OK):
FFPROBE = avprobe_bin
else:
try:
FFPROBE = (
subprocess.Popen(
['which', 'ffprobe'], stdout=subprocess.PIPE,
)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
if not FFPROBE:
try:
FFPROBE = (
subprocess.Popen(
['which', 'avprobe'], stdout=subprocess.PIPE,
)
.communicate()[0]
.strip()
.decode()
)
except Exception:
pass
FFPROBE = which('ffprobe') or which('avprobe')
if not FFPROBE:
FFPROBE = None
if CHECK_MEDIA:

View file

@ -192,10 +192,9 @@ def extract(file_path, output_destination):
cmd2 = cmd
if 'gunzip' not in cmd: # gunzip doesn't support password
cmd2.append('-p-') # don't prompt for password.
p = Popen(
res = Popen(
cmd2, stdout=devnull, stderr=devnull, startupinfo=info,
) # should extract files fine.
res = p.wait()
).wait() # should extract files fine.
if res == 0: # Both Linux and Windows return 0 for successful.
log.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination}')
success = 1
@ -210,10 +209,10 @@ def extract(file_path, output_destination):
# append password here.
passcmd = f'-p{password}'
cmd2.append(passcmd)
p = Popen(
proc = Popen(
cmd2, stdout=devnull, stderr=devnull, startupinfo=info,
) # should extract files fine.
res = p.wait()
)
res = proc.wait() # should extract files fine.
if (res >= 0 and platform == 'Windows') or res == 0:
log.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination} using password: {password}')
success = 1

View file

@ -2,10 +2,10 @@ from __future__ import annotations
import logging
import os
import platform
import re
import shlex
import subprocess
from subprocess import DEVNULL
import nzb2media
from nzb2media.utils.files import list_media_files
@ -212,10 +212,6 @@ def par2(dirname):
if nzb2media.PAR2CMD and parfile:
pwd = os.getcwd() # Get our Present Working Directory
os.chdir(dirname) # set directory to run par on.
if platform.system() == 'Windows':
bitbucket = open('NUL')
else:
bitbucket = open('/dev/null')
log.info(f'Running par2 on file {parfile}.')
command = [nzb2media.PAR2CMD, 'r', parfile, '*']
cmd = ''
@ -223,9 +219,7 @@ def par2(dirname):
cmd = f'{cmd} {item}'
log.debug(f'calling command:{cmd}')
try:
proc = subprocess.Popen(
command, stdout=bitbucket, stderr=bitbucket,
)
proc = subprocess.Popen(command, stdout=DEVNULL, stderr=DEVNULL)
proc.communicate()
result = proc.returncode
except Exception:
@ -233,7 +227,6 @@ def par2(dirname):
if result == 0:
log.info('par2 file processing succeeded')
os.chdir(pwd)
bitbucket.close()
# dict for custom groups

View file

@ -11,6 +11,7 @@ import shutil
import subprocess
import sys
import time
from subprocess import PIPE, DEVNULL
from babelfish import Language
@ -100,22 +101,20 @@ def is_video_good(video: pathlib.Path, status, require_lan=None):
return False
def zip_out(file, img, bitbucket):
procin = None
def zip_out(file, img):
proc = None
if os.path.isfile(file):
cmd = ['cat', file]
else:
cmd = [nzb2media.SEVENZIP, '-so', 'e', img, file]
try:
procin = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=bitbucket,
)
proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL)
except Exception:
log.error(f'Extracting [{file}] has failed')
return procin
return proc
def get_video_details(videofile, img=None, bitbucket=None):
def get_video_details(videofile, img=None):
video_details = {}
result = 1
file = videofile
@ -138,13 +137,11 @@ def get_video_details(videofile, img=None, bitbucket=None):
]
print_cmd(command)
if img:
procin = zip_out(file, img, bitbucket)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stdin=procin.stdout,
)
procin = zip_out(file, img)
proc = subprocess.Popen(command, stdout=PIPE, stdin=procin.stdout)
procin.stdout.close()
else:
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
proc = subprocess.Popen(command, stdout=PIPE)
out, err = proc.communicate()
result = proc.returncode
video_details = json.loads(out.decode())
@ -162,13 +159,11 @@ def get_video_details(videofile, img=None, bitbucket=None):
]
print_cmd(command)
if img:
procin = zip_out(file, img, bitbucket)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stdin=procin.stdout,
)
procin = zip_out(file, img)
proc = subprocess.Popen(command, stdout=PIPE, stdin=procin.stdout)
procin.stdout.close()
else:
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
proc = subprocess.Popen(command, stdout=PIPE)
out, err = proc.communicate()
result = proc.returncode
video_details = json.loads(out.decode())
@ -200,7 +195,7 @@ def check_vid_file(video_details, result):
return False
def build_commands(file, new_dir, movie_name, bitbucket):
def build_commands(file, new_dir, movie_name):
if isinstance(file, str):
input_file = file
if 'concat:' in file:
@ -228,16 +223,14 @@ def build_commands(file, new_dir, movie_name, bitbucket):
new_file = []
rem_vid = []
for vid in data['files']:
video_details, result = get_video_details(vid, img, bitbucket)
video_details, result = get_video_details(vid, img)
if not check_vid_file(
video_details, result,
): # lets not transcode menu or other clips that don't have audio and video.
rem_vid.append(vid)
data['files'] = [f for f in data['files'] if f not in rem_vid]
new_file = {img: {'name': data['name'], 'files': data['files']}}
video_details, result = get_video_details(
data['files'][0], img, bitbucket,
)
video_details, result = get_video_details(data['files'][0], img)
input_file = '-'
file = '-'
@ -752,7 +745,7 @@ def get_subs(file):
return subfiles
def extract_subs(file, newfile_path, bitbucket):
def extract_subs(file, newfile_path):
video_details, result = get_video_details(file)
if not video_details:
return
@ -815,9 +808,9 @@ def extract_subs(file, newfile_path, bitbucket):
result = 1 # set result to failed in case call fails.
try:
proc = subprocess.Popen(
command, stdout=bitbucket, stderr=bitbucket,
command, stdout=DEVNULL, stderr=DEVNULL,
)
out, err = proc.communicate()
proc_out, proc_error = proc.communicate()
result = proc.returncode
except Exception:
log.error('Extracting subtitle has failed')
@ -832,7 +825,7 @@ def extract_subs(file, newfile_path, bitbucket):
log.error('Extracting subtitles has failed')
def process_list(it, new_dir, bitbucket):
def process_list(it, new_dir):
rem_list = []
new_list = []
combine = []
@ -846,7 +839,7 @@ def process_list(it, new_dir, bitbucket):
and ext not in nzb2media.IGNOREEXTENSIONS
):
log.debug(f'Attempting to rip disk image: {item}')
new_list.extend(rip_iso(item, new_dir, bitbucket))
new_list.extend(rip_iso(item, new_dir))
rem_list.append(item)
elif (
re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', item)
@ -907,9 +900,7 @@ def process_list(it, new_dir, bitbucket):
return it, rem_list, new_list, success
def mount_iso(
item, new_dir, bitbucket,
): # Currently only supports Linux Mount when permissions allow.
def mount_iso(item, new_dir): # Currently only supports Linux Mount when permissions allow.
if platform.system() == 'Windows':
log.error(f'No mounting options available under Windows for image file {item}')
return []
@ -917,7 +908,7 @@ def mount_iso(
make_dir(mount_point)
cmd = ['mount', '-o', 'loop', item, mount_point]
print_cmd(cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket)
proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL)
out, err = proc.communicate()
nzb2media.MOUNTED = (
mount_point # Allows us to verify this has been done and then cleanup.
@ -951,16 +942,15 @@ def mount_iso(
return ['failure'] # If we got here, nothing matched our criteria
def rip_iso(item, new_dir, bitbucket):
def rip_iso(item, new_dir):
new_files = []
failure_dir = 'failure'
# Mount the ISO in your OS and call combineVTS.
if not nzb2media.SEVENZIP:
log.debug(f'No 7zip installed. Attempting to mount image file {item}')
try:
new_files = mount_iso(
item, new_dir, bitbucket,
) # Currently only works for Linux.
# Currently only works for Linux.
new_files = mount_iso(item, new_dir)
except Exception:
log.error(f'Failed to mount and extract from image file {item}')
new_files = [failure_dir]
@ -969,7 +959,7 @@ def rip_iso(item, new_dir, bitbucket):
try:
log.debug(f'Attempting to extract .vob or .mts from image file {item}')
print_cmd(cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket)
proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL)
out, err = proc.communicate()
file_match_gen = (
re.match(
@ -1040,7 +1030,7 @@ def rip_iso(item, new_dir, bitbucket):
new_files.append({item: {'name': name, 'files': combined}})
if not new_files:
log.error(f'No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {item}')
new_files = mount_iso(item, new_dir, bitbucket)
new_files = mount_iso(item, new_dir)
except Exception:
log.error(f'Failed to extract from image file {item}')
new_files = [failure_dir]
@ -1159,19 +1149,12 @@ def transcode_directory(dir_name):
make_dir(new_dir)
else:
new_dir = dir_name
if platform.system() == 'Windows':
bitbucket = open('NUL')
else:
bitbucket = open('/dev/null')
movie_name = os.path.splitext(os.path.split(dir_name)[1])[0]
file_list = nzb2media.list_media_files(
dir_name, media=True, audio=False, meta=False, archives=False,
)
file_list, rem_list, new_list, success = process_list(
file_list, new_dir, bitbucket,
)
file_list, rem_list, new_list, success = process_list(file_list, new_dir)
if not success:
bitbucket.close()
return 1, dir_name
for file in file_list:
@ -1180,12 +1163,12 @@ def transcode_directory(dir_name):
and os.path.splitext(file)[1] in nzb2media.IGNOREEXTENSIONS
):
continue
command, file = build_commands(file, new_dir, movie_name, bitbucket)
command, file = build_commands(file, new_dir, movie_name)
newfile_path = command[-1]
# transcoding files may remove the original file, so make sure to extract subtitles first
if nzb2media.SEXTRACT and isinstance(file, str):
extract_subs(file, newfile_path, bitbucket)
extract_subs(file, newfile_path)
try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason)
os.remove(newfile_path)
@ -1202,19 +1185,12 @@ def transcode_directory(dir_name):
result = 1 # set result to failed in case call fails.
try:
if isinstance(file, str):
proc = subprocess.Popen(
command, stdout=bitbucket, stderr=subprocess.PIPE,
)
proc = subprocess.Popen(command, stdout=DEVNULL, stderr=PIPE)
else:
img, data = next(file.items())
proc = subprocess.Popen(
command,
stdout=bitbucket,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
proc = subprocess.Popen(command, stdout=DEVNULL, stderr=PIPE, stdin=PIPE)
for vob in data['files']:
procin = zip_out(vob, img, bitbucket)
procin = zip_out(vob, img)
if procin:
log.debug(f'Feeding in file: {vob} to Transcoder')
shutil.copyfileobj(procin.stdout, proc.stdin)
@ -1258,7 +1234,7 @@ def transcode_directory(dir_name):
time.sleep(5) # play it safe and avoid failing to unmount.
cmd = ['umount', '-l', nzb2media.MOUNTED]
print_cmd(cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket)
proc = subprocess.Popen(cmd, stdout=PIPE, stderr=DEVNULL)
out, err = proc.communicate()
time.sleep(5)
os.rmdir(nzb2media.MOUNTED)
@ -1278,5 +1254,4 @@ def transcode_directory(dir_name):
not nzb2media.PROCESSOUTPUT and nzb2media.DUPLICATE
): # We postprocess the original files to CP/SB
new_dir = dir_name
bitbucket.close()
return final_result, new_dir

View file

@ -118,8 +118,8 @@ def external_script(output_destination, torrent_name, torrent_label, settings):
cmd = f'{cmd} {item}'
log.info(f'Running script {cmd} on file {file_path}.')
try:
p = Popen(command)
res = p.wait()
proc = Popen(command)
res = proc.wait()
if (
str(res) in nzb2media.USER_SCRIPT_SUCCESSCODES
): # Linux returns 0 for successful.

View file

@ -111,8 +111,8 @@ def restart():
if popen_list:
popen_list += nzb2media.SYS_ARGV
log.info(f'Restarting nzbToMedia with {popen_list}')
p = subprocess.Popen(popen_list, cwd=os.getcwd())
p.wait()
status = p.returncode
proc = subprocess.Popen(popen_list, cwd=os.getcwd())
proc.wait()
status = proc.returncode
os._exit(status)

View file

@ -11,6 +11,7 @@ import stat
import subprocess
import tarfile
import traceback
from subprocess import PIPE, STDOUT
from urllib.request import urlretrieve
import nzb2media
@ -161,52 +162,52 @@ class GitUpdateManager(UpdateManager):
def _run_git(self, git_path, args):
output = None
err = None
proc_out = None
proc_err = None
if not git_path:
log.debug('No git specified, can\'t use git commands')
exit_status = 1
return output, err, exit_status
proc_status = 1
return proc_out, proc_err, proc_status
cmd = f'{git_path} {args}'
try:
log.debug(f'Executing {cmd} with your shell in {nzb2media.APP_ROOT}')
p = subprocess.Popen(
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
shell=True,
cwd=nzb2media.APP_ROOT,
)
output, err = p.communicate()
exit_status = p.returncode
proc_out, proc_err = proc.communicate()
proc_status = proc.returncode
output = output.decode('utf-8')
proc_out = proc_out.decode('utf-8')
if output:
output = output.strip()
if proc_out:
proc_out = proc_out.strip()
if nzb2media.LOG_GIT:
log.debug(f'git output: {output}')
log.debug(f'git output: {proc_out}')
except OSError:
log.error(f'Command {cmd} didn\'t work')
exit_status = 1
proc_status = 1
exit_status = 128 if ('fatal:' in output) or err else exit_status
if exit_status == 0:
proc_status = 128 if ('fatal:' in proc_out) or proc_err else proc_status
if proc_status == 0:
log.debug(f'{cmd} : returned successful')
exit_status = 0
elif nzb2media.LOG_GIT and exit_status in (1, 128):
log.debug(f'{cmd} returned : {output}')
proc_status = 0
elif nzb2media.LOG_GIT and proc_status in (1, 128):
log.debug(f'{cmd} returned : {proc_out}')
else:
if nzb2media.LOG_GIT:
log.debug(f'{cmd} returned : {output}, treat as error for now')
exit_status = 1
log.debug(f'{cmd} returned : {proc_out}, treat as error for now')
proc_status = 1
return output, err, exit_status
return proc_out, proc_err, proc_status
def _find_installed_version(self):
"""