diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 30a399a0..6829d6b2 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -418,7 +418,7 @@ generalOptions = # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, leave this blank and set the remaining options below. - # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release + # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release outputDefault = #### Define custom settings below. outputVideoExtension = .mp4 diff --git a/core/__init__.py b/core/__init__.py index 4621ecfb..7fb32140 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -199,6 +199,7 @@ META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] +MOUNTED = None GETSUBS = False TRANSCODE = None CONCAT = None @@ -504,6 +505,7 @@ def configure_containers(): def configure_transcoder(): + global MOUNTED global GETSUBS global TRANSCODE global DUPLICATE @@ -548,6 +550,7 @@ def configure_transcoder(): global ALLOWSUBS global DEFAULTS + MOUNTED = None GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) DUPLICATE = int(CFG['Transcoder']['duplicate']) @@ -751,7 +754,15 @@ def configure_transcoder(): }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'SCODEC': 'mov_text' + }, + 'mkv-bluray': { + 'VEXTENSION': '.mkv', 'VCODEC': 'libx265', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'hevc', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, diff --git a/core/transcoder.py b/core/transcoder.py index 3228c810..e1850de2 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -9,7 +9,9 @@ from __future__ import ( import errno import json +import sys import os +import time import platform import re import shutil @@ -73,7 +75,10 @@ def is_video_good(videofile, status): def zip_out(file, img, bitbucket): procin = None - cmd = [core.SEVENZIP, '-so', 'e', img, file] + if os.path.isfile(file): + cmd = ['cat', file] + else: + cmd = [core.SEVENZIP, '-so', 'e', img, file] try: procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) except Exception: @@ -104,12 +109,11 @@ def get_video_details(videofile, img=None, bitbucket=None): result = proc.returncode video_details = json.loads(out.decode()) except Exception: - pass - if not video_details: - try: + try: # try this again without -show error in case of ffmpeg limitation command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] + print_cmd(command) if img: - procin = zip_out(file, img) + procin = zip_out(file, img, bitbucket) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) procin.stdout.close() else: @@ -122,6 +126,21 @@ def get_video_details(videofile, img=None, bitbucket=None): return video_details, result +def check_vid_file(video_details, result): + if result != 0: + return False + if video_details.get('error'): + return False + if not video_details.get('streams'): + return False + video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] + audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + if len(video_streams) > 0 and len(audio_streams) > 0: + return True + else: + return False + + def build_commands(file, new_dir, movie_name, bitbucket): if isinstance(file, string_types): input_file = file @@ -139,9 +158,18 @@ def build_commands(file, new_dir, movie_name, bitbucket): name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext' + new_file = file else: img, data = next(iteritems(file)) name = data['name'] + new_file = [] + rem_vid = [] + for vid in data['files']: + video_details, result = get_video_details(vid, img, bitbucket) + if not check_vid_file(video_details, result): #lets not transcode menu or other clips that don't have audio and video. + rem_vid.append(vid) + data['files'] = [ f for f in data['files'] if f not in rem_vid ] + new_file = {img: {'name': data['name'], 'files': data['files']}} video_details, result = get_video_details(data['files'][0], img, bitbucket) input_file = '-' file = '-' @@ -518,7 +546,7 @@ def build_commands(file, new_dir, movie_name, bitbucket): command.append(newfile_path) if platform.system() != 'Windows': command = core.NICENESS + command - return command + return command, new_file def get_subs(file): @@ -597,6 +625,7 @@ def process_list(it, new_dir, bitbucket): new_list = [] combine = [] vts_path = None + mts_path = None success = True for item in it: ext = os.path.splitext(item)[1].lower() @@ -612,6 +641,14 @@ def process_list(it, new_dir, bitbucket): except Exception: vts_path = os.path.split(item)[0] rem_list.append(item) + elif re.match('.+BDMV[/\\]SOURCE[/\\][0-9]+[0-9].[Mm][Tt][Ss]', item) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(item), 'TRANSCODER') + if not mts_path: + try: + mts_path = re.match('(.+BDMV[/\\]SOURCE)', item).groups()[0] + except Exception: + mts_path = os.path.split(item)[0] + rem_list.append(item) elif re.match('.+VIDEO_TS.', item) or re.match('.+VTS_[0-9][0-9]_[0-9].', item): rem_list.append(item) elif core.CONCAT and re.match('.+[cC][dD][0-9].', item): @@ -621,6 +658,8 @@ def process_list(it, new_dir, bitbucket): continue if vts_path: new_list.extend(combine_vts(vts_path)) + if mts_path: + new_list.extend(combine_mts(mts_path)) if combine: new_list.extend(combine_cd(combine)) for file in new_list: @@ -639,17 +678,53 @@ def process_list(it, new_dir, bitbucket): return it, rem_list, new_list, success +def mount_iso(item, new_dir, bitbucket): #Currently only supports Linux Mount when permissions allow. + if platform.system() == 'Windows': + logger.error('No mounting options available under Windows for image file {0}'.format(item), 'TRANSCODER') + return [] + mount_point = os.path.join(os.path.dirname(os.path.abspath(item)),'temp') + make_dir(mount_point) + cmd = ['mount', '-o', 'loop', item, mount_point] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + core.MOUNTED = mount_point # Allows us to verify this has been done and then cleanup. + for root, dirs, files in os.walk(mount_point): + for file in files: + full_path = os.path.join(root, file) + if re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', full_path) and '.vob' not in core.IGNOREEXTENSIONS: + logger.debug('Found VIDEO_TS image file: {0}'.format(full_path), 'TRANSCODER') + try: + vts_path = re.match('(.+VIDEO_TS)', full_path).groups()[0] + except Exception: + vts_path = os.path.split(full_path)[0] + return combine_vts(vts_path) + elif re.match('.+BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]', full_path) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(full_path), 'TRANSCODER') + try: + mts_path = re.match('(.+BDMV[/\\]STREAM)', full_path).groups()[0] + except Exception: + mts_path = os.path.split(full_path)[0] + return combine_mts(mts_path) + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file {0}'.format(mount_point), 'TRANSCODER') + return ['failure'] # If we got here, nothing matched our criteria + + def rip_iso(item, new_dir, bitbucket): new_files = [] failure_dir = 'failure' # Mount the ISO in your OS and call combineVTS. if not core.SEVENZIP: - logger.error('No 7zip installed. Can\'t extract image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.debug('No 7zip installed. Attempting to mount image file {0}'.format(item), 'TRANSCODER') + try: + new_files = mount_iso(item, new_dir, bitbucket) # Currently only works for Linux. + except Exception: + logger.error('Failed to mount and extract from image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] return new_files cmd = [core.SEVENZIP, 'l', item] try: - logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') + logger.debug('Attempting to extract .vob or .mts from image file {0}'.format(item), 'TRANSCODER') print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() @@ -663,31 +738,58 @@ def rip_iso(item, new_dir, bitbucket): if file_match ] combined = [] - for n in range(99): - concat = [] - m = 1 - while True: - vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) - if vts_name in file_list: - concat.append(vts_name) - m += 1 - else: + if file_list: # handle DVD + for n in range(99): + concat = [] + m = 1 + while True: + vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) + if vts_name in file_list: + concat.append(vts_name) + m += 1 + else: + break + if not concat: break - if not concat: - break - if core.CONCAT: - combined.extend(concat) - continue - name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + ) + new_files.append({item: {'name': name, 'files': concat}}) + else: #check BlueRay for BDMV/STREAM/XXXX.MTS + mts_list_gen = ( + re.match(r'.+(BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]).', line) + for line in out.decode().splitlines() ) - new_files.append({item: {'name': name, 'files': concat}}) - if core.CONCAT: + mts_list = [ + file_match.groups()[0] + for file_match in mts_list_gen + if file_match + ] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) # Sort all .mts files in numerical order + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + n = 0 + for mts_name in mts_list: + concat = [] + n += 1 + concat.append(mts_name) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + ) + new_files.append({item: {'name': name, 'files': concat}}) + if core.CONCAT and combined: name = os.path.splitext(os.path.split(item)[1])[0] new_files.append({item: {'name': name, 'files': combined}}) if not new_files: - logger.error('No VIDEO_TS folder found in image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {0}'.format(item), 'TRANSCODER') + new_files = mount_iso(item, new_dir, bitbucket) except Exception: logger.error('Failed to extract from image file {0}'.format(item), 'TRANSCODER') new_files = [failure_dir] @@ -696,25 +798,63 @@ def rip_iso(item, new_dir, bitbucket): def combine_vts(vts_path): new_files = [] - combined = '' + combined = [] + name = re.match(r'(.+)[/\\]VIDEO_TS', vts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) for n in range(99): - concat = '' + concat = [] m = 1 while True: vts_name = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m) if os.path.isfile(os.path.join(vts_path, vts_name)): - concat += '{file}|'.format(file=os.path.join(vts_path, vts_name)) + concat.append(os.path.join(vts_path, vts_name)) m += 1 else: break if not concat: break if core.CONCAT: - combined += '{files}|'.format(files=concat) + combined.extend(concat) continue - new_files.append('concat:{0}'.format(concat[:-1])) + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({vts_path: {'name': name, 'files': concat}}) if core.CONCAT: - new_files.append('concat:{0}'.format(combined[:-1])) + new_files.append({vts_path: {'name': name, 'files': combined}}) + return new_files + + +def combine_mts(mts_path): + new_files = [] + combined = [] + name = re.match(r'(.+)[/\\]BDMV[/\\]STREAM', mts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) + n = 0 + mts_list = [f for f in os.listdir(mts_path) if os.path.isfile(os.path.join(mts_path, f))] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + for mts_name in mts_list: ### need to sort all files [1 - 998].mts in order + concat = [] + concat.append(os.path.join(mts_path, mts_name)) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({mts_path: {'name': name, 'files': concat}}) + n += 1 + if core.CONCAT: + new_files.append({mts_path: {'name': name, 'files': combined}}) return new_files @@ -768,7 +908,7 @@ def transcode_directory(dir_name): for file in file_list: if isinstance(file, string_types) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: continue - command = build_commands(file, new_dir, movie_name, bitbucket) + command, file = build_commands(file, new_dir, movie_name, bitbucket) newfile_path = command[-1] # transcoding files may remove the original file, so make sure to extract subtitles first @@ -795,6 +935,7 @@ def transcode_directory(dir_name): for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: + logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() proc.communicate() @@ -826,6 +967,15 @@ def transcode_directory(dir_name): logger.error('Transcoding of video to {0} failed with result {1}'.format(newfile_path, result)) # this will be 0 (successful) it all are successful, else will return a positive integer for failure. final_result = final_result + result + if core.MOUNTED: # In case we mounted an .iso file, unmount here. + time.sleep(5) # play it safe and avoid failing to unmount. + cmd = ['umount', '-l', core.MOUNTED] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + time.sleep(5) + os.rmdir(core.MOUNTED) + core.MOUNTED = None if final_result == 0 and not core.DUPLICATE: for file in rem_list: try: diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 16785b46..862e7f78 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -202,7 +202,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToLidarr.py b/nzbToLidarr.py index 9681a408..c38f5975 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -187,7 +187,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToMedia.py b/nzbToMedia.py index 03f19a8d..12a72ae6 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -558,7 +558,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index fbc3de0c..680074cd 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -192,7 +192,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToRadarr.py b/nzbToRadarr.py index 504bbe52..f75ca350 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -197,7 +197,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 43d23d3d..00f18077 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -203,7 +203,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below.