diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 0f0ce7b7..28cd7e8f 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -57,8 +57,10 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp logger.debug(f'Received Directory: {input_directory} | Name: {input_name} | Category: {input_category}') # Confirm the category by parsing directory structure - input_directory, input_name, input_category, root = core.category_search(input_directory, input_name, input_category, - root, core.CATEGORIES) + input_directory, input_name, input_category, root = core.category_search( + input_directory, input_name, input_category, + root, core.CATEGORIES, + ) if input_category == '': input_category = 'UNCAT' @@ -110,10 +112,12 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp output_destination = os.path.join(core.OUTPUT_DIRECTORY, input_category, basename) elif unique_path: output_destination = os.path.normpath( - core.os.path.join(core.OUTPUT_DIRECTORY, input_category, core.sanitize_name(input_name).replace(' ', '.'))) + core.os.path.join(core.OUTPUT_DIRECTORY, input_category, core.sanitize_name(input_name).replace(' ', '.')), + ) else: output_destination = os.path.normpath( - core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) + core.os.path.join(core.OUTPUT_DIRECTORY, input_category), + ) if output_destination in input_directory: output_destination = input_directory @@ -128,7 +132,8 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if section_name in ['HeadPhones', 'Lidarr']: core.NOFLATTEN.extend( - input_category) # Make sure we preserve folder structure for HeadPhones. + input_category, + ) # Make sure we preserve folder structure for HeadPhones. now = datetime.datetime.now() @@ -150,13 +155,16 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if input_category in core.NOFLATTEN: if not os.path.basename(file_path) in output_destination: target_file = core.os.path.join( - core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) + core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name, + ) logger.debug(f'Setting outputDestination to {os.path.dirname(target_file)} to preserve folder structure') if root == 1: if not found_file: logger.debug(f'Looking for {input_name} in: {inputFile}') - if any([core.sanitize_name(input_name) in core.sanitize_name(inputFile), - core.sanitize_name(file_name) in core.sanitize_name(input_name)]): + if any([ + core.sanitize_name(input_name) in core.sanitize_name(inputFile), + core.sanitize_name(file_name) in core.sanitize_name(input_name), + ]): found_file = True logger.debug(f'Found file {full_file_name} that matches Torrent Name {input_name}') else: @@ -194,7 +202,8 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # Now check if video files exist in destination: if section_name in ['SickBeard', 'SiCKRAGE', 'NzbDrone', 'Sonarr', 'CouchPotato', 'Radarr', 'Watcher3']: num_videos = len( - core.list_media_files(output_destination, media=True, audio=False, meta=False, archives=False)) + core.list_media_files(output_destination, media=True, audio=False, meta=False, archives=False), + ) if num_videos > 0: logger.info(f'Found {num_videos} media files in {output_destination}') status = 0 @@ -248,11 +257,15 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if result.status_code != 0: if not core.TORRENT_RESUME_ON_FAILURE: - logger.error('A problem was reported in the autoProcess* script. ' - 'Torrent won\'t resume seeding (settings)') + logger.error( + 'A problem was reported in the autoProcess* script. ' + 'Torrent won\'t resume seeding (settings)', + ) elif client_agent != 'manual': - logger.error('A problem was reported in the autoProcess* script. ' - 'If torrent was paused we will resume seeding') + logger.error( + 'A problem was reported in the autoProcess* script. ' + 'If torrent was paused we will resume seeding', + ) core.resume_torrent(client_agent, input_hash, input_id, input_name) else: @@ -336,8 +349,10 @@ def main(args): input_name = os.path.basename(dir_name) - results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, - client_agent) + results = process_torrent( + dir_name, input_name, subsection, input_hash or None, input_id or None, + client_agent, + ) if results.status_code != 0: logger.error(f'A problem was reported when trying to perform a manual run for {section}:{subsection}.') result = results diff --git a/core/__init__.py b/core/__init__.py index fdf22f60..84c4d9aa 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -81,7 +81,15 @@ __version__ = '12.1.11' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] -TORRENT_CLIENTS = ['transmission', 'deluge', 'utorrent', 'rtorrent', 'qbittorrent', 'other', 'manual'] +TORRENT_CLIENTS = [ + 'transmission', + 'deluge', + 'utorrent', + 'rtorrent', + 'qbittorrent', + 'other', + 'manual', +] # sickbeard fork/branch constants FORK_DEFAULT = 'default' @@ -101,21 +109,109 @@ FORKS = { FORK_DEFAULT: {'dir': None}, FORK_FAILED: {'dirName': None, 'failed': None}, FORK_FAILED_TORRENT: {'dir': None, 'failed': None, 'process_method': None}, - FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, - FORK_SICKCHILL_API: {'path': None, 'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'is_priority': None, 'cmd': 'postprocess'}, - FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'cmd': 'postprocess'}, - FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, - FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None, 'cmd': 'postprocess'}, - FORK_MEDUSA_APIV2: {'proc_dir': None, 'resource': None, 'failed': None, 'process_method': None, 'force': None, 'type': None, 'delete_on': None, 'is_priority': None}, - FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, - FORK_SICKGEAR_API: {'path': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'is_priority': None, 'failed': None, 'cmd': 'sg.postprocess'}, - FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, + FORK_SICKCHILL: { + 'proc_dir': None, + 'failed': None, + 'process_method': None, + 'force': None, + 'delete_on': None, + 'force_next': None, + }, + FORK_SICKCHILL_API: { + 'path': None, + 'proc_dir': None, + 'failed': None, + 'process_method': None, + 'force': None, + 'force_replace': None, + 'return_data': None, + 'type': None, + 'delete': None, + 'force_next': None, + 'is_priority': None, + 'cmd': 'postprocess', + }, + FORK_SICKBEARD_API: { + 'path': None, + 'failed': None, + 'process_method': None, + 'force_replace': None, + 'return_data': None, + 'type': None, + 'delete': None, + 'force_next': None, + 'cmd': 'postprocess', + }, + FORK_MEDUSA: { + 'proc_dir': None, + 'failed': None, + 'process_method': None, + 'force': None, + 'delete_on': None, + 'ignore_subs': None, + }, + FORK_MEDUSA_API: { + 'path': None, + 'failed': None, + 'process_method': None, + 'force_replace': None, + 'return_data': None, + 'type': None, + 'delete_files': None, + 'is_priority': None, + 'cmd': 'postprocess', + }, + FORK_MEDUSA_APIV2: { + 'proc_dir': None, + 'resource': None, + 'failed': None, + 'process_method': None, + 'force': None, + 'type': None, + 'delete_on': None, + 'is_priority': None, + }, + FORK_SICKGEAR: { + 'dir': None, + 'failed': None, + 'process_method': None, + 'force': None, + }, + FORK_SICKGEAR_API: { + 'path': None, + 'process_method': None, + 'force_replace': None, + 'return_data': None, + 'type': None, + 'is_priority': None, + 'failed': None, + 'cmd': 'sg.postprocess', + }, + FORK_STHENO: { + 'proc_dir': None, + 'failed': None, + 'process_method': None, + 'force': None, + 'delete_on': None, + 'ignore_subs': None, + }, +} +ALL_FORKS = { + k: None + for k in set( + list( + itertools.chain.from_iterable( + [FORKS[x].keys() for x in FORKS.keys()], + ), + ), + ) } -ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} # SiCKRAGE OAuth2 SICKRAGE_OAUTH_CLIENT_ID = 'nzbtomedia' -SICKRAGE_OAUTH_TOKEN_URL = 'https://auth.sickrage.ca/realms/sickrage/protocol/openid-connect/token' +SICKRAGE_OAUTH_TOKEN_URL = ( + 'https://auth.sickrage.ca/realms/sickrage/protocol/openid-connect/token' +) # NZBGet Exit Codes NZBGET_POSTPROCESS_PAR_CHECK = 92 @@ -305,7 +401,11 @@ def configure_locale(): pass # For OSes that are poorly configured I'll just randomly force UTF-8 - if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + if not SYS_ENCODING or SYS_ENCODING in ( + 'ANSI_X3.4-1968', + 'US-ASCII', + 'ASCII', + ): SYS_ENCODING = 'UTF-8' @@ -315,7 +415,9 @@ def configure_migration(): # run migrate to convert old cfg to new style cfg plus fix any cfg missing values/options. if not config.migrate(): - logger.error(f'Unable to migrate config file {CONFIG_FILE}, exiting ...') + logger.error( + f'Unable to migrate config file {CONFIG_FILE}, exiting ...', + ) if 'NZBOP_SCRIPTDIR' in os.environ: pass # We will try and read config from Environment. else: @@ -371,7 +473,11 @@ def configure_general(): FFMPEG_PATH = CFG['General']['ffmpeg_path'] SYS_PATH = CFG['General']['sys_path'] CHECK_MEDIA = int(CFG['General']['check_media']) - REQUIRE_LAN = None if not CFG['General']['require_lan'] else CFG['General']['require_lan'].split(',') + REQUIRE_LAN = ( + None + if not CFG['General']['require_lan'] + else CFG['General']['require_lan'].split(',') + ) SAFE_MODE = int(CFG['General']['safe_mode']) NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) @@ -394,7 +500,9 @@ def configure_updates(): pass restart() else: - logger.error('Update failed, not restarting. Check your log for more information.') + logger.error( + 'Update failed, not restarting. Check your log for more information.', + ) # Set Current Version logger.info( @@ -426,7 +534,9 @@ def configure_remote_paths(): if REMOTE_PATHS: if isinstance(REMOTE_PATHS, list): - REMOTE_PATHS = ','.join(REMOTE_PATHS) # fix in case this imported as list. + REMOTE_PATHS = ','.join( + REMOTE_PATHS, + ) # fix in case this imported as list. REMOTE_PATHS = ( # /volume1/Public/,E:\|/volume2/share/,\\NAS\ @@ -446,16 +556,22 @@ def configure_niceness(): with open(os.devnull, 'w') as devnull: try: - subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() + subprocess.Popen( + ['nice'], stdout=devnull, stderr=devnull, + ).communicate() niceness = CFG['Posix']['niceness'] - if len(niceness.split(',')) > 1: # Allow passing of absolute command, not just value. + if ( + len(niceness.split(',')) > 1 + ): # Allow passing of absolute command, not just value. NICENESS.extend(niceness.split(',')) else: NICENESS.extend(['nice', f'-n{int(niceness)}']) except Exception: pass try: - subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() + subprocess.Popen( + ['ionice'], stdout=devnull, stderr=devnull, + ).communicate() try: ionice = CFG['Posix']['ionice_class'] NICENESS.extend(['ionice', f'-c{int(ionice)}']) @@ -485,8 +601,8 @@ def configure_containers(): re.compile('.rar$', re.I), ] COMPRESSED_CONTAINER += [ - re.compile(f'{ext}$', re.I) for ext in - CFG['Extensions']['compressedExtensions'] + re.compile(f'{ext}$', re.I) + for ext in CFG['Extensions']['compressedExtensions'] ] MEDIA_CONTAINER = CFG['Extensions']['mediaExtensions'] AUDIO_CONTAINER = CFG['Extensions']['audioExtensions'] @@ -557,11 +673,11 @@ def configure_transcoder(): TRANSCODE = int(CFG['Transcoder']['transcode']) DUPLICATE = int(CFG['Transcoder']['duplicate']) CONCAT = int(CFG['Transcoder']['concat']) - IGNOREEXTENSIONS = (CFG['Transcoder']['ignoreExtensions']) + IGNOREEXTENSIONS = CFG['Transcoder']['ignoreExtensions'] if isinstance(IGNOREEXTENSIONS, str): IGNOREEXTENSIONS = IGNOREEXTENSIONS.split(',') OUTPUTFASTSTART = int(CFG['Transcoder']['outputFastStart']) - GENERALOPTS = (CFG['Transcoder']['generalOptions']) + GENERALOPTS = CFG['Transcoder']['generalOptions'] if isinstance(GENERALOPTS, str): GENERALOPTS = GENERALOPTS.split(',') if GENERALOPTS == ['']: @@ -570,7 +686,7 @@ def configure_transcoder(): GENERALOPTS.append('-fflags') if '+genpts' not in GENERALOPTS: GENERALOPTS.append('+genpts') - OTHEROPTS = (CFG['Transcoder']['otherOptions']) + OTHEROPTS = CFG['Transcoder']['otherOptions'] if isinstance(OTHEROPTS, str): OTHEROPTS = OTHEROPTS.split(',') if OTHEROPTS == ['']: @@ -613,7 +729,11 @@ def configure_transcoder(): except Exception: pass try: - VBITRATE = int((CFG['Transcoder']['outputVideoBitrate'].strip()).replace('k', '000')) + VBITRATE = int( + (CFG['Transcoder']['outputVideoBitrate'].strip()).replace( + 'k', '000', + ), + ) except Exception: pass VRESOLUTION = CFG['Transcoder']['outputVideoResolution'] @@ -628,7 +748,11 @@ def configure_transcoder(): except Exception: pass try: - ABITRATE = int((CFG['Transcoder']['outputAudioBitrate'].strip()).replace('k', '000')) + ABITRATE = int( + (CFG['Transcoder']['outputAudioBitrate'].strip()).replace( + 'k', '000', + ), + ) except Exception: pass ACODEC2 = CFG['Transcoder']['outputAudioTrack2Codec'].strip() @@ -638,11 +762,17 @@ def configure_transcoder(): if ACODEC2_ALLOW == ['']: ACODEC2_ALLOW = [] try: - ACHANNELS2 = int(CFG['Transcoder']['outputAudioTrack2Channels'].strip()) + ACHANNELS2 = int( + CFG['Transcoder']['outputAudioTrack2Channels'].strip(), + ) except Exception: pass try: - ABITRATE2 = int((CFG['Transcoder']['outputAudioTrack2Bitrate'].strip()).replace('k', '000')) + ABITRATE2 = int( + (CFG['Transcoder']['outputAudioTrack2Bitrate'].strip()).replace( + 'k', '000', + ), + ) except Exception: pass ACODEC3 = CFG['Transcoder']['outputAudioOtherCodec'].strip() @@ -656,7 +786,11 @@ def configure_transcoder(): except Exception: pass try: - ABITRATE3 = int((CFG['Transcoder']['outputAudioOtherBitrate'].strip()).replace('k', '000')) + ABITRATE3 = int( + (CFG['Transcoder']['outputAudioOtherBitrate'].strip()).replace( + 'k', '000', + ), + ) except Exception: pass SCODEC = CFG['Transcoder']['outputSubtitleCodec'].strip() @@ -672,123 +806,505 @@ def configure_transcoder(): } transcode_defaults = { 'iPad': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': None, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'iPad-1080p': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '1920:1080', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '1920:1080', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': None, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'iPad-720p': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '1280:720', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': None, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'Apple-TV': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, - 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '1280:720', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'ac3', + 'ACODEC_ALLOW': ['ac3'], + 'ABITRATE': None, + 'ACHANNELS': 6, + 'ACODEC2': 'aac', + 'ACODEC2_ALLOW': ['libfaac'], + 'ABITRATE2': None, + 'ACHANNELS2': 2, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'iPod': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '1280:720', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 128000, + 'ACHANNELS': 2, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'iPhone': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '460:320', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '460:320', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 128000, + 'ACHANNELS': 2, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'PS3': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, - 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'ac3', + 'ACODEC_ALLOW': ['ac3'], + 'ABITRATE': None, + 'ACHANNELS': 6, + 'ACODEC2': 'aac', + 'ACODEC2_ALLOW': ['libfaac'], + 'ABITRATE2': None, + 'ACHANNELS2': 2, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'xbox': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'ac3', + 'ACODEC_ALLOW': ['ac3'], + 'ABITRATE': None, + 'ACHANNELS': 6, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'Roku-480p': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 128000, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'Roku-720p': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 128000, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'Roku-1080p': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 160000, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 160000, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, 'mkv': { - 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], - 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'VEXTENSION': '.mkv', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + 'mpeg2video', + ], + 'ACODEC': 'dts', + 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE': None, + 'ACHANNELS': 8, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': 'ac3', + 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE3': None, + 'ACHANNELS3': 8, 'SCODEC': 'mov_text', }, 'mkv-bluray': { - 'VEXTENSION': '.mkv', 'VCODEC': 'libx265', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'hevc', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], - 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'VEXTENSION': '.mkv', + 'VCODEC': 'libx265', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'hevc', + 'h265', + 'libx265', + 'h.265', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + 'mpeg2video', + ], + 'ACODEC': 'dts', + 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE': None, + 'ACHANNELS': 8, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': 'ac3', + 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE3': None, + 'ACHANNELS3': 8, 'SCODEC': 'mov_text', }, 'mp4-scene-release': { - 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': 19, 'VLEVEL': '3.1', - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], - 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, - 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, - 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'VEXTENSION': '.mp4', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': None, + 'VCRF': 19, + 'VLEVEL': '3.1', + 'VRESOLUTION': None, + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + 'mpeg2video', + ], + 'ACODEC': 'dts', + 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE': None, + 'ACHANNELS': 8, + 'ACODEC2': None, + 'ACODEC2_ALLOW': [], + 'ABITRATE2': None, + 'ACHANNELS2': None, + 'ACODEC3': 'ac3', + 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], + 'ABITRATE3': None, + 'ACHANNELS3': 8, 'SCODEC': 'mov_text', }, 'MKV-SD': { - 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': '1200k', 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': '720: -1', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, - 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, - 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'VEXTENSION': '.mkv', + 'VCODEC': 'libx264', + 'VPRESET': None, + 'VFRAMERATE': None, + 'VBITRATE': '1200k', + 'VCRF': None, + 'VLEVEL': None, + 'VRESOLUTION': '720: -1', + 'VCODEC_ALLOW': [ + 'libx264', + 'h264', + 'h.264', + 'AVC', + 'avc', + 'mpeg4', + 'msmpeg4', + 'MPEG-4', + ], + 'ACODEC': 'aac', + 'ACODEC_ALLOW': ['libfaac'], + 'ABITRATE': 128000, + 'ACHANNELS': 2, + 'ACODEC2': 'ac3', + 'ACODEC2_ALLOW': ['ac3'], + 'ABITRATE2': None, + 'ACHANNELS2': 6, + 'ACODEC3': None, + 'ACODEC3_ALLOW': [], + 'ABITRATE3': None, + 'ACHANNELS3': None, 'SCODEC': 'mov_text', }, } @@ -816,7 +1332,10 @@ def configure_transcoder(): ABITRATE3 = transcode_defaults[DEFAULTS]['ABITRATE3'] SCODEC = transcode_defaults[DEFAULTS]['SCODEC'] transcode_defaults = {} # clear memory - if transcode_defaults in ['mp4-scene-release'] and not OUTPUTQUALITYPERCENT: + if ( + transcode_defaults in ['mp4-scene-release'] + and not OUTPUTQUALITYPERCENT + ): OUTPUTQUALITYPERCENT = 100 if VEXTENSION in allow_subs: @@ -825,25 +1344,37 @@ def configure_transcoder(): VCODEC_ALLOW.extend([VCODEC]) for codec in VCODEC_ALLOW: if codec in codec_alias: - extra = [item for item in codec_alias[codec] if item not in VCODEC_ALLOW] + extra = [ + item for item in codec_alias[codec] if item not in VCODEC_ALLOW + ] VCODEC_ALLOW.extend(extra) if not ACODEC_ALLOW and ACODEC: ACODEC_ALLOW.extend([ACODEC]) for codec in ACODEC_ALLOW: if codec in codec_alias: - extra = [item for item in codec_alias[codec] if item not in ACODEC_ALLOW] + extra = [ + item for item in codec_alias[codec] if item not in ACODEC_ALLOW + ] ACODEC_ALLOW.extend(extra) if not ACODEC2_ALLOW and ACODEC2: ACODEC2_ALLOW.extend([ACODEC2]) for codec in ACODEC2_ALLOW: if codec in codec_alias: - extra = [item for item in codec_alias[codec] if item not in ACODEC2_ALLOW] + extra = [ + item + for item in codec_alias[codec] + if item not in ACODEC2_ALLOW + ] ACODEC2_ALLOW.extend(extra) if not ACODEC3_ALLOW and ACODEC3: ACODEC3_ALLOW.extend([ACODEC3]) for codec in ACODEC3_ALLOW: if codec in codec_alias: - extra = [item for item in codec_alias[codec] if item not in ACODEC3_ALLOW] + extra = [ + item + for item in codec_alias[codec] + if item not in ACODEC3_ALLOW + ] ACODEC3_ALLOW.extend(extra) @@ -859,10 +1390,17 @@ def configure_sections(section): # check for script-defied section and if None set to allow sections SECTIONS = CFG[ tuple(x for x in CFG if CFG[x].sections and CFG[x].isenabled()) - if not section else (section,) + if not section + else (section,) ] for section, subsections in SECTIONS.items(): - CATEGORIES.extend([subsection for subsection in subsections if CFG[section][subsection].isenabled()]) + CATEGORIES.extend( + [ + subsection + for subsection in subsections + if CFG[section][subsection].isenabled() + ], + ) CATEGORIES = list(set(CATEGORIES)) @@ -877,35 +1415,60 @@ def configure_utility_locations(): if platform.system() == 'Windows': FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg.exe') FFPROBE = os.path.join(FFMPEG_PATH, 'ffprobe.exe') - SEVENZIP = os.path.join(APP_ROOT, 'core', 'extractor', 'bin', platform.machine(), '7z.exe') + SEVENZIP = os.path.join( + APP_ROOT, 'core', 'extractor', 'bin', platform.machine(), '7z.exe', + ) SHOWEXTRACT = int(str(CFG['Windows']['show_extraction']), 0) if not (os.path.isfile(FFMPEG)): # problem FFMPEG = None - logger.warning('Failed to locate ffmpeg.exe. Transcoding disabled!') - logger.warning('Install ffmpeg with x264 support to enable this feature ...') + logger.warning( + 'Failed to locate ffmpeg.exe. Transcoding disabled!', + ) + logger.warning( + 'Install ffmpeg with x264 support to enable this feature ...', + ) if not (os.path.isfile(FFPROBE)): FFPROBE = None if CHECK_MEDIA: - logger.warning('Failed to locate ffprobe.exe. Video corruption detection disabled!') - logger.warning('Install ffmpeg with x264 support to enable this feature ...') + logger.warning( + 'Failed to locate ffprobe.exe. Video corruption detection disabled!', + ) + logger.warning( + 'Install ffmpeg with x264 support to enable this feature ...', + ) else: if SYS_PATH: os.environ['PATH'] += ':' + SYS_PATH try: - SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + SEVENZIP = ( + subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not SEVENZIP: try: - SEVENZIP = subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + SEVENZIP = ( + subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not SEVENZIP: try: - SEVENZIP = subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + SEVENZIP = ( + subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not SEVENZIP: @@ -914,7 +1477,12 @@ def configure_utility_locations(): 'Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!', ) try: - PAR2CMD = subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + PAR2CMD = ( + subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not PAR2CMD: @@ -934,18 +1502,34 @@ def configure_utility_locations(): FFMPEG = os.path.join(FFMPEG_PATH, 'avconv') else: try: - FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + FFMPEG = ( + subprocess.Popen( + ['which', 'ffmpeg'], stdout=subprocess.PIPE, + ) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not FFMPEG: try: - FFMPEG = subprocess.Popen(['which', 'avconv'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + FFMPEG = ( + subprocess.Popen( + ['which', 'avconv'], stdout=subprocess.PIPE, + ) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not FFMPEG: FFMPEG = None logger.warning('Failed to locate ffmpeg. Transcoding disabled!') - logger.warning('Install ffmpeg with x264 support to enable this feature ...') + logger.warning( + 'Install ffmpeg with x264 support to enable this feature ...', + ) if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access( os.path.join(FFMPEG_PATH, 'ffprobe'), @@ -959,19 +1543,37 @@ def configure_utility_locations(): FFPROBE = os.path.join(FFMPEG_PATH, 'avprobe') else: try: - FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + FFPROBE = ( + subprocess.Popen( + ['which', 'ffprobe'], stdout=subprocess.PIPE, + ) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not FFPROBE: try: - FFPROBE = subprocess.Popen(['which', 'avprobe'], stdout=subprocess.PIPE).communicate()[0].strip().decode() + FFPROBE = ( + subprocess.Popen( + ['which', 'avprobe'], stdout=subprocess.PIPE, + ) + .communicate()[0] + .strip() + .decode() + ) except Exception: pass if not FFPROBE: FFPROBE = None if CHECK_MEDIA: - logger.warning('Failed to locate ffprobe. Video corruption detection disabled!') - logger.warning('Install ffmpeg with x264 support to enable this feature ...') + logger.warning( + 'Failed to locate ffprobe. Video corruption detection disabled!', + ) + logger.warning( + 'Install ffmpeg with x264 support to enable this feature ...', + ) def check_python(): diff --git a/core/auto_process/books.py b/core/auto_process/books.py index f1b9db17..d7accbef 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -94,19 +94,29 @@ def process( logger.postprocess(f'{r.text}', section) - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status ' f'{r.status_code}', ) elif r.text == 'OK': - logger.postprocess(f'SUCCESS: ForceProcess for {dir_name} has been started in LazyLibrarian', section) + logger.postprocess( + f'SUCCESS: ForceProcess for {dir_name} has been started in LazyLibrarian', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) else: - logger.error(f'FAILED: ForceProcess of {dir_name} has Failed in LazyLibrarian', section) + logger.error( + f'FAILED: ForceProcess of {dir_name} has Failed in LazyLibrarian', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Returned log from {section} ' f'was not as expected.', diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 00e77bf0..c667ad78 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -96,14 +96,20 @@ def process( logger.debug(f'Opening URL: {url}', section) try: - r = requests.post(url, params=params, stream=True, verify=False, timeout=(30, 300)) + r = requests.post( + url, params=params, stream=True, verify=False, timeout=(30, 300), + ) except requests.ConnectionError: logger.error('Unable to open URL', section) return ProcessResult.failure( f'{section}: Failed to post-process - Unable to connect to ' f'{section}', ) - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status ' @@ -120,12 +126,17 @@ def process( success = True if success: - logger.postprocess('SUCCESS: This issue has been processed successfully', section) + logger.postprocess( + 'SUCCESS: This issue has been processed successfully', section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) else: - logger.warning('The issue does not appear to have successfully processed. Please check your Logs', section) + logger.warning( + 'The issue does not appear to have successfully processed. Please check your Logs', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Returned log from ' f'{section} was not as expected.', diff --git a/core/auto_process/common.py b/core/auto_process/common.py index b2e3237c..4b1c4c49 100644 --- a/core/auto_process/common.py +++ b/core/auto_process/common.py @@ -32,11 +32,22 @@ class ProcessResult(typing.NamedTuple): def command_complete(url, params, headers, section): try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) + r = requests.get( + url, + params=params, + headers=headers, + stream=True, + verify=False, + timeout=(30, 60), + ) except requests.ConnectionError: logger.error(f'Unable to open URL: {url}', section) return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return None else: @@ -44,17 +55,30 @@ def command_complete(url, params, headers, section): return r.json()['status'] except (ValueError, KeyError): # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error(f'{section} did not return expected json data.', section) + logger.error( + f'{section} did not return expected json data.', section, + ) return None def completed_download_handling(url2, headers, section='MAIN'): try: - r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) + r = requests.get( + url2, + params={}, + headers=headers, + stream=True, + verify=False, + timeout=(30, 60), + ) except requests.ConnectionError: logger.error(f'Unable to open URL: {url2}', section) return False - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return False else: diff --git a/core/auto_process/games.py b/core/auto_process/games.py index 44dcbc02..db125795 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -69,7 +69,9 @@ def process( url = core.utils.common.create_url(scheme, host, port, web_root) if not server_responding(url): logger.error('Server did not respond. Exiting', section) - return ProcessResult.failure(f'{section}: Failed to post-process - {section} did not respond.') + return ProcessResult.failure( + f'{section}: Failed to post-process - {section} did not respond.', + ) input_name, dir_name = convert_to_ascii(input_name, dir_name) @@ -104,30 +106,46 @@ def process( try: shutil.move(dir_name, os.path.join(library, input_name)) except Exception: - logger.error(f'Unable to move {dir_name} to {os.path.join(library, input_name)}', section) + logger.error( + f'Unable to move {dir_name} to {os.path.join(library, input_name)}', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Unable to move files', ) else: - logger.error('No library specified to move files to. Please edit your configuration.', section) + logger.error( + 'No library specified to move files to. Please edit your configuration.', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - No library defined in ' f'{section}', ) - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status ' f'{r.status_code}', ) elif result['success']: - logger.postprocess(f'SUCCESS: Status for {gamez_id} has been set to {download_status} in Gamez', section) + logger.postprocess( + f'SUCCESS: Status for {gamez_id} has been set to {download_status} in Gamez', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) else: - logger.error(f'FAILED: Status for {gamez_id} has NOT been updated in Gamez', section) + logger.error( + f'FAILED: Status for {gamez_id} has NOT been updated in Gamez', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Returned log from {section} ' f'was not as expected.', diff --git a/core/auto_process/managers/sickbeard.py b/core/auto_process/managers/sickbeard.py index 32324fab..a1c32e94 100644 --- a/core/auto_process/managers/sickbeard.py +++ b/core/auto_process/managers/sickbeard.py @@ -402,7 +402,8 @@ class SickBeard: self.delete_failed = int(self.sb_init.config.get('delete_failed', 0)) self.nzb_extraction_by = self.sb_init.config.get( - 'nzbExtractionBy', 'Downloader', + 'nzbExtractionBy', + 'Downloader', ) self.process_method = self.sb_init.config.get('process_method') self.remote_path = int(self.sb_init.config.get('remote_path', 0)) @@ -581,7 +582,8 @@ class SickBeard: ) except requests.ConnectionError: logger.error( - f'Unable to open URL: {self.url}', self.sb_init.section, + f'Unable to open URL: {self.url}', + self.sb_init.section, ) result = ProcessResult.failure( f'{self.sb_init.section}: Failed to post-process - Unable to ' diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 590e9585..0755f591 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -95,7 +95,9 @@ def process( route = web_root base_url = core.utils.common.create_url(scheme, host, port, route) if not apikey: - logger.info('No CouchPotato or Radarr apikey entered. Performing transcoder functions only') + logger.info( + 'No CouchPotato or Radarr apikey entered. Performing transcoder functions only', + ) release = None elif server_responding(base_url): if section == 'CouchPotato': @@ -104,7 +106,9 @@ def process( release = None else: logger.error('Server did not respond. Exiting', section) - return ProcessResult.failure(f'{section}: Failed to post-process - {section} did not respond.') + return ProcessResult.failure( + f'{section}: Failed to post-process - {section} did not respond.', + ) # pull info from release found if available release_id = None @@ -121,7 +125,9 @@ def process( except Exception: pass - if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + if not os.path.isdir(dir_name) and os.path.isfile( + dir_name, + ): # If the input directory is a file, assume single file download and split dir/name. dir_name = os.path.split(os.path.normpath(dir_name))[0] specific_path = os.path.join(dir_name, str(input_name)) @@ -134,8 +140,18 @@ def process( process_all_exceptions(input_name, dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) - if not list_media_files(dir_name, media=True, audio=False, meta=False, archives=False) and list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug(f'Checking for archives to extract in directory: {dir_name}') + if ( + not list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ) + and list_media_files( + dir_name, media=False, audio=False, meta=False, archives=True, + ) + and extract + ): + logger.debug( + f'Checking for archives to extract in directory: {dir_name}', + ) core.extract_files(dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) @@ -143,68 +159,115 @@ def process( valid_files = 0 num_files = 0 # Check video files for corruption - for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + for video in list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ): num_files += 1 if transcoder.is_video_good(video, status): good_files += 1 - if not core.REQUIRE_LAN or transcoder.is_video_good(video, status, require_lan=core.REQUIRE_LAN): + if not core.REQUIRE_LAN or transcoder.is_video_good( + video, status, require_lan=core.REQUIRE_LAN, + ): valid_files += 1 import_subs(video) rename_subs(dir_name) if num_files and valid_files == num_files: if status: - logger.info(f'Status shown as failed from Downloader, but {good_files} valid video files found. Setting as success.', section) + logger.info( + f'Status shown as failed from Downloader, but {good_files} valid video files found. Setting as success.', + section, + ) status = 0 elif num_files and valid_files < num_files: - logger.info('Status shown as success from Downloader, but corrupt video files found. Setting as failed.', section) + logger.info( + 'Status shown as success from Downloader, but corrupt video files found. Setting as failed.', + section, + ) status = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + if ( + 'NZBOP_VERSION' in os.environ + and os.environ['NZBOP_VERSION'][0:5] >= '14.0' + ): print('[NZB] MARK=BAD') if good_files == num_files: - logger.debug(f'Video marked as failed due to missing required language: {core.REQUIRE_LAN}', section) + logger.debug( + f'Video marked as failed due to missing required language: {core.REQUIRE_LAN}', + section, + ) else: - logger.debug('Video marked as failed due to missing playable audio or video', section) - if good_files < num_files and failure_link: # only report corrupt files + logger.debug( + 'Video marked as failed due to missing playable audio or video', + section, + ) + if ( + good_files < num_files and failure_link + ): # only report corrupt files failure_link += '&corrupt=true' elif client_agent == 'manual': - logger.warning(f'No media files found in directory {dir_name} to manually process.', section) + logger.warning( + f'No media files found in directory {dir_name} to manually process.', + section, + ) return ProcessResult( message='', status_code=0, # Success (as far as this script is concerned) ) else: - logger.warning(f'No media files found in directory {dir_name}. Processing this as a failed download', section) + logger.warning( + f'No media files found in directory {dir_name}. Processing this as a failed download', + section, + ) status = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + if ( + 'NZBOP_VERSION' in os.environ + and os.environ['NZBOP_VERSION'][0:5] >= '14.0' + ): print('[NZB] MARK=BAD') if status == 0: if core.TRANSCODE == 1: result, new_dir_name = transcoder.transcode_directory(dir_name) if result == 0: - logger.debug(f'Transcoding succeeded for files in {dir_name}', section) + logger.debug( + f'Transcoding succeeded for files in {dir_name}', section, + ) dir_name = new_dir_name - logger.debug(f'Config setting \'chmodDirectory\' currently set to {oct(chmod_directory)}', section) + logger.debug( + f'Config setting \'chmodDirectory\' currently set to {oct(chmod_directory)}', + section, + ) if chmod_directory: - logger.info(f'Attempting to set the octal permission of \'{oct(chmod_directory)}\' on directory \'{dir_name}\'', section) + logger.info( + f'Attempting to set the octal permission of \'{oct(chmod_directory)}\' on directory \'{dir_name}\'', + section, + ) core.rchmod(dir_name, chmod_directory) else: - logger.error(f'Transcoding failed for files in {dir_name}', section) + logger.error( + f'Transcoding failed for files in {dir_name}', section, + ) return ProcessResult( message=f'{section}: Failed to post-process - Transcoding failed', status_code=1, ) - for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + for video in list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ): if not release and '.cp(tt' not in video and imdbid: video_name, video_ext = os.path.splitext(video) video2 = f'{video_name}.cp({imdbid}){video_ext}' - if not (client_agent in [core.TORRENT_CLIENT_AGENT, 'manual'] and core.USE_LINK == 'move-sym'): + if not ( + client_agent in [core.TORRENT_CLIENT_AGENT, 'manual'] + and core.USE_LINK == 'move-sym' + ): logger.debug(f'Renaming: {video} to: {video2}') os.rename(video, video2) if not apikey: # If only using Transcoder functions, exit here. - logger.info('No CouchPotato or Radarr or Watcher3 apikey entered. Processing completed.') + logger.info( + 'No CouchPotato or Radarr or Watcher3 apikey entered. Processing completed.', + ) return ProcessResult( message=f'{section}: Successfully post-processed {input_name}', status_code=0, @@ -227,47 +290,91 @@ def process( url = f'{base_url}{command}' logger.debug(f'Opening URL: {url} with PARAMS: {params}', section) - logger.postprocess(f'Starting {method} scan for {input_name}', section) + logger.postprocess( + f'Starting {method} scan for {input_name}', section, + ) if section == 'Radarr': - payload = {'name': 'DownloadedMoviesScan', 'path': params['media_folder'], 'downloadClientId': download_id, 'importMode': import_mode} + payload = { + 'name': 'DownloadedMoviesScan', + 'path': params['media_folder'], + 'downloadClientId': download_id, + 'importMode': import_mode, + } if not download_id: payload.pop('downloadClientId') - logger.debug(f'Opening URL: {base_url} with PARAMS: {payload}', section) - logger.postprocess(f'Starting DownloadedMoviesScan scan for {input_name}', section) + logger.debug( + f'Opening URL: {base_url} with PARAMS: {payload}', section, + ) + logger.postprocess( + f'Starting DownloadedMoviesScan scan for {input_name}', section, + ) if section == 'Watcher3': - if input_name and os.path.isfile(os.path.join(dir_name, input_name)): - params['media_folder'] = os.path.join(params['media_folder'], input_name) - payload = {'apikey': apikey, 'path': params['media_folder'], 'guid': download_id, 'mode': 'complete'} + if input_name and os.path.isfile( + os.path.join(dir_name, input_name), + ): + params['media_folder'] = os.path.join( + params['media_folder'], input_name, + ) + payload = { + 'apikey': apikey, + 'path': params['media_folder'], + 'guid': download_id, + 'mode': 'complete', + } if not download_id: payload.pop('guid') - logger.debug(f'Opening URL: {base_url} with PARAMS: {payload}', section) - logger.postprocess(f'Starting postprocessing scan for {input_name}', section) + logger.debug( + f'Opening URL: {base_url} with PARAMS: {payload}', section, + ) + logger.postprocess( + f'Starting postprocessing scan for {input_name}', section, + ) try: if section == 'CouchPotato': - r = requests.get(url, params=params, verify=False, timeout=(30, 1800)) + r = requests.get( + url, params=params, verify=False, timeout=(30, 1800), + ) elif section == 'Watcher3': - r = requests.post(base_url, data=payload, verify=False, timeout=(30, 1800)) + r = requests.post( + base_url, data=payload, verify=False, timeout=(30, 1800), + ) else: - r = requests.post(base_url, data=json.dumps(payload), headers=headers, stream=True, verify=False, timeout=(30, 1800)) + r = requests.post( + base_url, + data=json.dumps(payload), + headers=headers, + stream=True, + verify=False, + timeout=(30, 1800), + ) except requests.ConnectionError: logger.error('Unable to open URL', section) return ProcessResult( - message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + message='{0}: Failed to post-process - Unable to connect to {0}'.format( + section, + ), status_code=1, ) result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult( message=f'{section}: Failed to post-process - Server returned status {r.status_code}', status_code=1, ) elif section == 'CouchPotato' and result['success']: - logger.postprocess(f'SUCCESS: Finished {method} scan for folder {dir_name}', section) + logger.postprocess( + f'SUCCESS: Finished {method} scan for folder {dir_name}', + section, + ) if method == 'manage': return ProcessResult( message=f'{section}: Successfully post-processed {input_name}', @@ -304,33 +411,60 @@ def process( ) else: core.FAILED = True - logger.postprocess(f'FAILED DOWNLOAD DETECTED FOR {input_name}', section) + logger.postprocess( + f'FAILED DOWNLOAD DETECTED FOR {input_name}', section, + ) if failure_link: report_nzb(failure_link, client_agent) if section == 'Radarr': - logger.postprocess(f'SUCCESS: Sending failed download to {section} for CDH processing', section) + logger.postprocess( + f'SUCCESS: Sending failed download to {section} for CDH processing', + section, + ) return ProcessResult( - message='{0}: Sending failed download back to {0}'.format(section), + message='{0}: Sending failed download back to {0}'.format( + section, + ), status_code=1, # Return as failed to flag this in the downloader. ) # Return failed flag, but log the event as successful. elif section == 'Watcher3': - logger.postprocess(f'Sending failed download to {section} for CDH processing', section) + logger.postprocess( + f'Sending failed download to {section} for CDH processing', + section, + ) path = remote_dir(dir_name) if remote_path else dir_name - if input_name and os.path.isfile(os.path.join(dir_name, input_name)): + if input_name and os.path.isfile( + os.path.join(dir_name, input_name), + ): path = os.path.join(path, input_name) - payload = {'apikey': apikey, 'path': path, 'guid': download_id, 'mode': 'failed'} - r = requests.post(base_url, data=payload, verify=False, timeout=(30, 1800)) + payload = { + 'apikey': apikey, + 'path': path, + 'guid': download_id, + 'mode': 'failed', + } + r = requests.post( + base_url, data=payload, verify=False, timeout=(30, 1800), + ) result = r.json() logger.postprocess(f'Watcher3 response: {result}') if result['status'] == 'finished': return ProcessResult( - message='{0}: Sending failed download back to {0}'.format(section), + message='{0}: Sending failed download back to {0}'.format( + section, + ), status_code=1, # Return as failed to flag this in the downloader. ) # Return failed flag, but log the event as successful. - if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: - logger.postprocess(f'Deleting failed files and folder {dir_name}', section) + if ( + delete_failed + and os.path.isdir(dir_name) + and not os.path.dirname(dir_name) == dir_name + ): + logger.postprocess( + f'Deleting failed files and folder {dir_name}', section, + ) remove_dir(dir_name) if not release_id and not media_id: @@ -339,12 +473,16 @@ def process( section, ) return ProcessResult( - message='{0}: Failed to post-process - Failed download not found in {0}'.format(section), + message='{0}: Failed to post-process - Failed download not found in {0}'.format( + section, + ), status_code=1, ) if release_id: - logger.postprocess(f'Setting failed release {input_name} to ignored ...', section) + logger.postprocess( + f'Setting failed release {input_name} to ignored ...', section, + ) url = f'{base_url}release.ignore' params = {'id': release_id} @@ -352,37 +490,59 @@ def process( logger.debug(f'Opening URL: {url} with PARAMS: {params}', section) try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + r = requests.get( + url, params=params, verify=False, timeout=(30, 120), + ) except requests.ConnectionError: logger.error(f'Unable to open URL {url}', section) return ProcessResult( - message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + message='{0}: Failed to post-process - Unable to connect to {0}'.format( + section, + ), status_code=1, ) result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error(f'Server returned status {r.status_code}', section) + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: + logger.error( + f'Server returned status {r.status_code}', section, + ) return ProcessResult( status_code=1, message=f'{section}: Failed to post-process - Server returned status {r.status_code}', ) elif result['success']: - logger.postprocess(f'SUCCESS: {input_name} has been set to ignored ...', section) + logger.postprocess( + f'SUCCESS: {input_name} has been set to ignored ...', + section, + ) else: - logger.warning(f'FAILED: Unable to set {input_name} to ignored!', section) + logger.warning( + f'FAILED: Unable to set {input_name} to ignored!', section, + ) return ProcessResult( message=f'{section}: Failed to post-process - Unable to set {input_name} to ignored', status_code=1, ) - logger.postprocess('Trying to snatch the next highest ranked release.', section) + logger.postprocess( + 'Trying to snatch the next highest ranked release.', section, + ) url = f'{base_url}movie.searcher.try_next' logger.debug(f'Opening URL: {url}', section) try: - r = requests.get(url, params={'media_id': media_id}, verify=False, timeout=(30, 600)) + r = requests.get( + url, + params={'media_id': media_id}, + verify=False, + timeout=(30, 600), + ) except requests.ConnectionError: logger.error(f'Unable to open URL {url}', section) return ProcessResult.failure( @@ -391,19 +551,28 @@ def process( ) result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status ' f'{r.status_code}', ) elif result['success']: - logger.postprocess('SUCCESS: Snatched the next highest release ...', section) + logger.postprocess( + 'SUCCESS: Snatched the next highest release ...', section, + ) return ProcessResult.success( f'{section}: Successfully snatched next highest release', ) else: - logger.postprocess('SUCCESS: Unable to find a new release to snatch now. CP will keep searching!', section) + logger.postprocess( + 'SUCCESS: Unable to find a new release to snatch now. CP will keep searching!', + section, + ) return ProcessResult.success( f'{section}: No new release found now. ' f'{section} will keep searching', @@ -411,7 +580,9 @@ def process( # Added a release that was not in the wanted list so confirm rename successful by finding this movie media.list. if not release: - download_id = None # we don't want to filter new releases based on this. + download_id = ( + None # we don't want to filter new releases based on this. + ) if no_status_check: return ProcessResult.success( @@ -422,7 +593,9 @@ def process( # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. timeout = time.time() + 60 * wait_for while time.time() < timeout: # only wait 2 (default) minutes, then return. - logger.postprocess('Checking for status change, please stand by ...', section) + logger.postprocess( + 'Checking for status change, please stand by ...', section, + ) if section == 'CouchPotato': release = get_release(base_url, imdbid, download_id, release_id) scan_id = None @@ -432,15 +605,23 @@ def process( try: release_id = list(release.keys())[0] release_status_new = release[release_id]['status'] - if release_status_old is None: # we didn't have a release before, but now we do. + if ( + release_status_old is None + ): # we didn't have a release before, but now we do. title = release[release_id]['title'] - logger.postprocess(f'SUCCESS: Movie {title} has now been added to CouchPotato with release status of [{str(release_status_new).upper()}]', section) + logger.postprocess( + f'SUCCESS: Movie {title} has now been added to CouchPotato with release status of [{str(release_status_new).upper()}]', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) if release_status_new != release_status_old: - logger.postprocess(f'SUCCESS: Release {release_id} has now been marked with a status of [{str(release_status_new).upper()}]', section) + logger.postprocess( + f'SUCCESS: Release {release_id} has now been marked with a status of [{str(release_status_new).upper()}]', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) @@ -450,27 +631,44 @@ def process( url = f'{base_url}/{scan_id}' command_status = command_complete(url, params, headers, section) if command_status: - logger.debug(f'The Scan command return status: {command_status}', section) + logger.debug( + f'The Scan command return status: {command_status}', + section, + ) if command_status in ['completed']: - logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + logger.debug( + 'The Scan command has completed successfully. Renaming was successful.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) elif command_status in ['failed']: - logger.debug('The Scan command has failed. Renaming was not successful.', section) + logger.debug( + 'The Scan command has failed. Renaming was not successful.', + section, + ) # return ProcessResult( # message='{0}: Failed to post-process {1}'.format(section, input_name), # status_code=1, # ) if not os.path.isdir(dir_name): - logger.postprocess(f'SUCCESS: Input Directory [{dir_name}] has been processed and removed', section) + logger.postprocess( + f'SUCCESS: Input Directory [{dir_name}] has been processed and removed', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) - elif not list_media_files(dir_name, media=True, audio=False, meta=False, archives=True): - logger.postprocess(f'SUCCESS: Input Directory [{dir_name}] has no remaining media files. This has been fully processed.', section) + elif not list_media_files( + dir_name, media=True, audio=False, meta=False, archives=True, + ): + logger.postprocess( + f'SUCCESS: Input Directory [{dir_name}] has no remaining media files. This has been fully processed.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) @@ -479,8 +677,13 @@ def process( time.sleep(10 * wait_for) # The status hasn't changed. we have waited wait_for minutes which is more than enough. uTorrent can resume seeding now. - if section == 'Radarr' and completed_download_handling(url2, headers, section=section): - logger.debug(f'The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {section}.', section) + if section == 'Radarr' and completed_download_handling( + url2, headers, section=section, + ): + logger.debug( + f'The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {section}.', + section, + ) return ProcessResult.success( f'{section}: Complete DownLoad Handling is enabled. Passing back ' f'to {section}', @@ -562,7 +765,10 @@ def get_release(base_url, imdb_id=None, download_id=None, release_id=None): if release['status'] not in ['snatched', 'downloaded', 'done']: continue if download_id: - if download_id.lower() != release['download_info']['id'].lower(): + if ( + download_id.lower() + != release['download_info']['id'].lower() + ): continue cur_id = release['_id'] @@ -589,7 +795,10 @@ def get_release(base_url, imdb_id=None, download_id=None, release_id=None): rem_id = set() for cur_id, x in results.items(): try: - if not find_download(str(x['download_info']['downloader']).lower(), x['download_info']['id']): + if not find_download( + str(x['download_info']['downloader']).lower(), + x['download_info']['id'], + ): rem_id.add(cur_id) except Exception: continue diff --git a/core/auto_process/music.py b/core/auto_process/music.py index a9367e54..b70efd7f 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -80,7 +80,9 @@ def process( f'{section}: Failed to post-process - {section} did not respond.', ) - if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + if not os.path.isdir(dir_name) and os.path.isfile( + dir_name, + ): # If the input directory is a file, assume single file download and split dir/name. dir_name = os.path.split(os.path.normpath(dir_name))[0] specific_path = os.path.join(dir_name, str(input_name)) @@ -93,8 +95,18 @@ def process( process_all_exceptions(input_name, dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) - if not list_media_files(dir_name, media=False, audio=True, meta=False, archives=False) and list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug(f'Checking for archives to extract in directory: {dir_name}') + if ( + not list_media_files( + dir_name, media=False, audio=True, meta=False, archives=False, + ) + and list_media_files( + dir_name, media=False, audio=False, meta=False, archives=True, + ) + and extract + ): + logger.debug( + f'Checking for archives to extract in directory: {dir_name}', + ) core.extract_files(dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) @@ -110,22 +122,31 @@ def process( 'dir': remote_dir(dir_name) if remote_path else dir_name, } - res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) + res = force_process( + params, url, apikey, input_name, dir_name, section, wait_for, + ) if res.status_code in [0, 1]: return res params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0], + 'dir': os.path.split(remote_dir(dir_name))[0] + if remote_path + else os.path.split(dir_name)[0], } - res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) + res = force_process( + params, url, apikey, input_name, dir_name, section, wait_for, + ) if res.status_code in [0, 1]: return res # The status hasn't changed. uTorrent can resume seeding now. - logger.warning(f'The music album does not appear to have changed status after {wait_for} minutes. Please check your Logs', section) + logger.warning( + f'The music album does not appear to have changed status after {wait_for} minutes. Please check your Logs', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - No change in wanted status', ) @@ -143,7 +164,14 @@ def process( data = json.dumps(data) try: logger.debug(f'Opening URL: {url} with data: {data}', section) - r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) + r = requests.post( + url, + data=data, + headers=headers, + stream=True, + verify=False, + timeout=(30, 1800), + ) except requests.ConnectionError: logger.error(f'Unable to open URL: {url}', section) return ProcessResult.failure( @@ -171,41 +199,64 @@ def process( break n += 1 if command_status: - logger.debug(f'The Scan command return status: {command_status}', section) + logger.debug( + f'The Scan command return status: {command_status}', section, + ) if not os.path.exists(dir_name): - logger.debug(f'The directory {dir_name} has been removed. Renaming was successful.', section) + logger.debug( + f'The directory {dir_name} has been removed. Renaming was successful.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) elif command_status and command_status in ['completed']: - logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + logger.debug( + 'The Scan command has completed successfully. Renaming was successful.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) elif command_status and command_status in ['failed']: - logger.debug('The Scan command has failed. Renaming was not successful.', section) + logger.debug( + 'The Scan command has failed. Renaming was not successful.', + section, + ) # return ProcessResult.failure( # f'{section}: Failed to post-process {input_name}' # ) else: - logger.debug(f'The Scan command did not return status completed. Passing back to {section} to attempt complete download handling.', section) + logger.debug( + f'The Scan command did not return status completed. Passing back to {section} to attempt complete download handling.', + section, + ) return ProcessResult( message=f'{section}: Passing back to {section} to attempt ' - f'Complete Download Handling', + f'Complete Download Handling', status_code=status, ) else: if section == 'Lidarr': - logger.postprocess(f'FAILED: The download failed. Sending failed download to {section} for CDH processing', section) + logger.postprocess( + f'FAILED: The download failed. Sending failed download to {section} for CDH processing', + section, + ) # Return as failed to flag this in the downloader. return ProcessResult.failure( f'{section}: Download Failed. Sending back to {section}', ) else: logger.warning('FAILED DOWNLOAD DETECTED', section) - if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: - logger.postprocess(f'Deleting failed files and folder {dir_name}', section) + if ( + delete_failed + and os.path.isdir(dir_name) + and not os.path.dirname(dir_name) == dir_name + ): + logger.postprocess( + f'Deleting failed files and folder {dir_name}', section, + ) remove_dir(dir_name) # Return as failed to flag this in the downloader. return ProcessResult.failure( @@ -215,7 +266,9 @@ def process( def get_status(url, apikey, dir_name): - logger.debug(f'Attempting to get current status for release:{os.path.basename(dir_name)}') + logger.debug( + f'Attempting to get current status for release:{os.path.basename(dir_name)}', + ) params = { 'apikey': apikey, @@ -241,10 +294,15 @@ def get_status(url, apikey, dir_name): return album['Status'].lower() -def force_process(params, url, apikey, input_name, dir_name, section, wait_for): +def force_process( + params, url, apikey, input_name, dir_name, section, wait_for, +): release_status = get_status(url, apikey, dir_name) if not release_status: - logger.error(f'Could not find a status for {input_name}, is it in the wanted list ?', section) + logger.error( + f'Could not find a status for {input_name}, is it in the wanted list ?', + section, + ) logger.debug(f'Opening URL: {url} with PARAMS: {params}', section) @@ -259,15 +317,25 @@ def force_process(params, url, apikey, input_name, dir_name, section, wait_for): logger.debug(f'Result: {r.text}', section) - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status {r.status_code}', ) elif r.text == 'OK': - logger.postprocess(f'SUCCESS: Post-Processing started for {input_name} in folder {dir_name} ...', section) + logger.postprocess( + f'SUCCESS: Post-Processing started for {input_name} in folder {dir_name} ...', + section, + ) else: - logger.error(f'FAILED: Post-Processing has NOT started for {input_name} in folder {dir_name}. exiting!', section) + logger.error( + f'FAILED: Post-Processing has NOT started for {input_name} in folder {dir_name}. exiting!', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Returned log from {section} ' f'was not as expected.', @@ -277,13 +345,21 @@ def force_process(params, url, apikey, input_name, dir_name, section, wait_for): timeout = time.time() + 60 * wait_for while time.time() < timeout: current_status = get_status(url, apikey, dir_name) - if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie. - logger.postprocess(f'SUCCESS: This release is now marked as status [{current_status}]', section) + if ( + current_status is not None and current_status != release_status + ): # Something has changed. CPS must have processed this movie. + logger.postprocess( + f'SUCCESS: This release is now marked as status [{current_status}]', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) if not os.path.isdir(dir_name): - logger.postprocess(f'SUCCESS: The input directory {dir_name} has been removed Processing must have finished.', section) + logger.postprocess( + f'SUCCESS: The input directory {dir_name} has been removed Processing must have finished.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 26e58db3..188dd6ff 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -97,7 +97,9 @@ def process( # Should be changed after refactor. fork, fork_params = init_sickbeard.auto_fork() elif not username and not apikey and not sso_username: - logger.info('No SickBeard / SiCKRAGE username or Sonarr apikey entered. Performing transcoder functions only') + logger.info( + 'No SickBeard / SiCKRAGE username or Sonarr apikey entered. Performing transcoder functions only', + ) fork, fork_params = 'None', {} else: logger.error('Server did not respond. Exiting', section) @@ -105,9 +107,14 @@ def process( f'{section}: Failed to post-process - {section} did not respond.', ) - if client_agent == core.TORRENT_CLIENT_AGENT and core.USE_LINK == 'move-sym': + if ( + client_agent == core.TORRENT_CLIENT_AGENT + and core.USE_LINK == 'move-sym' + ): process_method = 'symlink' - if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + if not os.path.isdir(dir_name) and os.path.isfile( + dir_name, + ): # If the input directory is a file, assume single file download and split dir/name. dir_name = os.path.split(os.path.normpath(dir_name))[0] specific_path = os.path.join(dir_name, str(input_name)) @@ -128,30 +135,52 @@ def process( if e.errno != errno.EEXIST: raise - if 'process_method' not in fork_params or (client_agent in ['nzbget', 'sabnzbd'] and nzb_extraction_by != 'Destination'): + if 'process_method' not in fork_params or ( + client_agent in ['nzbget', 'sabnzbd'] + and nzb_extraction_by != 'Destination' + ): if input_name: process_all_exceptions(input_name, dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) # Now check if tv files exist in destination. - if not list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): - if list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug(f'Checking for archives to extract in directory: {dir_name}') + if not list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ): + if ( + list_media_files( + dir_name, + media=False, + audio=False, + meta=False, + archives=True, + ) + and extract + ): + logger.debug( + f'Checking for archives to extract in directory: {dir_name}', + ) core.extract_files(dir_name) input_name, dir_name = convert_to_ascii(input_name, dir_name) - if list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): # Check that a video exists. if not, assume failed. + if list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ): # Check that a video exists. if not, assume failed. flatten(dir_name) # Check video files for corruption good_files = 0 valid_files = 0 num_files = 0 - for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + for video in list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ): num_files += 1 if transcoder.is_video_good(video, status): good_files += 1 - if not core.REQUIRE_LAN or transcoder.is_video_good(video, status, require_lan=core.REQUIRE_LAN): + if not core.REQUIRE_LAN or transcoder.is_video_good( + video, status, require_lan=core.REQUIRE_LAN, + ): valid_files += 1 import_subs(video) rename_subs(dir_name) @@ -164,54 +193,93 @@ def process( logger.info('Found corrupt videos. Setting status Failed') status = 1 failed = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + if ( + 'NZBOP_VERSION' in os.environ + and os.environ['NZBOP_VERSION'][0:5] >= '14.0' + ): print('[NZB] MARK=BAD') if good_files == num_files: - logger.debug(f'Video marked as failed due to missing required language: {core.REQUIRE_LAN}', section) + logger.debug( + f'Video marked as failed due to missing required language: {core.REQUIRE_LAN}', + section, + ) else: - logger.debug('Video marked as failed due to missing playable audio or video', section) - if good_files < num_files and failure_link: # only report corrupt files + logger.debug( + 'Video marked as failed due to missing playable audio or video', + section, + ) + if ( + good_files < num_files and failure_link + ): # only report corrupt files failure_link += '&corrupt=true' elif client_agent == 'manual': - logger.warning(f'No media files found in directory {dir_name} to manually process.', section) + logger.warning( + f'No media files found in directory {dir_name} to manually process.', + section, + ) # Success (as far as this script is concerned) return ProcessResult.success() elif nzb_extraction_by == 'Destination': - logger.info('Check for media files ignored because nzbExtractionBy is set to Destination.') + logger.info( + 'Check for media files ignored because nzbExtractionBy is set to Destination.', + ) if int(failed) == 0: logger.info('Setting Status Success.') status = 0 failed = 0 else: - logger.info('Downloader reported an error during download or verification. Processing this as a failed download.') + logger.info( + 'Downloader reported an error during download or verification. Processing this as a failed download.', + ) status = 1 failed = 1 else: - logger.warning(f'No media files found in directory {dir_name}. Processing this as a failed download', section) + logger.warning( + f'No media files found in directory {dir_name}. Processing this as a failed download', + section, + ) status = 1 failed = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + if ( + 'NZBOP_VERSION' in os.environ + and os.environ['NZBOP_VERSION'][0:5] >= '14.0' + ): print('[NZB] MARK=BAD') - if status == 0 and core.TRANSCODE == 1: # only transcode successful downloads + if ( + status == 0 and core.TRANSCODE == 1 + ): # only transcode successful downloads result, new_dir_name = transcoder.transcode_directory(dir_name) if result == 0: - logger.debug(f'SUCCESS: Transcoding succeeded for files in {dir_name}', section) + logger.debug( + f'SUCCESS: Transcoding succeeded for files in {dir_name}', + section, + ) dir_name = new_dir_name - logger.debug(f'Config setting \'chmodDirectory\' currently set to {oct(chmod_directory)}', section) + logger.debug( + f'Config setting \'chmodDirectory\' currently set to {oct(chmod_directory)}', + section, + ) if chmod_directory: - logger.info(f'Attempting to set the octal permission of \'{oct(chmod_directory)}\' on directory \'{dir_name}\'', section) + logger.info( + f'Attempting to set the octal permission of \'{oct(chmod_directory)}\' on directory \'{dir_name}\'', + section, + ) core.rchmod(dir_name, chmod_directory) else: - logger.error(f'FAILED: Transcoding failed for files in {dir_name}', section) + logger.error( + f'FAILED: Transcoding failed for files in {dir_name}', section, + ) return ProcessResult.failure( f'{section}: Failed to post-process - Transcoding failed', ) # Part of the refactor if init_sickbeard.fork_obj: - init_sickbeard.fork_obj.initialize(dir_name, input_name, failed, client_agent='manual') + init_sickbeard.fork_obj.initialize( + dir_name, input_name, failed, client_agent='manual', + ) # configure SB params to pass # We don't want to remove params, for the Forks that have been refactored. @@ -238,12 +306,20 @@ def process( del fork_params['quiet'] if param == 'type': - if 'type' in fork_params: # only set if we haven't already deleted for 'failed' above. + if ( + 'type' in fork_params + ): # only set if we haven't already deleted for 'failed' above. fork_params[param] = 'manual' if 'proc_type' in fork_params: del fork_params['proc_type'] - if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']: + if param in [ + 'dir_name', + 'dir', + 'proc_dir', + 'process_directory', + 'path', + ]: fork_params[param] = dir_name if remote_path: fork_params[param] = remote_dir(dir_name) @@ -284,26 +360,46 @@ def process( return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) - logger.postprocess('SUCCESS: The download succeeded, sending a post-process request', section) + logger.postprocess( + 'SUCCESS: The download succeeded, sending a post-process request', + section, + ) else: core.FAILED = True if failure_link: report_nzb(failure_link, client_agent) if 'failed' in fork_params: - logger.postprocess(f'FAILED: The download failed. Sending \'failed\' process request to {fork} branch', section) + logger.postprocess( + f'FAILED: The download failed. Sending \'failed\' process request to {fork} branch', + section, + ) elif section == 'NzbDrone': - logger.postprocess(f'FAILED: The download failed. Sending failed download to {fork} for CDH processing', section) + logger.postprocess( + f'FAILED: The download failed. Sending failed download to {fork} for CDH processing', + section, + ) # Return as failed to flag this in the downloader. return ProcessResult.failure( f'{section}: Download Failed. Sending back to {section}', ) else: - logger.postprocess(f'FAILED: The download failed. {fork} branch does not handle failed downloads. Nothing to process', section) - if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: - logger.postprocess(f'Deleting failed files and folder {dir_name}', section) + logger.postprocess( + f'FAILED: The download failed. {fork} branch does not handle failed downloads. Nothing to process', + section, + ) + if ( + delete_failed + and os.path.isdir(dir_name) + and not os.path.dirname(dir_name) == dir_name + ): + logger.postprocess( + f'Deleting failed files and folder {dir_name}', section, + ) remove_dir(dir_name) # Return as failed to flag this in the downloader. - return ProcessResult.failure(f'{section}: Failed to post-process. {section} does not support failed downloads') + return ProcessResult.failure( + f'{section}: Failed to post-process. {section} does not support failed downloads', + ) route = '' if section == 'SickBeard': @@ -328,10 +424,20 @@ def process( # params = {'sortKey': 'series.title', 'page': 1, 'pageSize': 1, 'sortDir': 'asc'} if remote_path: logger.debug(f'remote_path: {remote_dir(dir_name)}', section) - data = {'name': 'DownloadedEpisodesScan', 'path': remote_dir(dir_name), 'downloadClientId': download_id, 'importMode': import_mode} + data = { + 'name': 'DownloadedEpisodesScan', + 'path': remote_dir(dir_name), + 'downloadClientId': download_id, + 'importMode': import_mode, + } else: logger.debug(f'path: {dir_name}', section) - data = {'name': 'DownloadedEpisodesScan', 'path': dir_name, 'downloadClientId': download_id, 'importMode': import_mode} + data = { + 'name': 'DownloadedEpisodesScan', + 'path': dir_name, + 'downloadClientId': download_id, + 'importMode': import_mode, + } if not download_id: data.pop('downloadClientId') data = json.dumps(data) @@ -343,34 +449,59 @@ def process( else: s = requests.Session() - logger.debug(f'Opening URL: {url} with params: {fork_params}', section) + logger.debug( + f'Opening URL: {url} with params: {fork_params}', section, + ) if not apikey and username and password: login = f'{web_root}/login' login_params = {'username': username, 'password': password} r = s.get(login, verify=False, timeout=(30, 60)) if r.status_code in [401, 403] and r.cookies.get('_xsrf'): login_params['_xsrf'] = r.cookies.get('_xsrf') - s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60)) - r = s.get(url, auth=(username, password), params=fork_params, stream=True, verify=False, timeout=(30, 1800)) + s.post( + login, + data=login_params, + stream=True, + verify=False, + timeout=(30, 60), + ) + r = s.get( + url, + auth=(username, password), + params=fork_params, + stream=True, + verify=False, + timeout=(30, 1800), + ) elif section == 'SiCKRAGE': s = requests.Session() if api_version >= 2 and sso_username and sso_password: - oauth = OAuth2Session(client=LegacyApplicationClient(client_id=core.SICKRAGE_OAUTH_CLIENT_ID)) + oauth = OAuth2Session( + client=LegacyApplicationClient( + client_id=core.SICKRAGE_OAUTH_CLIENT_ID, + ), + ) oauth_token = oauth.fetch_token( client_id=core.SICKRAGE_OAUTH_CLIENT_ID, token_url=core.SICKRAGE_OAUTH_TOKEN_URL, username=sso_username, password=sso_password, ) - s.headers.update({'Authorization': 'Bearer ' + oauth_token['access_token']}) + s.headers.update( + {'Authorization': 'Bearer ' + oauth_token['access_token']}, + ) params = { 'path': fork_params['path'], 'failed': str(bool(fork_params['failed'])).lower(), 'processMethod': 'move', - 'forceReplace': str(bool(fork_params['force_replace'])).lower(), - 'returnData': str(bool(fork_params['return_data'])).lower(), + 'forceReplace': str( + bool(fork_params['force_replace']), + ).lower(), + 'returnData': str( + bool(fork_params['return_data']), + ).lower(), 'delete': str(bool(fork_params['delete'])).lower(), 'forceNext': str(bool(fork_params['force_next'])).lower(), 'nzbName': fork_params['nzbName'], @@ -378,10 +509,23 @@ def process( else: params = fork_params - r = s.get(url, params=params, stream=True, verify=False, timeout=(30, 1800)) + r = s.get( + url, + params=params, + stream=True, + verify=False, + timeout=(30, 1800), + ) elif section == 'NzbDrone': logger.debug(f'Opening URL: {url} with data: {data}', section) - r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) + r = requests.post( + url, + data=data, + headers=headers, + stream=True, + verify=False, + timeout=(30, 1800), + ) except requests.ConnectionError: logger.error(f'Unable to open URL: {url}', section) return ProcessResult.failure( @@ -389,7 +533,11 @@ def process( f'{section}', ) - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + if r.status_code not in [ + requests.codes.ok, + requests.codes.created, + requests.codes.accepted, + ]: logger.error(f'Server returned status {r.status_code}', section) return ProcessResult.failure( f'{section}: Failed to post-process - Server returned status ' @@ -412,7 +560,10 @@ def process( input_name = os.path.split(line)[1] if 'added to the queue' in line: queued = True - if 'Processing succeeded' in line or 'Successfully processed' in line: + if ( + 'Processing succeeded' in line + or 'Successfully processed' in line + ): success = True if queued: @@ -434,8 +585,14 @@ def process( scan_id = None started = False - if status != 0 and delete_failed and not os.path.dirname(dir_name) == dir_name: - logger.postprocess(f'Deleting failed files and folder {dir_name}', section) + if ( + status != 0 + and delete_failed + and not os.path.dirname(dir_name) == dir_name + ): + logger.postprocess( + f'Deleting failed files and folder {dir_name}', section, + ) remove_dir(dir_name) if success: @@ -453,19 +610,30 @@ def process( break n += 1 if command_status: - logger.debug(f'The Scan command return status: {command_status}', section) + logger.debug( + f'The Scan command return status: {command_status}', section, + ) if not os.path.exists(dir_name): - logger.debug(f'The directory {dir_name} has been removed. Renaming was successful.', section) + logger.debug( + f'The directory {dir_name} has been removed. Renaming was successful.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) elif command_status and command_status in ['completed']: - logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + logger.debug( + 'The Scan command has completed successfully. Renaming was successful.', + section, + ) return ProcessResult.success( f'{section}: Successfully post-processed {input_name}', ) elif command_status and command_status in ['failed']: - logger.debug('The Scan command has failed. Renaming was not successful.', section) + logger.debug( + 'The Scan command has failed. Renaming was not successful.', + section, + ) # return ProcessResult.failure( # f'{section}: Failed to post-process {input_name}' # ) @@ -478,11 +646,14 @@ def process( ) return ProcessResult( message=f'{section}: Complete DownLoad Handling is enabled. ' - f'Passing back to {section}', + f'Passing back to {section}', status_code=status, ) else: - logger.warning('The Scan command did not return a valid status. Renaming was not successful.', section) + logger.warning( + 'The Scan command did not return a valid status. Renaming was not successful.', + section, + ) return ProcessResult.failure( f'{section}: Failed to post-process {input_name}', ) diff --git a/core/configuration.py b/core/configuration.py index 0440b957..2f709f05 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -26,7 +26,9 @@ class Section(configobj.Section): for section_name, subsections in to_return.items(): for subsection in subsections: try: - value = list(ConfigObj.find_key(subsections, 'enabled'))[0] + value = list( + ConfigObj.find_key(subsections, 'enabled'), + )[0] except Exception: value = 0 @@ -165,7 +167,9 @@ class ConfigObj(configobj.ConfigObj, Section): for option, value in values.items(): if section in ['CouchPotato']: if option == ['outputDirectory']: - CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] + CFG_NEW['Torrent'][option] = os.path.split( + os.path.normpath(value), + )[0] values.pop(option) if section in ['CouchPotato', 'HeadPhones', 'Gamez', 'Mylar']: if option in ['username', 'password']: @@ -177,11 +181,20 @@ class ConfigObj(configobj.ConfigObj, Section): if option == 'failed_fork': # change this old format values['failed'] = 'auto' values.pop(option) - if option == 'outputDirectory': # move this to new location format - CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] + if ( + option == 'outputDirectory' + ): # move this to new location format + CFG_NEW['Torrent'][option] = os.path.split( + os.path.normpath(value), + )[0] values.pop(option) if section in ['Torrent']: - if option in ['compressedExtensions', 'mediaExtensions', 'metaExtensions', 'minSampleSize']: + if option in [ + 'compressedExtensions', + 'mediaExtensions', + 'metaExtensions', + 'minSampleSize', + ]: CFG_NEW['Extensions'][option] = value values.pop(option) if option == 'useLink': # Sym links supported now as well. @@ -193,7 +206,9 @@ class ConfigObj(configobj.ConfigObj, Section): if option == 'forceClean': CFG_NEW['General']['force_clean'] = value values.pop(option) - if option == 'qBittorrenHost': # We had a typo that is now fixed. + if ( + option == 'qBittorrenHost' + ): # We had a typo that is now fixed. CFG_NEW['Torrent']['qBittorrentHost'] = value values.pop(option) if section in ['Transcoder']: @@ -226,7 +241,9 @@ class ConfigObj(configobj.ConfigObj, Section): for option, value in values.items(): CFG_NEW[section][subsection][option] = value elif subsection in CFG_OLD[section].sections: - values = cleanup_values(CFG_OLD[section][subsection], section) + values = cleanup_values( + CFG_OLD[section][subsection], section, + ) if subsection not in CFG_NEW[section].sections: CFG_NEW[section][subsection] = {} for option, value in values.items(): @@ -243,7 +260,9 @@ class ConfigObj(configobj.ConfigObj, Section): subsection = None if section in list(chain.from_iterable(subsections.values())): subsection = section - section = ''.join([k for k, v in subsections.items() if subsection in v]) + section = ''.join( + [k for k, v in subsections.items() if subsection in v], + ) process_section(section, subsection) elif section in subsections.keys(): subsection = subsections[section] @@ -252,13 +271,19 @@ class ConfigObj(configobj.ConfigObj, Section): process_section(section, subsection) # migrate SiCRKAGE settings from SickBeard section to new dedicated SiCRKAGE section - if CFG_OLD['SickBeard']['tv']['enabled'] and CFG_OLD['SickBeard']['tv']['fork'] == 'sickrage-api': + if ( + CFG_OLD['SickBeard']['tv']['enabled'] + and CFG_OLD['SickBeard']['tv']['fork'] == 'sickrage-api' + ): for option, value in CFG_OLD['SickBeard']['tv'].items(): if option in CFG_NEW['SiCKRAGE']['tv']: CFG_NEW['SiCKRAGE']['tv'][option] = value # set API version to 1 if API key detected and no SSO username is set - if CFG_NEW['SiCKRAGE']['tv']['apikey'] and not CFG_NEW['SiCKRAGE']['tv']['sso_username']: + if ( + CFG_NEW['SiCKRAGE']['tv']['apikey'] + and not CFG_NEW['SiCKRAGE']['tv']['sso_username'] + ): CFG_NEW['SiCKRAGE']['tv']['api_version'] = 1 # disable SickBeard section @@ -281,30 +306,70 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new = config() try: - if 'NZBPO_NDCATEGORY' in os.environ and 'NZBPO_SBCATEGORY' in os.environ: - if os.environ['NZBPO_NDCATEGORY'] == os.environ['NZBPO_SBCATEGORY']: + if ( + 'NZBPO_NDCATEGORY' in os.environ + and 'NZBPO_SBCATEGORY' in os.environ + ): + if ( + os.environ['NZBPO_NDCATEGORY'] + == os.environ['NZBPO_SBCATEGORY'] + ): logger.warning( - '{x} category is set for SickBeard and Sonarr. Please check your config in NZBGet'.format(x=os.environ['NZBPO_NDCATEGORY']), + '{x} category is set for SickBeard and Sonarr. Please check your config in NZBGet'.format( + x=os.environ['NZBPO_NDCATEGORY'], + ), ) - if 'NZBPO_RACATEGORY' in os.environ and 'NZBPO_CPSCATEGORY' in os.environ: - if os.environ['NZBPO_RACATEGORY'] == os.environ['NZBPO_CPSCATEGORY']: + if ( + 'NZBPO_RACATEGORY' in os.environ + and 'NZBPO_CPSCATEGORY' in os.environ + ): + if ( + os.environ['NZBPO_RACATEGORY'] + == os.environ['NZBPO_CPSCATEGORY'] + ): logger.warning( - '{x} category is set for CouchPotato and Radarr. Please check your config in NZBGet'.format(x=os.environ['NZBPO_RACATEGORY']), + '{x} category is set for CouchPotato and Radarr. Please check your config in NZBGet'.format( + x=os.environ['NZBPO_RACATEGORY'], + ), ) - if 'NZBPO_RACATEGORY' in os.environ and 'NZBPO_W3CATEGORY' in os.environ: - if os.environ['NZBPO_RACATEGORY'] == os.environ['NZBPO_W3CATEGORY']: + if ( + 'NZBPO_RACATEGORY' in os.environ + and 'NZBPO_W3CATEGORY' in os.environ + ): + if ( + os.environ['NZBPO_RACATEGORY'] + == os.environ['NZBPO_W3CATEGORY'] + ): logger.warning( - '{x} category is set for Watcher3 and Radarr. Please check your config in NZBGet'.format(x=os.environ['NZBPO_RACATEGORY']), + '{x} category is set for Watcher3 and Radarr. Please check your config in NZBGet'.format( + x=os.environ['NZBPO_RACATEGORY'], + ), ) - if 'NZBPO_W3CATEGORY' in os.environ and 'NZBPO_CPSCATEGORY' in os.environ: - if os.environ['NZBPO_W3CATEGORY'] == os.environ['NZBPO_CPSCATEGORY']: + if ( + 'NZBPO_W3CATEGORY' in os.environ + and 'NZBPO_CPSCATEGORY' in os.environ + ): + if ( + os.environ['NZBPO_W3CATEGORY'] + == os.environ['NZBPO_CPSCATEGORY'] + ): logger.warning( - '{x} category is set for CouchPotato and Watcher3. Please check your config in NZBGet'.format(x=os.environ['NZBPO_W3CATEGORY']), + '{x} category is set for CouchPotato and Watcher3. Please check your config in NZBGet'.format( + x=os.environ['NZBPO_W3CATEGORY'], + ), ) - if 'NZBPO_LICATEGORY' in os.environ and 'NZBPO_HPCATEGORY' in os.environ: - if os.environ['NZBPO_LICATEGORY'] == os.environ['NZBPO_HPCATEGORY']: + if ( + 'NZBPO_LICATEGORY' in os.environ + and 'NZBPO_HPCATEGORY' in os.environ + ): + if ( + os.environ['NZBPO_LICATEGORY'] + == os.environ['NZBPO_HPCATEGORY'] + ): logger.warning( - '{x} category is set for HeadPhones and Lidarr. Please check your config in NZBGet'.format(x=os.environ['NZBPO_LICATEGORY']), + '{x} category is set for HeadPhones and Lidarr. Please check your config in NZBGet'.format( + x=os.environ['NZBPO_LICATEGORY'], + ), ) section = 'Nzb' key = 'NZBOP_DESTDIR' @@ -314,8 +379,20 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new[section][option] = value section = 'General' - env_keys = ['AUTO_UPDATE', 'CHECK_MEDIA', 'REQUIRE_LAN', 'SAFE_MODE', 'NO_EXTRACT_FAILED'] - cfg_keys = ['auto_update', 'check_media', 'require_lan', 'safe_mode', 'no_extract_failed'] + env_keys = [ + 'AUTO_UPDATE', + 'CHECK_MEDIA', + 'REQUIRE_LAN', + 'SAFE_MODE', + 'NO_EXTRACT_FAILED', + ] + cfg_keys = [ + 'auto_update', + 'check_media', + 'require_lan', + 'safe_mode', + 'no_extract_failed', + ] for index in range(len(env_keys)): key = f'NZBPO_{env_keys[index]}' if key in os.environ: @@ -336,12 +413,32 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'CouchPotato' env_cat_key = 'NZBPO_CPSCATEGORY' env_keys = [ - 'ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', - 'WAIT_FOR', 'WATCH_DIR', 'OMDBAPIKEY', + 'ENABLED', + 'APIKEY', + 'HOST', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'METHOD', + 'DELETE_FAILED', + 'REMOTE_PATH', + 'WAIT_FOR', + 'WATCH_DIR', + 'OMDBAPIKEY', ] cfg_keys = [ - 'enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', - 'wait_for', 'watch_dir', 'omdbapikey', + 'enabled', + 'apikey', + 'host', + 'port', + 'ssl', + 'web_root', + 'method', + 'delete_failed', + 'remote_path', + 'wait_for', + 'watch_dir', + 'omdbapikey', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -349,9 +446,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['Radarr'].sections: cfg_new['Radarr'][env_cat_key]['enabled'] = 0 @@ -361,12 +463,32 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'Watcher3' env_cat_key = 'NZBPO_W3CATEGORY' env_keys = [ - 'ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', - 'WAIT_FOR', 'WATCH_DIR', 'OMDBAPIKEY', + 'ENABLED', + 'APIKEY', + 'HOST', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'METHOD', + 'DELETE_FAILED', + 'REMOTE_PATH', + 'WAIT_FOR', + 'WATCH_DIR', + 'OMDBAPIKEY', ] cfg_keys = [ - 'enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', - 'wait_for', 'watch_dir', 'omdbapikey', + 'enabled', + 'apikey', + 'host', + 'port', + 'ssl', + 'web_root', + 'method', + 'delete_failed', + 'remote_path', + 'wait_for', + 'watch_dir', + 'omdbapikey', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -374,9 +496,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['Radarr'].sections: cfg_new['Radarr'][env_cat_key]['enabled'] = 0 @@ -386,12 +513,38 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'SickBeard' env_cat_key = 'NZBPO_SBCATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'PORT', 'APIKEY', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'TORRENT_NOLINK', - 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD', + 'ENABLED', + 'HOST', + 'PORT', + 'APIKEY', + 'USERNAME', + 'PASSWORD', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'FORK', + 'DELETE_FAILED', + 'TORRENT_NOLINK', + 'NZBEXTRACTIONBY', + 'REMOTE_PATH', + 'PROCESS_METHOD', ] cfg_keys = [ - 'enabled', 'host', 'port', 'apikey', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'Torrent_NoLink', - 'nzbExtractionBy', 'remote_path', 'process_method', + 'enabled', + 'host', + 'port', + 'apikey', + 'username', + 'password', + 'ssl', + 'web_root', + 'watch_dir', + 'fork', + 'delete_failed', + 'Torrent_NoLink', + 'nzbExtractionBy', + 'remote_path', + 'process_method', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -399,9 +552,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['SiCKRAGE'].sections: cfg_new['SiCKRAGE'][env_cat_key]['enabled'] = 0 @@ -411,12 +569,40 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'SiCKRAGE' env_cat_key = 'NZBPO_SRCATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'PORT', 'APIKEY', 'API_VERSION', 'SSO_USERNAME', 'SSO_PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', - 'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD', + 'ENABLED', + 'HOST', + 'PORT', + 'APIKEY', + 'API_VERSION', + 'SSO_USERNAME', + 'SSO_PASSWORD', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'FORK', + 'DELETE_FAILED', + 'TORRENT_NOLINK', + 'NZBEXTRACTIONBY', + 'REMOTE_PATH', + 'PROCESS_METHOD', ] cfg_keys = [ - 'enabled', 'host', 'port', 'apikey', 'api_version', 'sso_username', 'sso_password', 'ssl', 'web_root', 'watch_dir', 'fork', - 'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'remote_path', 'process_method', + 'enabled', + 'host', + 'port', + 'apikey', + 'api_version', + 'sso_username', + 'sso_password', + 'ssl', + 'web_root', + 'watch_dir', + 'fork', + 'delete_failed', + 'Torrent_NoLink', + 'nzbExtractionBy', + 'remote_path', + 'process_method', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -424,9 +610,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['SickBeard'].sections: cfg_new['SickBeard'][env_cat_key]['enabled'] = 0 @@ -435,17 +626,44 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'HeadPhones' env_cat_key = 'NZBPO_HPCATEGORY' - env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WAIT_FOR', 'WATCH_DIR', 'REMOTE_PATH', 'DELETE_FAILED'] - cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'wait_for', 'watch_dir', 'remote_path', 'delete_failed'] + env_keys = [ + 'ENABLED', + 'APIKEY', + 'HOST', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WAIT_FOR', + 'WATCH_DIR', + 'REMOTE_PATH', + 'DELETE_FAILED', + ] + cfg_keys = [ + 'enabled', + 'apikey', + 'host', + 'port', + 'ssl', + 'web_root', + 'wait_for', + 'watch_dir', + 'remote_path', + 'delete_failed', + ] if env_cat_key in os.environ: for index in range(len(env_keys)): key = f'NZBPO_HP{env_keys[index]}' if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['Lidarr'].sections: cfg_new['Lidarr'][env_cat_key]['enabled'] = 0 @@ -453,11 +671,27 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'Mylar' env_cat_key = 'NZBPO_MYCATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'APIKEY', 'SSL', 'WEB_ROOT', 'WATCH_DIR', + 'ENABLED', + 'HOST', + 'PORT', + 'USERNAME', + 'PASSWORD', + 'APIKEY', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', 'REMOTE_PATH', ] cfg_keys = [ - 'enabled', 'host', 'port', 'username', 'password', 'apikey', 'ssl', 'web_root', 'watch_dir', + 'enabled', + 'host', + 'port', + 'username', + 'password', + 'apikey', + 'ssl', + 'web_root', + 'watch_dir', 'remote_path', ] if env_cat_key in os.environ: @@ -466,51 +700,130 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 section = 'Gamez' env_cat_key = 'NZBPO_GZCATEGORY' - env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'LIBRARY', 'REMOTE_PATH'] - cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'library', 'remote_path'] + env_keys = [ + 'ENABLED', + 'APIKEY', + 'HOST', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'LIBRARY', + 'REMOTE_PATH', + ] + cfg_keys = [ + 'enabled', + 'apikey', + 'host', + 'port', + 'ssl', + 'web_root', + 'watch_dir', + 'library', + 'remote_path', + ] if env_cat_key in os.environ: for index in range(len(env_keys)): key = f'NZBPO_GZ{env_keys[index]}' if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 section = 'LazyLibrarian' env_cat_key = 'NZBPO_LLCATEGORY' - env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'REMOTE_PATH'] - cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'remote_path'] + env_keys = [ + 'ENABLED', + 'APIKEY', + 'HOST', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'REMOTE_PATH', + ] + cfg_keys = [ + 'enabled', + 'apikey', + 'host', + 'port', + 'ssl', + 'web_root', + 'watch_dir', + 'remote_path', + ] if env_cat_key in os.environ: for index in range(len(env_keys)): key = f'NZBPO_LL{env_keys[index]}' if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 section = 'NzbDrone' env_cat_key = 'NZBPO_NDCATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'IMPORTMODE', + 'ENABLED', + 'HOST', + 'APIKEY', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'FORK', + 'DELETE_FAILED', + 'TORRENT_NOLINK', + 'NZBEXTRACTIONBY', + 'WAIT_FOR', + 'DELETE_FAILED', + 'REMOTE_PATH', + 'IMPORTMODE', ] # new cfgKey added for importMode cfg_keys = [ - 'enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', 'importMode', + 'enabled', + 'host', + 'apikey', + 'port', + 'ssl', + 'web_root', + 'watch_dir', + 'fork', + 'delete_failed', + 'Torrent_NoLink', + 'nzbExtractionBy', + 'wait_for', + 'delete_failed', + 'remote_path', + 'importMode', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -518,9 +831,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['SickBeard'].sections: cfg_new['SickBeard'][env_cat_key]['enabled'] = 0 @@ -530,13 +848,41 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'Radarr' env_cat_key = 'NZBPO_RACATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'OMDBAPIKEY', 'IMPORTMODE', + 'ENABLED', + 'HOST', + 'APIKEY', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'FORK', + 'DELETE_FAILED', + 'TORRENT_NOLINK', + 'NZBEXTRACTIONBY', + 'WAIT_FOR', + 'DELETE_FAILED', + 'REMOTE_PATH', + 'OMDBAPIKEY', + 'IMPORTMODE', ] # new cfgKey added for importMode cfg_keys = [ - 'enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', 'omdbapikey', 'importMode', + 'enabled', + 'host', + 'apikey', + 'port', + 'ssl', + 'web_root', + 'watch_dir', + 'fork', + 'delete_failed', + 'Torrent_NoLink', + 'nzbExtractionBy', + 'wait_for', + 'delete_failed', + 'remote_path', + 'omdbapikey', + 'importMode', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -544,9 +890,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['CouchPotato'].sections: cfg_new['CouchPotato'][env_cat_key]['enabled'] = 0 @@ -556,12 +907,36 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'Lidarr' env_cat_key = 'NZBPO_LICATEGORY' env_keys = [ - 'ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', + 'ENABLED', + 'HOST', + 'APIKEY', + 'PORT', + 'SSL', + 'WEB_ROOT', + 'WATCH_DIR', + 'FORK', + 'DELETE_FAILED', + 'TORRENT_NOLINK', + 'NZBEXTRACTIONBY', + 'WAIT_FOR', + 'DELETE_FAILED', + 'REMOTE_PATH', ] cfg_keys = [ - 'enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', + 'enabled', + 'host', + 'apikey', + 'port', + 'ssl', + 'web_root', + 'watch_dir', + 'fork', + 'delete_failed', + 'Torrent_NoLink', + 'nzbExtractionBy', + 'wait_for', + 'delete_failed', + 'remote_path', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -569,16 +944,29 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['HeadPhones'].sections: cfg_new['HeadPhones'][env_cat_key]['enabled'] = 0 section = 'Extensions' - env_keys = ['COMPRESSEDEXTENSIONS', 'MEDIAEXTENSIONS', 'METAEXTENSIONS'] - cfg_keys = ['compressedExtensions', 'mediaExtensions', 'metaExtensions'] + env_keys = [ + 'COMPRESSEDEXTENSIONS', + 'MEDIAEXTENSIONS', + 'METAEXTENSIONS', + ] + cfg_keys = [ + 'compressedExtensions', + 'mediaExtensions', + 'metaExtensions', + ] for index in range(len(env_keys)): key = f'NZBPO_{env_keys[index]}' if key in os.environ: @@ -598,28 +986,82 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'Transcoder' env_keys = [ - 'TRANSCODE', 'DUPLICATE', 'IGNOREEXTENSIONS', 'OUTPUTFASTSTART', 'OUTPUTVIDEOPATH', - 'PROCESSOUTPUT', 'AUDIOLANGUAGE', 'ALLAUDIOLANGUAGES', 'SUBLANGUAGES', - 'ALLSUBLANGUAGES', 'EMBEDSUBS', 'BURNINSUBTITLE', 'EXTRACTSUBS', 'EXTERNALSUBDIR', - 'OUTPUTDEFAULT', 'OUTPUTVIDEOEXTENSION', 'OUTPUTVIDEOCODEC', 'VIDEOCODECALLOW', - 'OUTPUTVIDEOPRESET', 'OUTPUTVIDEOFRAMERATE', 'OUTPUTVIDEOBITRATE', 'OUTPUTAUDIOCODEC', - 'AUDIOCODECALLOW', 'OUTPUTAUDIOBITRATE', 'OUTPUTQUALITYPERCENT', 'GETSUBS', - 'OUTPUTAUDIOTRACK2CODEC', 'AUDIOCODEC2ALLOW', 'OUTPUTAUDIOTRACK2BITRATE', - 'OUTPUTAUDIOOTHERCODEC', 'AUDIOOTHERCODECALLOW', 'OUTPUTAUDIOOTHERBITRATE', - 'OUTPUTSUBTITLECODEC', 'OUTPUTAUDIOCHANNELS', 'OUTPUTAUDIOTRACK2CHANNELS', - 'OUTPUTAUDIOOTHERCHANNELS', 'OUTPUTVIDEORESOLUTION', + 'TRANSCODE', + 'DUPLICATE', + 'IGNOREEXTENSIONS', + 'OUTPUTFASTSTART', + 'OUTPUTVIDEOPATH', + 'PROCESSOUTPUT', + 'AUDIOLANGUAGE', + 'ALLAUDIOLANGUAGES', + 'SUBLANGUAGES', + 'ALLSUBLANGUAGES', + 'EMBEDSUBS', + 'BURNINSUBTITLE', + 'EXTRACTSUBS', + 'EXTERNALSUBDIR', + 'OUTPUTDEFAULT', + 'OUTPUTVIDEOEXTENSION', + 'OUTPUTVIDEOCODEC', + 'VIDEOCODECALLOW', + 'OUTPUTVIDEOPRESET', + 'OUTPUTVIDEOFRAMERATE', + 'OUTPUTVIDEOBITRATE', + 'OUTPUTAUDIOCODEC', + 'AUDIOCODECALLOW', + 'OUTPUTAUDIOBITRATE', + 'OUTPUTQUALITYPERCENT', + 'GETSUBS', + 'OUTPUTAUDIOTRACK2CODEC', + 'AUDIOCODEC2ALLOW', + 'OUTPUTAUDIOTRACK2BITRATE', + 'OUTPUTAUDIOOTHERCODEC', + 'AUDIOOTHERCODECALLOW', + 'OUTPUTAUDIOOTHERBITRATE', + 'OUTPUTSUBTITLECODEC', + 'OUTPUTAUDIOCHANNELS', + 'OUTPUTAUDIOTRACK2CHANNELS', + 'OUTPUTAUDIOOTHERCHANNELS', + 'OUTPUTVIDEORESOLUTION', ] cfg_keys = [ - 'transcode', 'duplicate', 'ignoreExtensions', 'outputFastStart', 'outputVideoPath', - 'processOutput', 'audioLanguage', 'allAudioLanguages', 'subLanguages', - 'allSubLanguages', 'embedSubs', 'burnInSubtitle', 'extractSubs', 'externalSubDir', - 'outputDefault', 'outputVideoExtension', 'outputVideoCodec', 'VideoCodecAllow', - 'outputVideoPreset', 'outputVideoFramerate', 'outputVideoBitrate', 'outputAudioCodec', - 'AudioCodecAllow', 'outputAudioBitrate', 'outputQualityPercent', 'getSubs', - 'outputAudioTrack2Codec', 'AudioCodec2Allow', 'outputAudioTrack2Bitrate', - 'outputAudioOtherCodec', 'AudioOtherCodecAllow', 'outputAudioOtherBitrate', - 'outputSubtitleCodec', 'outputAudioChannels', 'outputAudioTrack2Channels', - 'outputAudioOtherChannels', 'outputVideoResolution', + 'transcode', + 'duplicate', + 'ignoreExtensions', + 'outputFastStart', + 'outputVideoPath', + 'processOutput', + 'audioLanguage', + 'allAudioLanguages', + 'subLanguages', + 'allSubLanguages', + 'embedSubs', + 'burnInSubtitle', + 'extractSubs', + 'externalSubDir', + 'outputDefault', + 'outputVideoExtension', + 'outputVideoCodec', + 'VideoCodecAllow', + 'outputVideoPreset', + 'outputVideoFramerate', + 'outputVideoBitrate', + 'outputAudioCodec', + 'AudioCodecAllow', + 'outputAudioBitrate', + 'outputQualityPercent', + 'getSubs', + 'outputAudioTrack2Codec', + 'AudioCodec2Allow', + 'outputAudioTrack2Bitrate', + 'outputAudioOtherCodec', + 'AudioOtherCodecAllow', + 'outputAudioOtherBitrate', + 'outputSubtitleCodec', + 'outputAudioChannels', + 'outputAudioTrack2Channels', + 'outputAudioOtherChannels', + 'outputVideoResolution', ] for index in range(len(env_keys)): key = f'NZBPO_{env_keys[index]}' @@ -641,12 +1083,24 @@ class ConfigObj(configobj.ConfigObj, Section): section = 'UserScript' env_cat_key = 'NZBPO_USCATEGORY' env_keys = [ - 'USER_SCRIPT_MEDIAEXTENSIONS', 'USER_SCRIPT_PATH', 'USER_SCRIPT_PARAM', 'USER_SCRIPT_RUNONCE', - 'USER_SCRIPT_SUCCESSCODES', 'USER_SCRIPT_CLEAN', 'USDELAY', 'USREMOTE_PATH', + 'USER_SCRIPT_MEDIAEXTENSIONS', + 'USER_SCRIPT_PATH', + 'USER_SCRIPT_PARAM', + 'USER_SCRIPT_RUNONCE', + 'USER_SCRIPT_SUCCESSCODES', + 'USER_SCRIPT_CLEAN', + 'USDELAY', + 'USREMOTE_PATH', ] cfg_keys = [ - 'user_script_mediaExtensions', 'user_script_path', 'user_script_param', 'user_script_runOnce', - 'user_script_successCodes', 'user_script_clean', 'delay', 'remote_path', + 'user_script_mediaExtensions', + 'user_script_path', + 'user_script_param', + 'user_script_runOnce', + 'user_script_successCodes', + 'user_script_clean', + 'delay', + 'remote_path', ] if env_cat_key in os.environ: for index in range(len(env_keys)): @@ -654,9 +1108,14 @@ class ConfigObj(configobj.ConfigObj, Section): if key in os.environ: option = cfg_keys[index] value = os.environ[key] - if os.environ[env_cat_key] not in cfg_new[section].sections: + if ( + os.environ[env_cat_key] + not in cfg_new[section].sections + ): cfg_new[section][os.environ[env_cat_key]] = {} - cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]][ + option + ] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 except Exception as error: diff --git a/core/databases.py b/core/databases.py index 1f004d7a..beb11621 100644 --- a/core/databases.py +++ b/core/databases.py @@ -11,7 +11,9 @@ MAX_DB_VERSION = 2 def backup_database(version): logger.info('Backing up database before upgrade') if not backup_versioned_file(main_db.db_filename(), version): - logger.log_error_and_exit('Database backup failed, abort upgrading database') + logger.log_error_and_exit( + 'Database backup failed, abort upgrading database', + ) else: logger.info('Proceeding with upgrade') @@ -21,6 +23,7 @@ def backup_database(version): # ====================== # Add new migrations at the bottom of the list; subclass the previous migration. + class InitialSchema(main_db.SchemaUpgrade): def test(self): no_update = False @@ -30,7 +33,9 @@ class InitialSchema(main_db.SchemaUpgrade): return no_update def execute(self): - if not self.has_table('downloads') and not self.has_table('db_version'): + if not self.has_table('downloads') and not self.has_table( + 'db_version', + ): queries = [ 'CREATE TABLE db_version (db_version INTEGER);', 'CREATE TABLE downloads (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));', diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index f8c91cf1..6c53a572 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -19,14 +19,46 @@ def extract(file_path, output_destination): if not os.path.exists(core.SEVENZIP): core.logger.error('EXTRACTOR: Could not find 7-zip, Exiting') return False - wscriptlocation = os.path.join(os.environ['WINDIR'], 'system32', 'wscript.exe') - invislocation = os.path.join(core.APP_ROOT, 'core', 'extractor', 'bin', 'invisible.vbs') - cmd_7zip = [wscriptlocation, invislocation, str(core.SHOWEXTRACT), core.SEVENZIP, 'x', '-y'] - ext_7zip = ['.rar', '.zip', '.tar.gz', 'tgz', '.tar.bz2', '.tbz', '.tar.lzma', '.tlz', '.7z', '.xz', '.gz'] + wscriptlocation = os.path.join( + os.environ['WINDIR'], 'system32', 'wscript.exe', + ) + invislocation = os.path.join( + core.APP_ROOT, 'core', 'extractor', 'bin', 'invisible.vbs', + ) + cmd_7zip = [ + wscriptlocation, + invislocation, + str(core.SHOWEXTRACT), + core.SEVENZIP, + 'x', + '-y', + ] + ext_7zip = [ + '.rar', + '.zip', + '.tar.gz', + 'tgz', + '.tar.bz2', + '.tbz', + '.tar.lzma', + '.tlz', + '.7z', + '.xz', + '.gz', + ] extract_commands = dict.fromkeys(ext_7zip, cmd_7zip) # Using unix else: - required_cmds = ['unrar', 'unzip', 'tar', 'unxz', 'unlzma', '7zr', 'bunzip2', 'gunzip'] + required_cmds = [ + 'unrar', + 'unzip', + 'tar', + 'unxz', + 'unlzma', + '7zr', + 'bunzip2', + 'gunzip', + ] # ## Possible future suport: # gunzip: gz (cmd will delete original archive) # ## the following do not extract to dest dir @@ -38,10 +70,14 @@ def extract(file_path, output_destination): '.rar': ['unrar', 'x', '-o+', '-y'], '.tar': ['tar', '-xf'], '.zip': ['unzip'], - '.tar.gz': ['tar', '-xzf'], '.tgz': ['tar', '-xzf'], - '.tar.bz2': ['tar', '-xjf'], '.tbz': ['tar', '-xjf'], - '.tar.lzma': ['tar', '--lzma', '-xf'], '.tlz': ['tar', '--lzma', '-xf'], - '.tar.xz': ['tar', '--xz', '-xf'], '.txz': ['tar', '--xz', '-xf'], + '.tar.gz': ['tar', '-xzf'], + '.tgz': ['tar', '-xzf'], + '.tar.bz2': ['tar', '-xjf'], + '.tbz': ['tar', '-xjf'], + '.tar.lzma': ['tar', '--lzma', '-xf'], + '.tlz': ['tar', '--lzma', '-xf'], + '.tar.xz': ['tar', '--xz', '-xf'], + '.txz': ['tar', '--xz', '-xf'], '.7z': ['7zr', 'x'], '.gz': ['gunzip'], } @@ -50,26 +86,43 @@ def extract(file_path, output_destination): devnull = open(os.devnull, 'w') for cmd in required_cmds: if call( - ['which', cmd], stdout=devnull, + ['which', cmd], + stdout=devnull, stderr=devnull, ): # note, returns 0 if exists, or 1 if doesn't exist. for k, v in extract_commands.items(): if cmd in v[0]: - if not call(['which', '7zr'], stdout=devnull, stderr=devnull): # we do have '7zr' + if not call( + ['which', '7zr'], + stdout=devnull, + stderr=devnull, + ): # we do have '7zr' extract_commands[k] = ['7zr', 'x', '-y'] - elif not call(['which', '7z'], stdout=devnull, stderr=devnull): # we do have '7z' + elif not call( + ['which', '7z'], stdout=devnull, stderr=devnull, + ): # we do have '7z' extract_commands[k] = ['7z', 'x', '-y'] - elif not call(['which', '7za'], stdout=devnull, stderr=devnull): # we do have '7za' + elif not call( + ['which', '7za'], + stdout=devnull, + stderr=devnull, + ): # we do have '7za' extract_commands[k] = ['7za', 'x', '-y'] else: - core.logger.error(f'EXTRACTOR: {cmd} not found, disabling support for {k}') + core.logger.error( + f'EXTRACTOR: {cmd} not found, disabling support for {k}', + ) del extract_commands[k] devnull.close() else: - core.logger.warning('EXTRACTOR: Cannot determine which tool to use when called from Transmission') + core.logger.warning( + 'EXTRACTOR: Cannot determine which tool to use when called from Transmission', + ) if not extract_commands: - core.logger.warning('EXTRACTOR: No archive extracting programs found, plugin will be disabled') + core.logger.warning( + 'EXTRACTOR: No archive extracting programs found, plugin will be disabled', + ) ext = os.path.splitext(file_path) cmd = [] @@ -79,9 +132,19 @@ def extract(file_path, output_destination): cmd = extract_commands[f'.tar{ext[1]}'] else: # Try gunzip cmd = extract_commands[ext[1]] - elif ext[1] in ('.1', '.01', '.001') and os.path.splitext(ext[0])[1] in ('.rar', '.zip', '.7z'): + elif ext[1] in ('.1', '.01', '.001') and os.path.splitext(ext[0])[1] in ( + '.rar', + '.zip', + '.7z', + ): cmd = extract_commands[os.path.splitext(ext[0])[1]] - elif ext[1] in ('.cb7', '.cba', '.cbr', '.cbt', '.cbz'): # don't extract these comic book archives. + elif ext[1] in ( + '.cb7', + '.cba', + '.cbr', + '.cbt', + '.cbz', + ): # don't extract these comic book archives. return False else: if ext[1] in extract_commands: @@ -93,8 +156,13 @@ def extract(file_path, output_destination): # Create outputDestination folder core.make_dir(output_destination) - if core.PASSWORDS_FILE and os.path.isfile(os.path.normpath(core.PASSWORDS_FILE)): - passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDS_FILE))] + if core.PASSWORDS_FILE and os.path.isfile( + os.path.normpath(core.PASSWORDS_FILE), + ): + passwords = [ + line.strip() + for line in open(os.path.normpath(core.PASSWORDS_FILE)) + ] else: passwords = [] @@ -110,7 +178,9 @@ def extract(file_path, output_destination): orig_files.append(os.path.join(directory, file)) pwd = os.getcwd() # Get our Present Working Directory - os.chdir(output_destination) # Not all unpack commands accept full paths, so just extract into this directory + os.chdir( + output_destination, + ) # Not all unpack commands accept full paths, so just extract into this directory devnull = open(os.devnull, 'w') try: # now works same for nt and *nix @@ -124,30 +194,42 @@ def extract(file_path, output_destination): cmd2 = cmd if not 'gunzip' in cmd: # gunzip doesn't support password cmd2.append('-p-') # don't prompt for password. - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + p = Popen( + cmd2, stdout=devnull, stderr=devnull, startupinfo=info, + ) # should extract files fine. res = p.wait() if res == 0: # Both Linux and Windows return 0 for successful. - core.logger.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination}') + core.logger.info( + f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination}', + ) success = 1 elif len(passwords) > 0 and not 'gunzip' in cmd: core.logger.info('EXTRACTOR: Attempting to extract with passwords') for password in passwords: - if password == '': # if edited in windows or otherwise if blank lines. + if ( + password == '' + ): # if edited in windows or otherwise if blank lines. continue cmd2 = cmd # append password here. passcmd = f'-p{password}' cmd2.append(passcmd) - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + p = Popen( + cmd2, stdout=devnull, stderr=devnull, startupinfo=info, + ) # should extract files fine. res = p.wait() if (res >= 0 and platform == 'Windows') or res == 0: - core.logger.info(f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination} using password: {password}') + core.logger.info( + f'EXTRACTOR: Extraction was successful for {file_path} to {output_destination} using password: {password}', + ) success = 1 break else: continue except Exception: - core.logger.error(f'EXTRACTOR: Extraction failed for {file_path}. Could not call command {cmd}') + core.logger.error( + f'EXTRACTOR: Extraction failed for {file_path}. Could not call command {cmd}', + ) os.chdir(pwd) return False @@ -167,10 +249,14 @@ def extract(file_path, output_destination): for file in files: if not os.path.join(directory, file) in orig_files: try: - shutil.copymode(file_path, os.path.join(directory, file)) + shutil.copymode( + file_path, os.path.join(directory, file), + ) except Exception: pass return True else: - core.logger.error(f'EXTRACTOR: Extraction failed for {file_path}. Result was {res}') + core.logger.error( + f'EXTRACTOR: Extraction failed for {file_path}. Result was {res}', + ) return False diff --git a/core/github_api.py b/core/github_api.py index 6359c37b..dd2087e7 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -47,7 +47,10 @@ class GitHub: """ return self._access_api( [ - 'repos', self.github_repo_user, self.github_repo, 'compare', + 'repos', + self.github_repo_user, + self.github_repo, + 'compare', f'{base}...{head}', ], params={'per_page': per_page}, diff --git a/core/logger.py b/core/logger.py index 1516412e..0d66563e 100644 --- a/core/logger.py +++ b/core/logger.py @@ -88,9 +88,18 @@ class NTMRotatingLogHandler: console.setFormatter( DispatchingFormatter( { - 'nzbtomedia': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), - 'postprocess': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), - 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), + 'nzbtomedia': logging.Formatter( + '[%(asctime)s] [%(levelname)s]::%(message)s', + '%H:%M:%S', + ), + 'postprocess': logging.Formatter( + '[%(asctime)s] [%(levelname)s]::%(message)s', + '%H:%M:%S', + ), + 'db': logging.Formatter( + '[%(asctime)s] [%(levelname)s]::%(message)s', + '%H:%M:%S', + ), }, logging.Formatter('%(message)s'), ), @@ -119,16 +128,27 @@ class NTMRotatingLogHandler: def _config_handler(self): """Configure a file handler to log at file_name and return it.""" - file_handler = logging.FileHandler(self.log_file_path, encoding='utf-8') + file_handler = logging.FileHandler( + self.log_file_path, encoding='utf-8', + ) file_handler.setLevel(DB) file_handler.setFormatter( DispatchingFormatter( { - 'nzbtomedia': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), - 'postprocess': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), - 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), + 'nzbtomedia': logging.Formatter( + '%(asctime)s %(levelname)-8s::%(message)s', + '%Y-%m-%d %H:%M:%S', + ), + 'postprocess': logging.Formatter( + '%(asctime)s %(levelname)-8s::%(message)s', + '%Y-%m-%d %H:%M:%S', + ), + 'db': logging.Formatter( + '%(asctime)s %(levelname)-8s::%(message)s', + '%Y-%m-%d %H:%M:%S', + ), }, logging.Formatter('%(message)s'), ), @@ -194,7 +214,10 @@ class NTMRotatingLogHandler: # check the size and see if we need to rotate if self.writes_since_check >= 10: - if os.path.isfile(self.log_file_path) and os.path.getsize(self.log_file_path) >= LOG_SIZE: + if ( + os.path.isfile(self.log_file_path) + and os.path.getsize(self.log_file_path) >= LOG_SIZE + ): self._rotate_logs() self.writes_since_check = 0 else: @@ -203,14 +226,18 @@ class NTMRotatingLogHandler: try: message = f'{section.upper()}: {to_log}' except UnicodeError: - message = f'{section.upper()}: Message contains non-utf-8 string' + message = ( + f'{section.upper()}: Message contains non-utf-8 string' + ) out_line = message ntm_logger = logging.getLogger('nzbtomedia') pp_logger = logging.getLogger('postprocess') db_logger = logging.getLogger('db') - pp_logger.postprocess = functools.partial(pp_logger.log, POSTPROCESS) + pp_logger.postprocess = functools.partial( + pp_logger.log, POSTPROCESS, + ) db_logger.db = functools.partial(db_logger.log, DB) try: if log_level == DEBUG: diff --git a/core/main_db.py b/core/main_db.py index 02c4be58..df2813b0 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -59,7 +59,8 @@ class DBConnection: sql_result = cursor.fetchone()[0] else: logger.log( - f'{self.filename}: {query} with args {args}', logger.DB, + f'{self.filename}: {query} with args {args}', + logger.DB, ) cursor = self.connection.cursor() cursor.execute(query, args) @@ -68,7 +69,10 @@ class DBConnection: # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError as error: - if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + if ( + 'unable to open database file' in error.args[0] + or 'database is locked' in error.args[0] + ): logger.log(f'DB error: {error}', logger.WARNING) attempt += 1 time.sleep(1) @@ -76,7 +80,9 @@ class DBConnection: logger.log(f'DB error: {error}', logger.ERROR) raise except sqlite3.DatabaseError as error: - logger.log(f'Fatal error executing query: {error}', logger.ERROR) + logger.log( + f'Fatal error executing query: {error}', logger.ERROR, + ) raise return sql_result @@ -97,16 +103,26 @@ class DBConnection: sql_result.append(self.connection.execute(qu[0])) elif len(qu) > 1: if log_transaction: - logger.log(f'{qu[0]} with args {qu[1]}', logger.DEBUG) - sql_result.append(self.connection.execute(qu[0], qu[1])) + logger.log( + f'{qu[0]} with args {qu[1]}', logger.DEBUG, + ) + sql_result.append( + self.connection.execute(qu[0], qu[1]), + ) self.connection.commit() - logger.log(f'Transaction with {len(querylist)} query\'s executed', logger.DEBUG) + logger.log( + f'Transaction with {len(querylist)} query\'s executed', + logger.DEBUG, + ) return sql_result except sqlite3.OperationalError as error: sql_result = [] if self.connection: self.connection.rollback() - if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + if ( + 'unable to open database file' in error.args[0] + or 'database is locked' in error.args[0] + ): logger.log(f'DB error: {error}', logger.WARNING) attempt += 1 time.sleep(1) @@ -116,7 +132,9 @@ class DBConnection: except sqlite3.DatabaseError as error: if self.connection: self.connection.rollback() - logger.log(f'Fatal error executing query: {error}', logger.ERROR) + logger.log( + f'Fatal error executing query: {error}', logger.ERROR, + ) raise return sql_result @@ -135,14 +153,18 @@ class DBConnection: sql_result = self.connection.execute(query) else: logger.log( - f'{self.filename}: {query} with args {args}', logger.DB, + f'{self.filename}: {query} with args {args}', + logger.DB, ) sql_result = self.connection.execute(query, args) self.connection.commit() # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError as error: - if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + if ( + 'unable to open database file' in error.args[0] + or 'database is locked' in error.args[0] + ): logger.log(f'DB error: {error}', logger.WARNING) attempt += 1 time.sleep(1) @@ -150,7 +172,9 @@ class DBConnection: logger.log(f'DB error: {error}', logger.ERROR) raise except sqlite3.DatabaseError as error: - logger.log(f'Fatal error executing query: {error}', logger.ERROR) + logger.log( + f'Fatal error executing query: {error}', logger.ERROR, + ) raise return sql_result @@ -165,12 +189,8 @@ class DBConnection: return sql_results def upsert(self, table_name, value_dict, key_dict): - def gen_params(my_dict): - return [ - f'{k} = ?' - for k in my_dict.keys() - ] + return [f'{k} = ?' for k in my_dict.keys()] changes_before = self.connection.total_changes items = list(value_dict.values()) + list(key_dict.values()) @@ -199,10 +219,7 @@ class DBConnection: def table_info(self, table_name): # FIXME ? binding is not supported here, but I cannot find a way to escape a string manually cursor = self.connection.execute(f'PRAGMA table_info({table_name})') - return { - column['name']: {'type': column['type']} - for column in cursor - } + return {column['name']: {'type': column['type']} for column in cursor} def sanity_check_database(connection, sanity_check): @@ -221,23 +238,28 @@ class DBSanityCheck: # = Upgrade API = # =============== + def upgrade_database(connection, schema): logger.log('Checking database structure...', logger.MESSAGE) _process_upgrade(connection, schema) def pretty_name(class_name): - return ' '.join([x.group() for x in re.finditer('([A-Z])([a-z0-9]+)', class_name)]) + return ' '.join( + [x.group() for x in re.finditer('([A-Z])([a-z0-9]+)', class_name)], + ) def _process_upgrade(connection, upgrade_class): instance = upgrade_class(connection) logger.log( - f'Checking {pretty_name(upgrade_class.__name__)} database upgrade', logger.DEBUG, + f'Checking {pretty_name(upgrade_class.__name__)} database upgrade', + logger.DEBUG, ) if not instance.test(): logger.log( - f'Database upgrade required: {pretty_name(upgrade_class.__name__)}', logger.MESSAGE, + f'Database upgrade required: {pretty_name(upgrade_class.__name__)}', + logger.MESSAGE, ) try: instance.execute() @@ -247,11 +269,13 @@ def _process_upgrade(connection, upgrade_class): ) raise logger.log( - f'{upgrade_class.__name__} upgrade completed', logger.DEBUG, + f'{upgrade_class.__name__} upgrade completed', + logger.DEBUG, ) else: logger.log( - f'{upgrade_class.__name__} upgrade not required', logger.DEBUG, + f'{upgrade_class.__name__} upgrade not required', + logger.DEBUG, ) for upgradeSubClass in upgrade_class.__subclasses__(): @@ -264,7 +288,15 @@ class SchemaUpgrade: self.connection = connection def has_table(self, table_name): - return len(self.connection.action('SELECT 1 FROM sqlite_master WHERE name = ?;', (table_name,)).fetchall()) > 0 + return ( + len( + self.connection.action( + 'SELECT 1 FROM sqlite_master WHERE name = ?;', + (table_name,), + ).fetchall(), + ) + > 0 + ) def has_column(self, table_name, column): return column in self.connection.table_info(table_name) @@ -282,5 +314,7 @@ class SchemaUpgrade: def inc_db_version(self): new_version = self.check_db_version() + 1 - self.connection.action('UPDATE db_version SET db_version = ?', [new_version]) + self.connection.action( + 'UPDATE db_version SET db_version = ?', [new_version], + ) return new_version diff --git a/core/plugins/downloaders/configuration.py b/core/plugins/downloaders/configuration.py index 5c122d7f..3de80f5b 100644 --- a/core/plugins/downloaders/configuration.py +++ b/core/plugins/downloaders/configuration.py @@ -1,5 +1,7 @@ from __future__ import annotations from core.plugins.downloaders.nzb.configuration import configure_nzbs -from core.plugins.downloaders.torrent.configuration import configure_torrent_class +from core.plugins.downloaders.torrent.configuration import ( + configure_torrent_class, +) from core.plugins.downloaders.torrent.configuration import configure_torrents diff --git a/core/plugins/downloaders/nzb/configuration.py b/core/plugins/downloaders/nzb/configuration.py index 0262e060..594406fb 100644 --- a/core/plugins/downloaders/nzb/configuration.py +++ b/core/plugins/downloaders/nzb/configuration.py @@ -14,5 +14,7 @@ def configure_nzbs(config): def configure_sabnzbd(config): core.SABNZBD_HOST = config['sabnzbd_host'] - core.SABNZBD_PORT = int(config['sabnzbd_port'] or 8080) # defaults to accommodate NzbGet + core.SABNZBD_PORT = int( + config['sabnzbd_port'] or 8080, + ) # defaults to accommodate NzbGet core.SABNZBD_APIKEY = config['sabnzbd_apikey'] diff --git a/core/plugins/downloaders/nzb/utils.py b/core/plugins/downloaders/nzb/utils.py index 649be19d..5dcc2f48 100644 --- a/core/plugins/downloaders/nzb/utils.py +++ b/core/plugins/downloaders/nzb/utils.py @@ -30,7 +30,12 @@ def get_nzoid(input_name): try: result = r.json() clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) + slots.extend( + [ + (slot['nzo_id'], slot['filename']) + for slot in result['queue']['slots'] + ], + ) except Exception: logger.warning('Data from SABnzbd queue could not be parsed') params['mode'] = 'history' @@ -42,7 +47,12 @@ def get_nzoid(input_name): try: result = r.json() clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) + slots.extend( + [ + (slot['nzo_id'], slot['name']) + for slot in result['history']['slots'] + ], + ) except Exception: logger.warning('Data from SABnzbd history could not be parsed') try: diff --git a/core/plugins/downloaders/torrent/configuration.py b/core/plugins/downloaders/torrent/configuration.py index 9c9e53e2..59555680 100644 --- a/core/plugins/downloaders/torrent/configuration.py +++ b/core/plugins/downloaders/torrent/configuration.py @@ -6,9 +6,15 @@ from core.plugins.downloaders.torrent.utils import create_torrent_class def configure_torrents(config): torrent_config = config['Torrent'] - core.TORRENT_CLIENT_AGENT = torrent_config['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent | synods | other - core.OUTPUT_DIRECTORY = torrent_config['outputDirectory'] # /abs/path/to/complete/ - core.TORRENT_DEFAULT_DIRECTORY = torrent_config['default_downloadDirectory'] + core.TORRENT_CLIENT_AGENT = torrent_config[ + 'clientAgent' + ] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent | synods | other + core.OUTPUT_DIRECTORY = torrent_config[ + 'outputDirectory' + ] # /abs/path/to/complete/ + core.TORRENT_DEFAULT_DIRECTORY = torrent_config[ + 'default_downloadDirectory' + ] core.TORRENT_NO_MANUAL = int(torrent_config['no_manual'], 0) configure_torrent_linking(torrent_config) @@ -29,13 +35,15 @@ def configure_torrent_linking(config): def configure_flattening(config): - core.NOFLATTEN = (config['noFlatten']) + core.NOFLATTEN = config['noFlatten'] if isinstance(core.NOFLATTEN, str): core.NOFLATTEN = core.NOFLATTEN.split(',') def configure_torrent_categories(config): - core.CATEGORIES = (config['categories']) # music,music_videos,pictures,software + core.CATEGORIES = config[ + 'categories' + ] # music,music_videos,pictures,software if isinstance(core.CATEGORIES, str): core.CATEGORIES = core.CATEGORIES.split(',') @@ -54,7 +62,9 @@ def configure_torrent_deletion(config): def configure_utorrent(config): - core.UTORRENT_WEB_UI = config['uTorrentWEBui'] # http://localhost:8090/gui/ + core.UTORRENT_WEB_UI = config[ + 'uTorrentWEBui' + ] # http://localhost:8090/gui/ core.UTORRENT_USER = config['uTorrentUSR'] # mysecretusr core.UTORRENT_PASSWORD = config['uTorrentPWD'] # mysecretpwr diff --git a/core/plugins/downloaders/torrent/utils.py b/core/plugins/downloaders/torrent/utils.py index 26dfff53..a5eaa649 100644 --- a/core/plugins/downloaders/torrent/utils.py +++ b/core/plugins/downloaders/torrent/utils.py @@ -30,7 +30,9 @@ def create_torrent_class(client_agent): def pause_torrent(client_agent, input_hash, input_id, input_name): - logger.debug(f'Stopping torrent {input_name} in {client_agent} while processing') + logger.debug( + f'Stopping torrent {input_name} in {client_agent} while processing', + ) try: if client_agent == 'utorrent' and core.TORRENT_CLASS != '': core.TORRENT_CLASS.stop(input_hash) @@ -44,7 +46,9 @@ def pause_torrent(client_agent, input_hash, input_id, input_name): core.TORRENT_CLASS.pause(input_hash) time.sleep(5) except Exception: - logger.warning(f'Failed to stop torrent {input_name} in {client_agent}') + logger.warning( + f'Failed to stop torrent {input_name} in {client_agent}', + ) def resume_torrent(client_agent, input_hash, input_id, input_name): @@ -64,7 +68,9 @@ def resume_torrent(client_agent, input_hash, input_id, input_name): core.TORRENT_CLASS.resume(input_hash) time.sleep(5) except Exception: - logger.warning(f'Failed to start torrent {input_name} in {client_agent}') + logger.warning( + f'Failed to start torrent {input_name} in {client_agent}', + ) def remove_torrent(client_agent, input_hash, input_id, input_name): @@ -84,6 +90,8 @@ def remove_torrent(client_agent, input_hash, input_id, input_name): core.TORRENT_CLASS.delete_permanently(input_hash) time.sleep(5) except Exception: - logger.warning(f'Failed to delete torrent {input_name} in {client_agent}') + logger.warning( + f'Failed to delete torrent {input_name} in {client_agent}', + ) else: resume_torrent(client_agent, input_hash, input_id, input_name) diff --git a/core/plugins/plex.py b/core/plugins/plex.py index 5b1179ac..be20f505 100644 --- a/core/plugins/plex.py +++ b/core/plugins/plex.py @@ -15,10 +15,11 @@ def configure_plex(config): if plex_section: if isinstance(plex_section, list): - plex_section = ','.join(plex_section) # fix in case this imported as list. + plex_section = ','.join( + plex_section, + ) # fix in case this imported as list. plex_section = [ - tuple(item.split(',')) - for item in plex_section.split('|') + tuple(item.split(',')) for item in plex_section.split('|') ] core.PLEX_SECTION = plex_section @@ -35,7 +36,9 @@ def plex_update(category): section = None if not core.PLEX_SECTION: return - logger.debug(f'Attempting to update Plex Library for category {category}.', 'PLEX') + logger.debug( + f'Attempting to update Plex Library for category {category}.', 'PLEX', + ) for item in core.PLEX_SECTION: if item[0] == category: section = item[1] diff --git a/core/plugins/subtitles.py b/core/plugins/subtitles.py index 3c50b0c1..e53b358b 100644 --- a/core/plugins/subtitles.py +++ b/core/plugins/subtitles.py @@ -14,7 +14,9 @@ def import_subs(filename): if not core.GETSUBS: return try: - subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) + subliminal.region.configure( + 'dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}, + ) except Exception: pass @@ -27,37 +29,54 @@ def import_subs(filename): if not languages: return - logger.info(f'Attempting to download subtitles for {filename}', 'SUBTITLES') + logger.info( + f'Attempting to download subtitles for {filename}', 'SUBTITLES', + ) try: video = subliminal.scan_video(filename) subtitles = subliminal.download_best_subtitles({video}, languages) subliminal.save_subtitles(video, subtitles[video]) for subtitle in subtitles[video]: - subtitle_path = subliminal.subtitle.get_subtitle_path(video.name, subtitle.language) + subtitle_path = subliminal.subtitle.get_subtitle_path( + video.name, subtitle.language, + ) os.chmod(subtitle_path, 0o644) except Exception as e: - logger.error(f'Failed to download subtitles for {filename} due to: {e}', 'SUBTITLES') + logger.error( + f'Failed to download subtitles for {filename} due to: {e}', + 'SUBTITLES', + ) def rename_subs(path): filepaths = [] sub_ext = ['.srt', '.sub', '.idx'] - vidfiles = core.list_media_files(path, media=True, audio=False, meta=False, archives=False) - if not vidfiles or len(vidfiles) > 1: # If there is more than 1 video file, or no video files, we can't rename subs. + vidfiles = core.list_media_files( + path, media=True, audio=False, meta=False, archives=False, + ) + if ( + not vidfiles or len(vidfiles) > 1 + ): # If there is more than 1 video file, or no video files, we can't rename subs. return name = os.path.splitext(os.path.split(vidfiles[0])[1])[0] for directory, _, filenames in os.walk(path): for filename in filenames: filepaths.extend([os.path.join(directory, filename)]) - subfiles = [item for item in filepaths if os.path.splitext(item)[1] in sub_ext] + subfiles = [ + item for item in filepaths if os.path.splitext(item)[1] in sub_ext + ] subfiles.sort() # This should sort subtitle names by language (alpha) and Number (where multiple) renamed = [] for sub in subfiles: subname, ext = os.path.splitext(os.path.basename(sub)) - if name in subname: # The sub file name already includes the video name. + if ( + name in subname + ): # The sub file name already includes the video name. continue - words = re.findall('[a-zA-Z]+', str(subname)) # find whole words in string + words = re.findall( + '[a-zA-Z]+', str(subname), + ) # find whole words in string # parse the words for language descriptors. lan = None for word in words: @@ -78,8 +97,12 @@ def rename_subs(path): new_sub_name = name else: new_sub_name = f'{name}.{str(lan)}' - new_sub = os.path.join(directory, new_sub_name) # full path and name less ext - if f'{new_sub}{ext}' in renamed: # If duplicate names, add unique number before ext. + new_sub = os.path.join( + directory, new_sub_name, + ) # full path and name less ext + if ( + f'{new_sub}{ext}' in renamed + ): # If duplicate names, add unique number before ext. for i in range(1, len(renamed) + 1): if f'{new_sub}.{i}{ext}' in renamed: continue @@ -87,7 +110,9 @@ def rename_subs(path): break new_sub = f'{new_sub}{ext}' # add extension now if os.path.isfile(new_sub): # Don't copy over existing - final check. - logger.debug(f'Unable to rename sub file {sub} as destination {new_sub} already exists') + logger.debug( + f'Unable to rename sub file {sub} as destination {new_sub} already exists', + ) continue logger.debug( f'Renaming sub file from {sub} to {new_sub}', diff --git a/core/processor/manual.py b/core/processor/manual.py index d50e936b..f895e16e 100644 --- a/core/processor/manual.py +++ b/core/processor/manual.py @@ -35,24 +35,36 @@ def process(): ) core.DOWNLOAD_INFO = get_download_info( - os.path.basename(dir_name), 0, + os.path.basename(dir_name), + 0, ) if core.DOWNLOAD_INFO: - logger.info(f'Found download info for {os.path.basename(dir_name)}, setting variables now ...') - client_agent = core.DOWNLOAD_INFO[0]['client_agent'] or 'manual' + logger.info( + f'Found download info for {os.path.basename(dir_name)}, setting variables now ...', + ) + client_agent = ( + core.DOWNLOAD_INFO[0]['client_agent'] or 'manual' + ) download_id = core.DOWNLOAD_INFO[0]['input_id'] or '' else: - logger.info(f'Unable to locate download info for {os.path.basename(dir_name)}, continuing to try and process this release ...') + logger.info( + f'Unable to locate download info for {os.path.basename(dir_name)}, continuing to try and process this release ...', + ) client_agent = 'manual' download_id = '' - if client_agent and client_agent.lower() not in core.NZB_CLIENTS: + if ( + client_agent + and client_agent.lower() not in core.NZB_CLIENTS + ): continue input_name = os.path.basename(dir_name) results = nzb.process( - dir_name, input_name, 0, + dir_name, + input_name, + 0, client_agent=client_agent, download_id=download_id or None, input_category=subsection, diff --git a/core/processor/nzb.py b/core/processor/nzb.py index b58fcbdc..3c5d2d0a 100644 --- a/core/processor/nzb.py +++ b/core/processor/nzb.py @@ -22,7 +22,15 @@ from core.utils.files import extract_files from core.utils.download_info import update_download_info_status -def process(input_directory, input_name=None, status=0, client_agent='manual', download_id=None, input_category=None, failure_link=None): +def process( + input_directory, + input_name=None, + status=0, + client_agent='manual', + download_id=None, + input_category=None, + failure_link=None, +): if core.SAFE_MODE and input_directory == core.NZB_DEFAULT_DIRECTORY: logger.error( f'The input directory:[{input_directory}] is the Default Download Directory. Please configure category directories to prevent processing of other media.', @@ -36,7 +44,9 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d download_id = get_nzoid(input_name) if client_agent != 'manual' and not core.DOWNLOAD_INFO: - logger.debug(f'Adding NZB download info for directory {input_directory} to database') + logger.debug( + f'Adding NZB download info for directory {input_directory} to database', + ) my_db = main_db.DBConnection() @@ -91,7 +101,9 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d section_name = section.keys()[0] logger.info(f'Auto-detected SECTION:{section_name}') else: - logger.error(f'Unable to locate a section with subsection:{input_category} enabled in your autoProcessMedia.cfg, exiting!') + logger.error( + f'Unable to locate a section with subsection:{input_category} enabled in your autoProcessMedia.cfg, exiting!', + ) return ProcessResult( status_code=-1, message='', @@ -103,25 +115,35 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d try: if int(cfg.get('remote_path')) and not core.REMOTE_PATHS: - logger.error(f'Remote Path is enabled for {section_name}:{input_category} but no Network mount points are defined. Please check your autoProcessMedia.cfg, exiting!') + logger.error( + f'Remote Path is enabled for {section_name}:{input_category} but no Network mount points are defined. Please check your autoProcessMedia.cfg, exiting!', + ) return ProcessResult( status_code=-1, message='', ) except Exception: remote_path = cfg.get('remote_path') - logger.error(f'Remote Path {remote_path} is not valid for {section_name}:{input_category} Please set this to either 0 to disable or 1 to enable!') + logger.error( + f'Remote Path {remote_path} is not valid for {section_name}:{input_category} Please set this to either 0 to disable or 1 to enable!', + ) input_name, input_directory = convert_to_ascii(input_name, input_directory) if extract == 1 and not (status > 0 and core.NOEXTRACTFAILED): - logger.debug(f'Checking for archives to extract in directory: {input_directory}') + logger.debug( + f'Checking for archives to extract in directory: {input_directory}', + ) extract_files(input_directory) - logger.info(f'Calling {section_name}:{input_category} to post-process:{input_name}') + logger.info( + f'Calling {section_name}:{input_category} to post-process:{input_name}', + ) if section_name == 'UserScript': - result = external_script(input_directory, input_name, input_category, section[usercat]) + result = external_script( + input_directory, input_name, input_category, section[usercat], + ) else: process_map = { 'CouchPotato': movies.process, @@ -155,7 +177,13 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d if client_agent != 'manual': # update download status in our DB update_download_info_status(input_name, 1) - if section_name not in ['UserScript', 'NzbDrone', 'Sonarr', 'Radarr', 'Lidarr']: + if section_name not in [ + 'UserScript', + 'NzbDrone', + 'Sonarr', + 'Radarr', + 'Lidarr', + ]: # cleanup our processing folders of any misc unwanted files and empty directories clean_dir(input_directory, section_name, input_category) diff --git a/core/processor/nzbget.py b/core/processor/nzbget.py index e0f42b11..fd64537a 100644 --- a/core/processor/nzbget.py +++ b/core/processor/nzbget.py @@ -65,11 +65,17 @@ def _parse_health_status(): # Unpack was skipped due to nzb-file properties # or due to errors during par-check if int(os.environ['NZBPP_HEALTH']) < 1000: - logger.warning('Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \'failed\'') + logger.warning( + 'Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \'failed\'', + ) status = 1 else: - logger.info('Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful') - logger.info('Please check your Par-check/repair settings for future downloads.') + logger.info( + 'Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful', + ) + logger.info( + 'Please check your Par-check/repair settings for future downloads.', + ) return status @@ -89,7 +95,9 @@ def check_version(): version = os.environ['NZBOP_VERSION'] # Check if the script is called from nzbget 11.0 or later if version[0:5] < '11.0': - logger.error(f'NZBGet Version {version} is not supported. Please update NZBGet.') + logger.error( + f'NZBGet Version {version} is not supported. Please update NZBGet.', + ) sys.exit(core.NZBGET_POSTPROCESS_ERROR) logger.info(f'Script triggered from NZBGet Version {version}.') diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 58d83ec1..733b407a 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -11,21 +11,57 @@ from core import logger from core.utils.files import list_media_files reverse_list = [ - r'\.\d{2}e\d{2}s\.', r'\.[pi]0801\.', r'\.p027\.', r'\.[pi]675\.', r'\.[pi]084\.', r'\.p063\.', - r'\b[45]62[xh]\.', r'\.yarulb\.', r'\.vtd[hp]\.', - r'\.ld[.-]?bew\.', r'\.pir.?(dov|dvd|bew|db|rb)\.', r'\brdvd\.', r'\.vts\.', r'\.reneercs\.', - r'\.dcv\.', r'\b(pir|mac)dh\b', r'\.reporp\.', r'\.kcaper\.', - r'\.lanretni\.', r'\b3ca\b', r'\.cstn\.', + r'\.\d{2}e\d{2}s\.', + r'\.[pi]0801\.', + r'\.p027\.', + r'\.[pi]675\.', + r'\.[pi]084\.', + r'\.p063\.', + r'\b[45]62[xh]\.', + r'\.yarulb\.', + r'\.vtd[hp]\.', + r'\.ld[.-]?bew\.', + r'\.pir.?(dov|dvd|bew|db|rb)\.', + r'\brdvd\.', + r'\.vts\.', + r'\.reneercs\.', + r'\.dcv\.', + r'\b(pir|mac)dh\b', + r'\.reporp\.', + r'\.kcaper\.', + r'\.lanretni\.', + r'\b3ca\b', + r'\.cstn\.', ] reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) season_pattern = re.compile(r'(.*\.\d{2}e\d{2}s\.)(.*)', flags=re.IGNORECASE) word_pattern = re.compile(r'([^A-Z0-9]*[A-Z0-9]+)') media_list = [ - r'\.s\d{2}e\d{2}\.', r'\.1080[pi]\.', r'\.720p\.', r'\.576[pi]', r'\.480[pi]\.', r'\.360p\.', - r'\.[xh]26[45]\b', r'\.bluray\.', r'\.[hp]dtv\.', - r'\.web[.-]?dl\.', r'\.(vod|dvd|web|bd|br).?rip\.', r'\.dvdr\b', r'\.stv\.', r'\.screener\.', r'\.vcd\.', - r'\bhd(cam|rip)\b', r'\.proper\.', r'\.repack\.', - r'\.internal\.', r'\bac3\b', r'\.ntsc\.', r'\.pal\.', r'\.secam\.', r'\bdivx\b', r'\bxvid\b', + r'\.s\d{2}e\d{2}\.', + r'\.1080[pi]\.', + r'\.720p\.', + r'\.576[pi]', + r'\.480[pi]\.', + r'\.360p\.', + r'\.[xh]26[45]\b', + r'\.bluray\.', + r'\.[hp]dtv\.', + r'\.web[.-]?dl\.', + r'\.(vod|dvd|web|bd|br).?rip\.', + r'\.dvdr\b', + r'\.stv\.', + r'\.screener\.', + r'\.vcd\.', + r'\bhd(cam|rip)\b', + r'\.proper\.', + r'\.repack\.', + r'\.internal\.', + r'\bac3\b', + r'\.ntsc\.', + r'\.pal\.', + r'\.secam\.', + r'\bdivx\b', + r'\bxvid\b', ] media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) garbage_name = re.compile(r'^[a-zA-Z0-9]*$') @@ -72,9 +108,14 @@ def strip_groups(filename): def rename_file(filename, newfile_path): if os.path.isfile(newfile_path): - newfile_path = os.path.splitext(newfile_path)[0] + '.NTM' + os.path.splitext(newfile_path)[1] + newfile_path = ( + os.path.splitext(newfile_path)[0] + + '.NTM' + + os.path.splitext(newfile_path)[1] + ) logger.debug( - f'Replacing file name {filename} with download name {newfile_path}', 'EXCEPTION', + f'Replacing file name {filename} with download name {newfile_path}', + 'EXCEPTION', ) try: os.rename(filename, newfile_path) @@ -84,16 +125,25 @@ def rename_file(filename, newfile_path): def replace_filename(filename, dirname, name): head, file_extension = os.path.splitext(os.path.basename(filename)) - if media_pattern.search(os.path.basename(dirname).replace(' ', '.')) is not None: + if ( + media_pattern.search(os.path.basename(dirname).replace(' ', '.')) + is not None + ): newname = os.path.basename(dirname).replace(' ', '.') - logger.debug(f'Replacing file name {head} with directory name {newname}', 'EXCEPTION') + logger.debug( + f'Replacing file name {head} with directory name {newname}', + 'EXCEPTION', + ) elif media_pattern.search(name.replace(' ', '.').lower()) is not None: newname = name.replace(' ', '.') logger.debug( - f'Replacing file name {head} with download name {newname}', 'EXCEPTION', + f'Replacing file name {head} with download name {newname}', + 'EXCEPTION', ) else: - logger.warning(f'No name replacement determined for {head}', 'EXCEPTION') + logger.warning( + f'No name replacement determined for {head}', 'EXCEPTION', + ) newname = name newfile = newname + file_extension newfile_path = os.path.join(dirname, newfile) @@ -120,7 +170,8 @@ def reverse_filename(filename, dirname, name): newname = head[::-1].title() newname = newname.replace(' ', '.') logger.debug( - f'Reversing filename {head} to {newname}', 'EXCEPTION', + f'Reversing filename {head} to {newname}', + 'EXCEPTION', ) newfile = newname + file_extension newfile_path = os.path.join(dirname, newfile) @@ -144,16 +195,21 @@ def rename_script(dirname): continue if len(cmd) == 2 and os.path.isfile(os.path.join(dirname, cmd[0])): orig = os.path.join(dirname, cmd[0]) - dest = os.path.join(dirname, cmd[1].split('\\')[-1].split('/')[-1]) + dest = os.path.join( + dirname, cmd[1].split('\\')[-1].split('/')[-1], + ) if os.path.isfile(dest): continue logger.debug( - f'Renaming file {orig} to {dest}', 'EXCEPTION', + f'Renaming file {orig} to {dest}', + 'EXCEPTION', ) try: os.rename(orig, dest) except Exception as error: - logger.error(f'Unable to rename file due to: {error}', 'EXCEPTION') + logger.error( + f'Unable to rename file due to: {error}', 'EXCEPTION', + ) def par2(dirname): @@ -182,16 +238,21 @@ def par2(dirname): cmd = f'{cmd} {item}' logger.debug(f'calling command:{cmd}', 'PAR2') try: - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc = subprocess.Popen( + command, stdout=bitbucket, stderr=bitbucket, + ) proc.communicate() result = proc.returncode except Exception: - logger.error(f'par2 file processing for {parfile} has failed', 'PAR2') + logger.error( + f'par2 file processing for {parfile} has failed', 'PAR2', + ) if result == 0: logger.info('par2 file processing succeeded', 'PAR2') os.chdir(pwd) bitbucket.close() + # dict for custom groups # we can add more to this list # _customgroups = {'Q o Q': process_qoq, '-ECI': process_eci} diff --git a/core/transcoder.py b/core/transcoder.py index 0708b81d..f754e61b 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -23,16 +23,33 @@ def is_video_good(videofile, status, require_lan=None): file_name_ext = os.path.basename(videofile) file_name, file_ext = os.path.splitext(file_name_ext) disable = False - if file_ext not in core.MEDIA_CONTAINER or not core.FFPROBE or not core.CHECK_MEDIA or file_ext in ['.iso'] or (status > 0 and core.NOEXTRACTFAILED): + if ( + file_ext not in core.MEDIA_CONTAINER + or not core.FFPROBE + or not core.CHECK_MEDIA + or file_ext in ['.iso'] + or (status > 0 and core.NOEXTRACTFAILED) + ): disable = True else: test_details, res = get_video_details(core.TEST_FILE) if res != 0 or test_details.get('error'): disable = True - logger.info('DISABLED: ffprobe failed to analyse test file. Stopping corruption check.', 'TRANSCODER') + logger.info( + 'DISABLED: ffprobe failed to analyse test file. Stopping corruption check.', + 'TRANSCODER', + ) if test_details.get('streams'): - vid_streams = [item for item in test_details['streams'] if 'codec_type' in item and item['codec_type'] == 'video'] - aud_streams = [item for item in test_details['streams'] if 'codec_type' in item and item['codec_type'] == 'audio'] + vid_streams = [ + item + for item in test_details['streams'] + if 'codec_type' in item and item['codec_type'] == 'video' + ] + aud_streams = [ + item + for item in test_details['streams'] + if 'codec_type' in item and item['codec_type'] == 'audio' + ] if not (len(vid_streams) > 0 and len(aud_streams) > 0): disable = True logger.info( @@ -40,12 +57,17 @@ def is_video_good(videofile, status, require_lan=None): 'TRANSCODER', ) if disable: - if status: # if the download was 'failed', assume bad. If it was successful, assume good. + if ( + status + ): # if the download was 'failed', assume bad. If it was successful, assume good. return False else: return True - logger.info(f'Checking [{file_name_ext}] for corruption, please stand by ...', 'TRANSCODER') + logger.info( + f'Checking [{file_name_ext}] for corruption, please stand by ...', + 'TRANSCODER', + ) video_details, result = get_video_details(videofile) if result != 0: @@ -53,20 +75,42 @@ def is_video_good(videofile, status, require_lan=None): return False if video_details.get('error'): error_details = video_details.get('error') - logger.info(f'FAILED: [{file_name_ext}] returned error [{error_details}].', 'TRANSCODER') + logger.info( + f'FAILED: [{file_name_ext}] returned error [{error_details}].', + 'TRANSCODER', + ) return False if video_details.get('streams'): - video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] - audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + video_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'video' + ] + audio_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'audio' + ] if require_lan: - valid_audio = [item for item in audio_streams if 'tags' in item and 'language' in item['tags'] and item['tags']['language'] in require_lan] + valid_audio = [ + item + for item in audio_streams + if 'tags' in item + and 'language' in item['tags'] + and item['tags']['language'] in require_lan + ] else: valid_audio = audio_streams if len(video_streams) > 0 and len(valid_audio) > 0: - logger.info(f'SUCCESS: [{file_name_ext}] has no corruption.', 'TRANSCODER') + logger.info( + f'SUCCESS: [{file_name_ext}] has no corruption.', 'TRANSCODER', + ) return True else: - logger.info(f'FAILED: [{file_name_ext}] has {len(video_streams)} video streams and {len(audio_streams)} audio streams. Assume corruption.', 'TRANSCODER') + logger.info( + f'FAILED: [{file_name_ext}] has {len(video_streams)} video streams and {len(audio_streams)} audio streams. Assume corruption.', + 'TRANSCODER', + ) return False @@ -77,7 +121,9 @@ def zip_out(file, img, bitbucket): else: cmd = [core.SEVENZIP, '-so', 'e', img, file] try: - procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + procin = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=bitbucket, + ) except Exception: logger.error(f'Extracting [{file}] has failed', 'TRANSCODER') return procin @@ -94,13 +140,22 @@ def get_video_details(videofile, img=None, bitbucket=None): if img: videofile = '-' command = [ - core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', '-show_error', + core.FFPROBE, + '-v', + 'quiet', + print_format, + 'json', + '-show_format', + '-show_streams', + '-show_error', videofile, ] print_cmd(command) if img: procin = zip_out(file, img, bitbucket) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stdin=procin.stdout, + ) procin.stdout.close() else: proc = subprocess.Popen(command, stdout=subprocess.PIPE) @@ -109,11 +164,22 @@ def get_video_details(videofile, img=None, bitbucket=None): video_details = json.loads(out.decode()) except Exception: try: # try this again without -show error in case of ffmpeg limitation - command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] + command = [ + core.FFPROBE, + '-v', + 'quiet', + print_format, + 'json', + '-show_format', + '-show_streams', + videofile, + ] print_cmd(command) if img: procin = zip_out(file, img, bitbucket) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stdin=procin.stdout, + ) procin.stdout.close() else: proc = subprocess.Popen(command, stdout=subprocess.PIPE) @@ -132,8 +198,16 @@ def check_vid_file(video_details, result): return False if not video_details.get('streams'): return False - video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] - audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + video_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'video' + ] + audio_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'audio' + ] if len(video_streams) > 0 and len(audio_streams) > 0: return True else: @@ -152,11 +226,15 @@ def build_commands(file, new_dir, movie_name, bitbucket): if check and core.CONCAT: name = movie_name elif check: - name = (f'{movie_name}.cd{check.groups()[0]}') + name = f'{movie_name}.cd{check.groups()[0]}' elif core.CONCAT and re.match('(.+)[cC][dD][0-9]', name): name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) - if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. - core.VEXTENSION = f'-transcoded{core.VEXTENSION}' # adds '-transcoded.ext' + if ( + ext == core.VEXTENSION and new_dir == directory + ): # we need to change the name to prevent overwriting itself. + core.VEXTENSION = ( + f'-transcoded{core.VEXTENSION}' # adds '-transcoded.ext' + ) new_file = file else: img, data = next(file.items()) @@ -165,15 +243,21 @@ def build_commands(file, new_dir, movie_name, bitbucket): rem_vid = [] for vid in data['files']: video_details, result = get_video_details(vid, img, bitbucket) - if not check_vid_file(video_details, result): # lets not transcode menu or other clips that don't have audio and video. + if not check_vid_file( + video_details, result, + ): # lets not transcode menu or other clips that don't have audio and video. rem_vid.append(vid) data['files'] = [f for f in data['files'] if f not in rem_vid] new_file = {img: {'name': data['name'], 'files': data['files']}} - video_details, result = get_video_details(data['files'][0], img, bitbucket) + video_details, result = get_video_details( + data['files'][0], img, bitbucket, + ) input_file = '-' file = '-' - newfile_path = os.path.normpath(os.path.join(new_dir, name) + core.VEXTENSION) + newfile_path = os.path.normpath( + os.path.join(new_dir, name) + core.VEXTENSION, + ) map_cmd = [] video_cmd = [] @@ -184,7 +268,7 @@ def build_commands(file, new_dir, movie_name, bitbucket): other_cmd = [] if not video_details or not video_details.get( - 'streams', + 'streams', ): # we couldn't read streams with ffprobe. Set defaults to try transcoding. video_streams = [] audio_streams = [] @@ -228,7 +312,9 @@ def build_commands(file, new_dir, movie_name, bitbucket): if core.SCODEC and core.ALLOWSUBS: sub_cmd.extend(['-c:s', core.SCODEC]) - elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format! + elif ( + core.ALLOWSUBS + ): # Not every subtitle codec can be used for every video container format! sub_cmd.extend(['-c:s', 'copy']) else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options sub_cmd.extend(['-sn']) # Don't copy the subtitles over @@ -237,15 +323,28 @@ def build_commands(file, new_dir, movie_name, bitbucket): other_cmd.extend(['-movflags', '+faststart']) else: - video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] - audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] - sub_streams = [item for item in video_details['streams'] if item['codec_type'] == 'subtitle'] + video_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'video' + ] + audio_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'audio' + ] + sub_streams = [ + item + for item in video_details['streams'] + if item['codec_type'] == 'subtitle' + ] if core.VEXTENSION not in ['.mkv', '.mpegts']: sub_streams = [ - item for item in video_details['streams'] if - item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item[ - 'codec_name' - ] != 'pgssub' + item + for item in video_details['streams'] + if item['codec_type'] == 'subtitle' + and item['codec_name'] != 'hdmv_pgs_subtitle' + and item['codec_name'] != 'pgssub' ] for video in video_streams: @@ -258,7 +357,9 @@ def build_commands(file, new_dir, movie_name, bitbucket): video_cmd.extend(['-c:v', 'copy']) else: video_cmd.extend(['-c:v', core.VCODEC]) - if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001): + if core.VFRAMERATE and not ( + core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001 + ): video_cmd.extend(['-r', str(core.VFRAMERATE)]) if scale: w_scale = width / float(scale.split(':')[0]) @@ -288,7 +389,9 @@ def build_commands(file, new_dir, movie_name, bitbucket): no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v'] if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy): video_cmd[1] = core.VCODEC - if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding. + if ( + core.VCODEC == 'copy' + ): # force copy. therefore ignore all other video transcoding. video_cmd = ['-c:v', 'copy'] map_cmd.extend(['-map', '0:{index}'.format(index=video['index'])]) break # Only one video needed @@ -299,63 +402,99 @@ def build_commands(file, new_dir, movie_name, bitbucket): if audio_streams: for i, val in reversed(list(enumerate(audio_streams))): try: - if 'Commentary' in val.get('tags').get('title'): # Split out commentry tracks. + if 'Commentary' in val.get('tags').get( + 'title', + ): # Split out commentry tracks. commentary.append(val) del audio_streams[i] except Exception: continue try: - audio1 = [item for item in audio_streams if item['tags']['language'] == core.ALANGUAGE] + audio1 = [ + item + for item in audio_streams + if item['tags']['language'] == core.ALANGUAGE + ] except Exception: # no language tags. Assume only 1 language. audio1 = audio_streams try: - audio2 = [item for item in audio1 if item['codec_name'] in core.ACODEC_ALLOW] + audio2 = [ + item + for item in audio1 + if item['codec_name'] in core.ACODEC_ALLOW + ] except Exception: audio2 = [] try: - audio3 = [item for item in audio_streams if item['tags']['language'] != core.ALANGUAGE] + audio3 = [ + item + for item in audio_streams + if item['tags']['language'] != core.ALANGUAGE + ] except Exception: audio3 = [] try: - audio4 = [item for item in audio3 if item['codec_name'] in core.ACODEC_ALLOW] + audio4 = [ + item + for item in audio3 + if item['codec_name'] in core.ACODEC_ALLOW + ] except Exception: audio4 = [] if audio2: # right (or only) language and codec... - map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio2[0]['index'])], + ) a_mapped.extend([audio2[0]['index']]) bitrate = int(float(audio2[0].get('bit_rate', 0))) / 1000 channels = int(float(audio2[0].get('channels', 0))) audio_cmd.extend([f'-c:a:{used_audio}', 'copy']) elif audio1: # right (or only) language, wrong codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio1[0]['index'])], + ) a_mapped.extend([audio1[0]['index']]) bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 channels = int(float(audio1[0].get('channels', 0))) - audio_cmd.extend([f'-c:a:{used_audio}', core.ACODEC if core.ACODEC else 'copy']) + audio_cmd.extend( + [f'-c:a:{used_audio}', core.ACODEC if core.ACODEC else 'copy'], + ) elif audio4: # wrong language, right codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio4[0]['index'])], + ) a_mapped.extend([audio4[0]['index']]) bitrate = int(float(audio4[0].get('bit_rate', 0))) / 1000 channels = int(float(audio4[0].get('channels', 0))) audio_cmd.extend([f'-c:a:{used_audio}', 'copy']) - elif audio3: # wrong language, wrong codec. just pick the default audio track - map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])]) + elif ( + audio3 + ): # wrong language, wrong codec. just pick the default audio track + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio3[0]['index'])], + ) a_mapped.extend([audio3[0]['index']]) bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 channels = int(float(audio3[0].get('channels', 0))) - audio_cmd.extend([f'-c:a:{used_audio}', core.ACODEC if core.ACODEC else 'copy']) + audio_cmd.extend( + [f'-c:a:{used_audio}', core.ACODEC if core.ACODEC else 'copy'], + ) if core.ACHANNELS and channels and channels > core.ACHANNELS: audio_cmd.extend([f'-ac:a:{used_audio}', str(core.ACHANNELS)]) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC - if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1): + if core.ABITRATE and not ( + core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1 + ): audio_cmd.extend([f'-b:a:{used_audio}', str(core.ABITRATE)]) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC if core.OUTPUTQUALITYPERCENT: - audio_cmd.extend([f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)]) + audio_cmd.extend( + [f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)], + ) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC if audio_cmd[1] in ['aac', 'dts']: @@ -364,21 +503,33 @@ def build_commands(file, new_dir, movie_name, bitbucket): if core.ACODEC2_ALLOW: used_audio += 1 try: - audio5 = [item for item in audio1 if item['codec_name'] in core.ACODEC2_ALLOW] + audio5 = [ + item + for item in audio1 + if item['codec_name'] in core.ACODEC2_ALLOW + ] except Exception: audio5 = [] try: - audio6 = [item for item in audio3 if item['codec_name'] in core.ACODEC2_ALLOW] + audio6 = [ + item + for item in audio3 + if item['codec_name'] in core.ACODEC2_ALLOW + ] except Exception: audio6 = [] if audio5: # right language and codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio5[0]['index'])], + ) a_mapped.extend([audio5[0]['index']]) bitrate = int(float(audio5[0].get('bit_rate', 0))) / 1000 channels = int(float(audio5[0].get('channels', 0))) audio_cmd2.extend([f'-c:a:{used_audio}', 'copy']) elif audio1: # right language wrong codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio1[0]['index'])], + ) a_mapped.extend([audio1[0]['index']]) bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 channels = int(float(audio1[0].get('channels', 0))) @@ -387,13 +538,19 @@ def build_commands(file, new_dir, movie_name, bitbucket): else: audio_cmd2.extend([f'-c:a:{used_audio}', 'copy']) elif audio6: # wrong language, right codec - map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio6[0]['index'])], + ) a_mapped.extend([audio6[0]['index']]) bitrate = int(float(audio6[0].get('bit_rate', 0))) / 1000 channels = int(float(audio6[0].get('channels', 0))) audio_cmd2.extend([f'-c:a:{used_audio}', 'copy']) - elif audio3: # wrong language, wrong codec just pick the default audio track - map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])]) + elif ( + audio3 + ): # wrong language, wrong codec just pick the default audio track + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio3[0]['index'])], + ) a_mapped.extend([audio3[0]['index']]) bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 channels = int(float(audio3[0].get('channels', 0))) @@ -403,21 +560,29 @@ def build_commands(file, new_dir, movie_name, bitbucket): audio_cmd2.extend([f'-c:a:{used_audio}', 'copy']) if core.ACHANNELS2 and channels and channels > core.ACHANNELS2: - audio_cmd2.extend([f'-ac:a:{used_audio}', str(core.ACHANNELS2)]) + audio_cmd2.extend( + [f'-ac:a:{used_audio}', str(core.ACHANNELS2)], + ) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 - if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1): + if core.ABITRATE2 and not ( + core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1 + ): audio_cmd2.extend([f'-b:a:{used_audio}', str(core.ABITRATE2)]) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 if core.OUTPUTQUALITYPERCENT: - audio_cmd2.extend([f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)]) + audio_cmd2.extend( + [f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)], + ) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 if audio_cmd2[1] in ['aac', 'dts']: audio_cmd2[2:2] = ['-strict', '-2'] - if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[1:]: # check for duplicate output track. + if ( + a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[1:] + ): # check for duplicate output track. del map_cmd[-2:] else: audio_cmd.extend(audio_cmd2) @@ -428,7 +593,9 @@ def build_commands(file, new_dir, movie_name, bitbucket): if audio['index'] in a_mapped: continue used_audio += 1 - map_cmd.extend(['-map', '0:{index}'.format(index=audio['index'])]) + map_cmd.extend( + ['-map', '0:{index}'.format(index=audio['index'])], + ) audio_cmd3 = [] bitrate = int(float(audio.get('bit_rate', 0))) / 1000 channels = int(float(audio.get('channels', 0))) @@ -441,15 +608,23 @@ def build_commands(file, new_dir, movie_name, bitbucket): audio_cmd3.extend([f'-c:a:{used_audio}', 'copy']) if core.ACHANNELS3 and channels and channels > core.ACHANNELS3: - audio_cmd3.extend([f'-ac:a:{used_audio}', str(core.ACHANNELS3)]) + audio_cmd3.extend( + [f'-ac:a:{used_audio}', str(core.ACHANNELS3)], + ) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 - if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1): - audio_cmd3.extend([f'-b:a:{used_audio}', str(core.ABITRATE3)]) + if core.ABITRATE3 and not ( + core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1 + ): + audio_cmd3.extend( + [f'-b:a:{used_audio}', str(core.ABITRATE3)], + ) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 if core.OUTPUTQUALITYPERCENT > 0: - audio_cmd3.extend([f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)]) + audio_cmd3.extend( + [f'-q:a:{used_audio}', str(core.OUTPUTQUALITYPERCENT)], + ) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 if audio_cmd3[1] in ['aac', 'dts']: @@ -461,7 +636,9 @@ def build_commands(file, new_dir, movie_name, bitbucket): n = 0 for lan in core.SLANGUAGES: try: - subs1 = [item for item in sub_streams if item['tags']['language'] == lan] + subs1 = [ + item for item in sub_streams if item['tags']['language'] == lan + ] except Exception: subs1 = [] if core.BURN and not subs1 and not burnt and os.path.isfile(file): @@ -476,11 +653,16 @@ def build_commands(file, new_dir, movie_name, bitbucket): if sub_streams[index]['index'] == sub['index']: subloc = index break - video_cmd.extend(['-vf', f'subtitles={input_file}:si={subloc}']) + video_cmd.extend( + ['-vf', f'subtitles={input_file}:si={subloc}'], + ) burnt = 1 if not core.ALLOWSUBS: break - if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these. + if ( + sub['codec_name'] in ['dvd_subtitle', 'VobSub'] + and core.SCODEC == 'mov_text' + ): # We can't convert these. continue map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) s_mapped.extend([sub['index']]) @@ -491,7 +673,10 @@ def build_commands(file, new_dir, movie_name, bitbucket): break if sub['index'] in s_mapped: continue - if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these. + if ( + sub['codec_name'] in ['dvd_subtitle', 'VobSub'] + and core.SCODEC == 'mov_text' + ): # We can't convert these. continue map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) s_mapped.extend([sub['index']]) @@ -516,11 +701,17 @@ def build_commands(file, new_dir, movie_name, bitbucket): if not sub_details or not sub_details.get('streams'): continue if core.SCODEC == 'mov_text': - subcode = [stream['codec_name'] for stream in sub_details['streams']] - if set(subcode).intersection(['dvd_subtitle', 'VobSub']): # We can't convert these. + subcode = [ + stream['codec_name'] for stream in sub_details['streams'] + ] + if set(subcode).intersection( + ['dvd_subtitle', 'VobSub'], + ): # We can't convert these. continue command.extend(['-i', subfile]) - lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0] + lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split( + '-', + )[0] metlan = None try: if len(lan) == 3: @@ -530,10 +721,12 @@ def build_commands(file, new_dir, movie_name, bitbucket): except Exception: pass if metlan: - meta_cmd.extend([ - f'-metadata:s:s:{len(s_mapped) + n}', - f'language={metlan.alpha3}', - ]) + meta_cmd.extend( + [ + f'-metadata:s:s:{len(s_mapped) + n}', + f'language={metlan.alpha3}', + ], + ) n += 1 map_cmd.extend(['-map', f'{n}:0']) @@ -565,7 +758,11 @@ def get_subs(file): for directory, _, filenames in os.walk(path): for filename in filenames: filepaths.extend([os.path.join(directory, filename)]) - subfiles = [item for item in filepaths if os.path.splitext(item)[1] in sub_ext and name in item] + subfiles = [ + item + for item in filepaths + if os.path.splitext(item)[1] in sub_ext and name in item + ] return subfiles @@ -582,17 +779,20 @@ def extract_subs(file, newfile_path, bitbucket): try: sub_streams = [ - item for item in video_details['streams'] if - item['codec_type'] == 'subtitle' and item['tags']['language'] in core.SLANGUAGES and item[ - 'codec_name' - ] != 'hdmv_pgs_subtitle' and item['codec_name'] != 'pgssub' + item + for item in video_details['streams'] + if item['codec_type'] == 'subtitle' + and item['tags']['language'] in core.SLANGUAGES + and item['codec_name'] != 'hdmv_pgs_subtitle' + and item['codec_name'] != 'pgssub' ] except Exception: sub_streams = [ - item for item in video_details['streams'] if - item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item[ - 'codec_name' - ] != 'pgssub' + item + for item in video_details['streams'] + if item['codec_type'] == 'subtitle' + and item['codec_name'] != 'hdmv_pgs_subtitle' + and item['codec_name'] != 'pgssub' ] num = len(sub_streams) for n in range(num): @@ -610,8 +810,16 @@ def extract_subs(file, newfile_path, bitbucket): output_file = os.path.join(subdir, f'{name}.{lan}.{n}.srt') command = [ - core.FFMPEG, '-loglevel', 'warning', '-i', file, '-vn', '-an', - f'-codec:{idx}', 'srt', output_file, + core.FFMPEG, + '-loglevel', + 'warning', + '-i', + file, + '-vn', + '-an', + f'-codec:{idx}', + 'srt', + output_file, ] if platform.system() != 'Windows': command = core.NICENESS + command @@ -620,7 +828,9 @@ def extract_subs(file, newfile_path, bitbucket): print_cmd(command) result = 1 # set result to failed in case call fails. try: - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc = subprocess.Popen( + command, stdout=bitbucket, stderr=bitbucket, + ) out, err = proc.communicate() result = proc.returncode except Exception: @@ -645,11 +855,17 @@ def process_list(it, new_dir, bitbucket): success = True for item in it: ext = os.path.splitext(item)[1].lower() - if ext in ['.iso', '.bin', '.img'] and ext not in core.IGNOREEXTENSIONS: + if ( + ext in ['.iso', '.bin', '.img'] + and ext not in core.IGNOREEXTENSIONS + ): logger.debug(f'Attempting to rip disk image: {item}', 'TRANSCODER') new_list.extend(rip_iso(item, new_dir, bitbucket)) rem_list.append(item) - elif re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', item) and '.vob' not in core.IGNOREEXTENSIONS: + elif ( + re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', item) + and '.vob' not in core.IGNOREEXTENSIONS + ): logger.debug(f'Found VIDEO_TS image file: {item}', 'TRANSCODER') if not vts_path: try: @@ -657,15 +873,22 @@ def process_list(it, new_dir, bitbucket): except Exception: vts_path = os.path.split(item)[0] rem_list.append(item) - elif re.match('.+BDMV[/\\]SOURCE[/\\][0-9]+[0-9].[Mm][Tt][Ss]', item) and '.mts' not in core.IGNOREEXTENSIONS: + elif ( + re.match('.+BDMV[/\\]SOURCE[/\\][0-9]+[0-9].[Mm][Tt][Ss]', item) + and '.mts' not in core.IGNOREEXTENSIONS + ): logger.debug(f'Found MTS image file: {item}', 'TRANSCODER') if not mts_path: try: - mts_path = re.match('(.+BDMV[/\\]SOURCE)', item).groups()[0] + mts_path = re.match('(.+BDMV[/\\]SOURCE)', item).groups()[ + 0 + ] except Exception: mts_path = os.path.split(item)[0] rem_list.append(item) - elif re.match('.+VIDEO_TS.', item) or re.match('.+VTS_[0-9][0-9]_[0-9].', item): + elif re.match('.+VIDEO_TS.', item) or re.match( + '.+VTS_[0-9][0-9]_[0-9].', item, + ): rem_list.append(item) elif core.CONCAT and re.match('.+[cC][dD][0-9].', item): rem_list.append(item) @@ -679,24 +902,39 @@ def process_list(it, new_dir, bitbucket): if combine: new_list.extend(combine_cd(combine)) for file in new_list: - if isinstance(file, str) and 'concat:' not in file and not os.path.isfile(file): + if ( + isinstance(file, str) + and 'concat:' not in file + and not os.path.isfile(file) + ): success = False break if success and new_list: it.extend(new_list) for item in rem_list: it.remove(item) - logger.debug(f'Successfully extracted .vob file {new_list[0]} from disk image', 'TRANSCODER') + logger.debug( + f'Successfully extracted .vob file {new_list[0]} from disk image', + 'TRANSCODER', + ) elif new_list and not success: new_list = [] rem_list = [] - logger.error('Failed extracting .vob files from disk image. Stopping transcoding.', 'TRANSCODER') + logger.error( + 'Failed extracting .vob files from disk image. Stopping transcoding.', + 'TRANSCODER', + ) return it, rem_list, new_list, success -def mount_iso(item, new_dir, bitbucket): # Currently only supports Linux Mount when permissions allow. +def mount_iso( + item, new_dir, bitbucket, +): # Currently only supports Linux Mount when permissions allow. if platform.system() == 'Windows': - logger.error(f'No mounting options available under Windows for image file {item}', 'TRANSCODER') + logger.error( + f'No mounting options available under Windows for image file {item}', + 'TRANSCODER', + ) return [] mount_point = os.path.join(os.path.dirname(os.path.abspath(item)), 'temp') make_dir(mount_point) @@ -704,25 +942,42 @@ def mount_iso(item, new_dir, bitbucket): # Currently only supports Linux Mount print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() - core.MOUNTED = mount_point # Allows us to verify this has been done and then cleanup. + core.MOUNTED = ( + mount_point # Allows us to verify this has been done and then cleanup. + ) for root, dirs, files in os.walk(mount_point): for file in files: full_path = os.path.join(root, file) - if re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', full_path) and '.vob' not in core.IGNOREEXTENSIONS: - logger.debug(f'Found VIDEO_TS image file: {full_path}', 'TRANSCODER') + if ( + re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', full_path) + and '.vob' not in core.IGNOREEXTENSIONS + ): + logger.debug( + f'Found VIDEO_TS image file: {full_path}', 'TRANSCODER', + ) try: vts_path = re.match('(.+VIDEO_TS)', full_path).groups()[0] except Exception: vts_path = os.path.split(full_path)[0] return combine_vts(vts_path) - elif re.match('.+BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]', full_path) and '.mts' not in core.IGNOREEXTENSIONS: - logger.debug(f'Found MTS image file: {full_path}', 'TRANSCODER') + elif ( + re.match('.+BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]', full_path) + and '.mts' not in core.IGNOREEXTENSIONS + ): + logger.debug( + f'Found MTS image file: {full_path}', 'TRANSCODER', + ) try: - mts_path = re.match('(.+BDMV[/\\]STREAM)', full_path).groups()[0] + mts_path = re.match( + '(.+BDMV[/\\]STREAM)', full_path, + ).groups()[0] except Exception: mts_path = os.path.split(full_path)[0] return combine_mts(mts_path) - logger.error(f'No VIDEO_TS or BDMV/SOURCE folder found in image file {mount_point}', 'TRANSCODER') + logger.error( + f'No VIDEO_TS or BDMV/SOURCE folder found in image file {mount_point}', + 'TRANSCODER', + ) return ['failure'] # If we got here, nothing matched our criteria @@ -731,21 +986,34 @@ def rip_iso(item, new_dir, bitbucket): failure_dir = 'failure' # Mount the ISO in your OS and call combineVTS. if not core.SEVENZIP: - logger.debug(f'No 7zip installed. Attempting to mount image file {item}', 'TRANSCODER') + logger.debug( + f'No 7zip installed. Attempting to mount image file {item}', + 'TRANSCODER', + ) try: - new_files = mount_iso(item, new_dir, bitbucket) # Currently only works for Linux. + new_files = mount_iso( + item, new_dir, bitbucket, + ) # Currently only works for Linux. except Exception: - logger.error(f'Failed to mount and extract from image file {item}', 'TRANSCODER') + logger.error( + f'Failed to mount and extract from image file {item}', + 'TRANSCODER', + ) new_files = [failure_dir] return new_files cmd = [core.SEVENZIP, 'l', item] try: - logger.debug(f'Attempting to extract .vob or .mts from image file {item}', 'TRANSCODER') + logger.debug( + f'Attempting to extract .vob or .mts from image file {item}', + 'TRANSCODER', + ) print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() file_match_gen = ( - re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line) + re.match( + r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line, + ) for line in out.decode().splitlines() ) file_list = [ @@ -771,7 +1039,8 @@ def rip_iso(item, new_dir, bitbucket): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, + name=os.path.splitext(os.path.split(item)[1])[0], + x=n + 1, ) new_files.append({item: {'name': name, 'files': concat}}) else: # check BlueRay for BDMV/STREAM/XXXX.MTS @@ -785,9 +1054,13 @@ def rip_iso(item, new_dir, bitbucket): if file_match ] if sys.version_info[0] == 2: # Python2 sorting - mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) # Sort all .mts files in numerical order + mts_list.sort( + key=lambda f: int(filter(str.isdigit, f)), + ) # Sort all .mts files in numerical order else: # Python3 sorting - mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + mts_list.sort( + key=lambda f: int(''.join(filter(str.isdigit, f))), + ) n = 0 for mts_name in mts_list: concat = [] @@ -797,14 +1070,18 @@ def rip_iso(item, new_dir, bitbucket): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n, + name=os.path.splitext(os.path.split(item)[1])[0], + x=n, ) new_files.append({item: {'name': name, 'files': concat}}) if core.CONCAT and combined: name = os.path.splitext(os.path.split(item)[1])[0] new_files.append({item: {'name': name, 'files': combined}}) if not new_files: - logger.error(f'No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {item}', 'TRANSCODER') + logger.error( + f'No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {item}', + 'TRANSCODER', + ) new_files = mount_iso(item, new_dir, bitbucket) except Exception: logger.error(f'Failed to extract from image file {item}', 'TRANSCODER') @@ -836,7 +1113,8 @@ def combine_vts(vts_path): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=name, x=n + 1, + name=name, + x=n + 1, ) new_files.append({vts_path: {'name': name, 'files': concat}}) if core.CONCAT: @@ -853,7 +1131,11 @@ def combine_mts(mts_path): else: name = os.path.basename(name) n = 0 - mts_list = [f for f in os.listdir(mts_path) if os.path.isfile(os.path.join(mts_path, f))] + mts_list = [ + f + for f in os.listdir(mts_path) + if os.path.isfile(os.path.join(mts_path, f)) + ] if sys.version_info[0] == 2: # Python2 sorting mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) else: # Python3 sorting @@ -865,7 +1147,8 @@ def combine_mts(mts_path): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=name, x=n + 1, + name=name, + x=n + 1, ) new_files.append({mts_path: {'name': name, 'files': concat}}) n += 1 @@ -876,12 +1159,17 @@ def combine_mts(mts_path): def combine_cd(combine): new_files = [] - for item in {re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine}: + for item in { + re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine + }: concat = '' for n in range(99): files = [ - file for file in combine if - n + 1 == int(re.match('.+[cC][dD]([0-9]+).', file).groups()[0]) and item in file + file + for file in combine + if n + 1 + == int(re.match('.+[cC][dD]([0-9]+).', file).groups()[0]) + and item in file ] if files: concat += f'{files[0]}|' @@ -917,14 +1205,21 @@ def transcode_directory(dir_name): else: bitbucket = open('/dev/null') movie_name = os.path.splitext(os.path.split(dir_name)[1])[0] - file_list = core.list_media_files(dir_name, media=True, audio=False, meta=False, archives=False) - file_list, rem_list, new_list, success = process_list(file_list, new_dir, bitbucket) + file_list = core.list_media_files( + dir_name, media=True, audio=False, meta=False, archives=False, + ) + file_list, rem_list, new_list, success = process_list( + file_list, new_dir, bitbucket, + ) if not success: bitbucket.close() return 1, dir_name for file in file_list: - if isinstance(file, str) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: + if ( + isinstance(file, str) + and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS + ): continue command, file = build_commands(file, new_dir, movie_name, bitbucket) newfile_path = command[-1] @@ -936,7 +1231,9 @@ def transcode_directory(dir_name): try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) os.remove(newfile_path) except OSError as e: - if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist + if ( + e.errno != errno.ENOENT + ): # Ignore the error if it's just telling us that the file doesn't exist logger.debug(f'Error when removing transcoding target: {e}') except Exception as e: logger.debug(f'Error when removing transcoding target: {e}') @@ -946,10 +1243,17 @@ def transcode_directory(dir_name): result = 1 # set result to failed in case call fails. try: if isinstance(file, str): - proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE) + proc = subprocess.Popen( + command, stdout=bitbucket, stderr=subprocess.PIPE, + ) else: img, data = next(file.items()) - proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + proc = subprocess.Popen( + command, + stdout=bitbucket, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + ) for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: @@ -968,7 +1272,9 @@ def transcode_directory(dir_name): name = os.path.splitext(os.path.split(file)[1])[0] subname = os.path.split(sub)[1] newname = os.path.splitext(os.path.split(newfile_path)[1])[0] - newpath = os.path.join(core.SUBSDIR, subname.replace(name, newname)) + newpath = os.path.join( + core.SUBSDIR, subname.replace(name, newname), + ) if not os.path.isfile(newpath): os.rename(sub, newpath) @@ -978,13 +1284,17 @@ def transcode_directory(dir_name): except Exception: pass logger.info(f'Transcoding of video to {newfile_path} succeeded') - if os.path.isfile(newfile_path) and (file in new_list or not core.DUPLICATE): + if os.path.isfile(newfile_path) and ( + file in new_list or not core.DUPLICATE + ): try: os.unlink(file) except Exception: pass else: - logger.error(f'Transcoding of video to {newfile_path} failed with result {result}') + logger.error( + f'Transcoding of video to {newfile_path} failed with result {result}', + ) # this will be 0 (successful) it all are successful, else will return a positive integer for failure. final_result = final_result + result if core.MOUNTED: # In case we mounted an .iso file, unmount here. @@ -1002,10 +1312,14 @@ def transcode_directory(dir_name): os.unlink(file) except Exception: pass - if not os.listdir(new_dir): # this is an empty directory and we didn't transcode into it. + if not os.listdir( + new_dir, + ): # this is an empty directory and we didn't transcode into it. os.rmdir(new_dir) new_dir = dir_name - if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB + if ( + not core.PROCESSOUTPUT and core.DUPLICATE + ): # We postprocess the original files to CP/SB new_dir = dir_name bitbucket.close() return final_result, new_dir diff --git a/core/user_scripts.py b/core/user_scripts.py index d7d18e65..d9182950 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -14,12 +14,18 @@ from core.auto_process.common import ProcessResult def external_script(output_destination, torrent_name, torrent_label, settings): final_result = 0 # start at 0. num_files = 0 - core.USER_SCRIPT_MEDIAEXTENSIONS = settings.get('user_script_mediaExtensions', '') + core.USER_SCRIPT_MEDIAEXTENSIONS = settings.get( + 'user_script_mediaExtensions', '', + ) try: if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): - core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.lower().split(',') + core.USER_SCRIPT_MEDIAEXTENSIONS = ( + core.USER_SCRIPT_MEDIAEXTENSIONS.lower().split(',') + ) except Exception: - logger.error('user_script_mediaExtensions could not be set', 'USERSCRIPT') + logger.error( + 'user_script_mediaExtensions could not be set', 'USERSCRIPT', + ) core.USER_SCRIPT_MEDIAEXTENSIONS = [] core.USER_SCRIPT = settings.get('user_script_path', '') @@ -42,7 +48,9 @@ def external_script(output_destination, torrent_name, torrent_label, settings): core.USER_SCRIPT_SUCCESSCODES = settings.get('user_script_successCodes', 0) try: if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): - core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',') + core.USER_SCRIPT_SUCCESSCODES = ( + core.USER_SCRIPT_SUCCESSCODES.split(',') + ) except Exception: logger.error('user_script_successCodes could not be set', 'USERSCRIPT') core.USER_SCRIPT_SUCCESSCODES = 0 @@ -51,11 +59,20 @@ def external_script(output_destination, torrent_name, torrent_label, settings): core.USER_SCRIPT_RUNONCE = int(settings.get('user_script_runOnce', 1)) if core.CHECK_MEDIA: - for video in list_media_files(output_destination, media=True, audio=False, meta=False, archives=False): + for video in list_media_files( + output_destination, + media=True, + audio=False, + meta=False, + archives=False, + ): if transcoder.is_video_good(video, 0): import_subs(video) else: - logger.info(f'Corrupt video file found {video}. Deleting.', 'USERSCRIPT') + logger.info( + f'Corrupt video file found {video}. Deleting.', + 'USERSCRIPT', + ) os.unlink(video) for dirpath, _, filenames in os.walk(output_destination): @@ -63,11 +80,19 @@ def external_script(output_destination, torrent_name, torrent_label, settings): file_path = core.os.path.join(dirpath, file) file_name, file_extension = os.path.splitext(file) - logger.debug(f'Checking file {file} to see if this should be processed.', 'USERSCRIPT') + logger.debug( + f'Checking file {file} to see if this should be processed.', + 'USERSCRIPT', + ) - if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or 'all' in core.USER_SCRIPT_MEDIAEXTENSIONS: + if ( + file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS + or 'all' in core.USER_SCRIPT_MEDIAEXTENSIONS + ): num_files += 1 - if core.USER_SCRIPT_RUNONCE == 1 and num_files > 1: # we have already run once, so just continue to get number of files. + if ( + core.USER_SCRIPT_RUNONCE == 1 and num_files > 1 + ): # we have already run once, so just continue to get number of files. continue command = [core.USER_SCRIPT] for param in core.USER_SCRIPT_PARAM: @@ -95,19 +120,31 @@ def external_script(output_destination, torrent_name, torrent_label, settings): cmd = '' for item in command: cmd = f'{cmd} {item}' - logger.info(f'Running script {cmd} on file {file_path}.', 'USERSCRIPT') + logger.info( + f'Running script {cmd} on file {file_path}.', 'USERSCRIPT', + ) try: p = Popen(command) res = p.wait() - if str(res) in core.USER_SCRIPT_SUCCESSCODES: # Linux returns 0 for successful. + if ( + str(res) in core.USER_SCRIPT_SUCCESSCODES + ): # Linux returns 0 for successful. logger.info(f'UserScript {command[0]} was successfull') result = 0 else: - logger.error(f'UserScript {command[0]} has failed with return code: {res}', 'USERSCRIPT') - logger.info(f'If the UserScript completed successfully you should add {res} to the user_script_successCodes', 'USERSCRIPT') + logger.error( + f'UserScript {command[0]} has failed with return code: {res}', + 'USERSCRIPT', + ) + logger.info( + f'If the UserScript completed successfully you should add {res} to the user_script_successCodes', + 'USERSCRIPT', + ) result = int(1) except Exception: - logger.error(f'UserScript {command[0]} has failed', 'USERSCRIPT') + logger.error( + f'UserScript {command[0]} has failed', 'USERSCRIPT', + ) result = int(1) final_result += result @@ -116,14 +153,25 @@ def external_script(output_destination, torrent_name, torrent_label, settings): for file in filenames: file_name, file_extension = os.path.splitext(file) - if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or core.USER_SCRIPT_MEDIAEXTENSIONS == 'ALL': + if ( + file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS + or core.USER_SCRIPT_MEDIAEXTENSIONS == 'ALL' + ): num_files_new += 1 - if core.USER_SCRIPT_CLEAN == int(1) and num_files_new == 0 and final_result == 0: - logger.info(f'All files have been processed. Cleaning outputDirectory {output_destination}') + if ( + core.USER_SCRIPT_CLEAN == int(1) + and num_files_new == 0 + and final_result == 0 + ): + logger.info( + f'All files have been processed. Cleaning outputDirectory {output_destination}', + ) remove_dir(output_destination) elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0: - logger.info(f'{num_files} files were processed, but {num_files_new} still remain. outputDirectory will not be cleaned.') + logger.info( + f'{num_files} files were processed, but {num_files_new} still remain. outputDirectory will not be cleaned.', + ) return ProcessResult( status_code=final_result, message='User Script Completed', diff --git a/core/utils/common.py b/core/utils/common.py index fe7d3a40..c0b7e296 100644 --- a/core/utils/common.py +++ b/core/utils/common.py @@ -13,7 +13,9 @@ from core.utils.paths import flatten_dir def flatten(output_destination): - return flatten_dir(output_destination, list_media_files(output_destination)) + return flatten_dir( + output_destination, list_media_files(output_destination), + ) def clean_dir(path, section, subsection): @@ -21,7 +23,9 @@ def clean_dir(path, section, subsection): min_size = int(cfg.get('minSize', 0)) delete_ignored = int(cfg.get('delete_ignored', 0)) try: - files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) + files = list_media_files( + path, min_size=min_size, delete_ignored=delete_ignored, + ) except Exception: files = [] return clean_directory(path, files) @@ -37,21 +41,20 @@ def process_dir(path, link): # Generate list of sync files sync_files = ( - item for item in dir_contents + item + for item in dir_contents if os.path.splitext(item)[1] in ['.!sync', '.bts'] ) # Generate a list of file paths filepaths = ( - os.path.join(path, item) for item in dir_contents + os.path.join(path, item) + for item in dir_contents if item not in ['Thumbs.db', 'thumbs.db'] ) # Generate a list of media files - mediafiles = ( - item for item in filepaths - if os.path.isfile(item) - ) + mediafiles = (item for item in filepaths if os.path.isfile(item)) if any(sync_files): logger.info('') @@ -60,26 +63,23 @@ def process_dir(path, link): try: move_file(mediafile, path, link) except Exception as e: - logger.error(f'Failed to move {os.path.split(mediafile)[1]} to its own directory: {e}') + logger.error( + f'Failed to move {os.path.split(mediafile)[1]} to its own directory: {e}', + ) # removeEmptyFolders(path, removeRoot=False) # Generate all path contents - path_contents = ( - os.path.join(path, item) - for item in os.listdir(path) - ) + path_contents = (os.path.join(path, item) for item in os.listdir(path)) # Generate all directories from path contents - directories = ( - path for path in path_contents - if os.path.isdir(path) - ) + directories = (path for path in path_contents if os.path.isdir(path)) for directory in directories: dir_contents = os.listdir(directory) sync_files = ( - item for item in dir_contents + item + for item in dir_contents if os.path.splitext(item)[1] in ['.!sync', '.bts'] ) if not any(dir_contents) or any(sync_files): @@ -101,7 +101,9 @@ def get_dirs(section, subsection, link='hard'): try: to_return.extend(process_dir(directory, link)) except Exception as e: - logger.error(f'Failed to add directories from {watch_directory} for post-processing: {e}') + logger.error( + f'Failed to add directories from {watch_directory} for post-processing: {e}', + ) if core.USE_LINK == 'move': try: @@ -109,10 +111,14 @@ def get_dirs(section, subsection, link='hard'): if os.path.exists(output_directory): to_return.extend(process_dir(output_directory, link)) except Exception as e: - logger.error(f'Failed to add directories from {core.OUTPUT_DIRECTORY} for post-processing: {e}') + logger.error( + f'Failed to add directories from {core.OUTPUT_DIRECTORY} for post-processing: {e}', + ) if not to_return: - logger.debug(f'No directories identified in {section}:{subsection} for post-processing') + logger.debug( + f'No directories identified in {section}:{subsection} for post-processing', + ) return list(set(to_return)) diff --git a/core/utils/encoding.py b/core/utils/encoding.py index bbb930f1..5a2358c2 100644 --- a/core/utils/encoding.py +++ b/core/utils/encoding.py @@ -24,7 +24,7 @@ def char_replace(name_in): if (len(name) != 1) & (Idx < (len(name) - 1)): # Detect UTF-8 if ((name[Idx] == 0xC2) | (name[Idx] == 0xC3)) & ( - (name[Idx + 1] >= 0xA0) & (name[Idx + 1] <= 0xFF) + (name[Idx + 1] >= 0xA0) & (name[Idx + 1] <= 0xFF) ): encoding = 'utf-8' break @@ -56,7 +56,9 @@ def char_replace(name_in): def convert_to_ascii(input_name, dir_name): ascii_convert = int(core.CFG['ASCII']['convert']) - if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. + if ( + ascii_convert == 0 or os.name == 'nt' + ): # just return if we don't want to convert or on windows os and '\' is replaced!. return input_name, dir_name encoded, input_name = char_replace(input_name) @@ -77,14 +79,22 @@ def convert_to_ascii(input_name, dir_name): for subdirname in dirnames: encoded, subdirname2 = char_replace(subdirname) if encoded: - logger.info(f'Renaming directory to: {subdirname2}.', 'ENCODER') - os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) + logger.info( + f'Renaming directory to: {subdirname2}.', 'ENCODER', + ) + os.rename( + os.path.join(dirname, subdirname), + os.path.join(dirname, subdirname2), + ) for dirname, _, filenames in os.walk(dir_name): for filename in filenames: encoded, filename2 = char_replace(filename) if encoded: logger.info(f'Renaming file to: {filename2}.', 'ENCODER') - os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) + os.rename( + os.path.join(dirname, filename), + os.path.join(dirname, filename2), + ) return input_name, dir_name diff --git a/core/utils/files.py b/core/utils/files.py index 59fe495b..ffc11a16 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -20,7 +20,9 @@ from core.utils.paths import make_dir def move_file(mediafile, path, link): - logger.debug(f'Found file {os.path.split(mediafile)[1]} in root directory {path}.') + logger.debug( + f'Found file {os.path.split(mediafile)[1]} in root directory {path}.', + ) new_path = None file_ext = os.path.splitext(mediafile)[1] try: @@ -32,7 +34,9 @@ def move_file(mediafile, path, link): album = f.album # create new path - new_path = os.path.join(path, f'{sanitize_name(artist)} - {sanitize_name(album)}') + new_path = os.path.join( + path, f'{sanitize_name(artist)} - {sanitize_name(album)}', + ) elif file_ext in core.MEDIA_CONTAINER: f = guessit.guessit(mediafile) @@ -44,7 +48,9 @@ def move_file(mediafile, path, link): new_path = os.path.join(path, sanitize_name(title)) except Exception as e: - logger.error(f'Exception parsing name for media file: {os.path.split(mediafile)[1]}: {e}') + logger.error( + f'Exception parsing name for media file: {os.path.split(mediafile)[1]}: {e}', + ) if not new_path: title = os.path.splitext(os.path.basename(mediafile))[0] @@ -58,14 +64,19 @@ def move_file(mediafile, path, link): # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path2 = os.path.join( + os.path.join(os.path.split(new_path)[0], 'new'), + os.path.split(new_path)[1], + ) new_path = new_path2 # create new path if it does not exist if not os.path.exists(new_path): make_dir(new_path) - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + newfile = os.path.join( + new_path, sanitize_name(os.path.split(mediafile)[1]), + ) try: newfile = newfile.encode(core.SYS_ENCODING) except Exception: @@ -84,7 +95,9 @@ def is_min_size(input_name, min_size): try: input_size = get_dir_size(os.path.dirname(input_name)) except Exception: - logger.error(f'Failed to get file size for {input_name}', 'MINSIZE') + logger.error( + f'Failed to get file size for {input_name}', 'MINSIZE', + ) return True # Ignore files under a certain size @@ -100,7 +113,15 @@ def is_archive_file(filename): return False -def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): +def is_media_file( + mediafile, + media=True, + audio=True, + meta=True, + archives=True, + other=False, + otherext=None, +): if otherext is None: otherext = [] @@ -113,16 +134,28 @@ def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, o except Exception: pass - return any([ - (media and file_ext.lower() in core.MEDIA_CONTAINER), - (audio and file_ext.lower() in core.AUDIO_CONTAINER), - (meta and file_ext.lower() in core.META_CONTAINER), - (archives and is_archive_file(mediafile)), - (other and (file_ext.lower() in otherext or 'all' in otherext)), - ]) + return any( + [ + (media and file_ext.lower() in core.MEDIA_CONTAINER), + (audio and file_ext.lower() in core.AUDIO_CONTAINER), + (meta and file_ext.lower() in core.META_CONTAINER), + (archives and is_archive_file(mediafile)), + (other and (file_ext.lower() in otherext or 'all' in otherext)), + ], + ) -def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): +def list_media_files( + path, + min_size=0, + delete_ignored=0, + media=True, + audio=True, + meta=True, + archives=True, + other=False, + otherext=None, +): if otherext is None: otherext = [] @@ -130,13 +163,17 @@ def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, if not os.path.isdir(path): if os.path.isfile(path): # Single file downloads. cur_file = os.path.split(path)[1] - if is_media_file(cur_file, media, audio, meta, archives, other, otherext): + if is_media_file( + cur_file, media, audio, meta, archives, other, otherext, + ): # Optionally ignore sample files if is_sample(path) or not is_min_size(path, min_size): if delete_ignored == 1: try: os.unlink(path) - logger.debug(f'Ignored file {cur_file} has been removed ...') + logger.debug( + f'Ignored file {cur_file} has been removed ...', + ) except Exception: pass else: @@ -149,15 +186,31 @@ def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, # if it's a folder do it recursively if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): - files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) + files += list_media_files( + full_cur_file, + min_size, + delete_ignored, + media, + audio, + meta, + archives, + other, + otherext, + ) - elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): + elif is_media_file( + cur_file, media, audio, meta, archives, other, otherext, + ): # Optionally ignore sample files - if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): + if is_sample(full_cur_file) or not is_min_size( + full_cur_file, min_size, + ): if delete_ignored == 1: try: os.unlink(full_cur_file) - logger.debug(f'Ignored file {cur_file} has been removed ...') + logger.debug( + f'Ignored file {cur_file} has been removed ...', + ) except Exception: pass continue @@ -171,7 +224,9 @@ def extract_files(src, dst=None, keep_archive=None): extracted_folder = [] extracted_archive = [] - for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): + for inputFile in list_media_files( + src, media=False, audio=False, meta=False, archives=True, + ): dir_path = os.path.dirname(inputFile) full_file_name = os.path.basename(inputFile) archive_name = os.path.splitext(full_file_name)[0] @@ -188,13 +243,17 @@ def extract_files(src, dst=None, keep_archive=None): logger.error(f'Extraction failed for: {full_file_name}') for folder in extracted_folder: - for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): + for inputFile in list_media_files( + folder, media=False, audio=False, meta=False, archives=True, + ): full_file_name = os.path.basename(inputFile) archive_name = os.path.splitext(full_file_name)[0] archive_name = re.sub(r'part[0-9]+', '', archive_name) if archive_name not in extracted_archive or keep_archive: continue # don't remove if we haven't extracted this archive, or if we want to preserve them. - logger.info(f'Removing extracted archive {full_file_name} from folder {folder} ...') + logger.info( + f'Removing extracted archive {full_file_name} from folder {folder} ...', + ) try: if not os.access(inputFile, os.W_OK): os.chmod(inputFile, stat.S_IWUSR) @@ -211,24 +270,35 @@ def backup_versioned_file(old_file, version): while not os.path.isfile(new_file): if not os.path.isfile(old_file): - logger.log(f'Not creating backup, {old_file} doesn\'t exist', logger.DEBUG) + logger.log( + f'Not creating backup, {old_file} doesn\'t exist', logger.DEBUG, + ) break try: - logger.log('Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) + logger.log( + 'Trying to back up {old} to {new]'.format( + old=old_file, new=new_file, + ), + logger.DEBUG, + ) shutil.copy(old_file, new_file) logger.log('Backup done', logger.DEBUG) break except Exception as error: logger.log( - f'Error while trying to back up {old_file} to {new_file} : {error}', logger.WARNING, + f'Error while trying to back up {old_file} to {new_file} : {error}', + logger.WARNING, ) num_tries += 1 time.sleep(1) logger.log('Trying again.', logger.DEBUG) if num_tries >= 10: - logger.log(f'Unable to back up {old_file} to {new_file} please do it manually.', logger.ERROR) + logger.log( + f'Unable to back up {old_file} to {new_file} please do it manually.', + logger.ERROR, + ) return False return True diff --git a/core/utils/identification.py b/core/utils/identification.py index 7755fb44..7bbd3ff8 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -32,7 +32,9 @@ def find_imdbid(dir_name, input_name, omdb_api_key): if 'NZBPR__DNZB_MOREINFO' in os.environ: dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') if dnzb_more_info != '': - regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) + regex = re.compile( + r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE, + ) m = regex.match(dnzb_more_info) if m: imdbid = m.group(1) @@ -57,15 +59,19 @@ def find_imdbid(dir_name, input_name, omdb_api_key): url = 'http://www.omdbapi.com' if not omdb_api_key: - logger.info('Unable to determine imdbID: No api key provided for omdbapi.com.') + logger.info( + 'Unable to determine imdbID: No api key provided for omdbapi.com.', + ) return logger.debug(f'Opening URL: {url}') try: r = requests.get( - url, params={'apikey': omdb_api_key, 'y': year, 't': title}, - verify=False, timeout=(60, 300), + url, + params={'apikey': omdb_api_key, 'y': year, 't': title}, + verify=False, + timeout=(60, 300), ) except requests.ConnectionError: logger.error(f'Unable to open URL {url}') @@ -89,7 +95,9 @@ def find_imdbid(dir_name, input_name, omdb_api_key): return imdbid -def category_search(input_directory, input_name, input_category, root, categories): +def category_search( + input_directory, input_name, input_category, root, categories, +): tordir = False if input_directory is None: # =Nothing to process here. @@ -98,64 +106,103 @@ def category_search(input_directory, input_name, input_category, root, categorie pathlist = os.path.normpath(input_directory).split(os.sep) if input_category and input_category in pathlist: - logger.debug(f'SEARCH: Found the Category: {input_category} in directory structure') + logger.debug( + f'SEARCH: Found the Category: {input_category} in directory structure', + ) elif input_category: - logger.debug(f'SEARCH: Could not find the category: {input_category} in the directory structure') + logger.debug( + f'SEARCH: Could not find the category: {input_category} in the directory structure', + ) else: try: - input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. - logger.debug(f'SEARCH: Found Category: {input_category} in directory structure') + input_category = list(set(pathlist) & set(categories))[ + -1 + ] # assume last match is most relevant category. + logger.debug( + f'SEARCH: Found Category: {input_category} in directory structure', + ) except IndexError: input_category = '' - logger.debug('SEARCH: Could not find a category in the directory structure') - if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file + logger.debug( + 'SEARCH: Could not find a category in the directory structure', + ) + if not os.path.isdir(input_directory) and os.path.isfile( + input_directory, + ): # If the input directory is a file if not input_name: input_name = os.path.split(os.path.normpath(input_directory))[1] return input_directory, input_name, input_category, root - if input_category and os.path.isdir(os.path.join(input_directory, input_category)): + if input_category and os.path.isdir( + os.path.join(input_directory, input_category), + ): logger.info( f'SEARCH: Found category directory {input_category} in input directory directory {input_directory}', ) input_directory = os.path.join(input_directory, input_category) logger.info(f'SEARCH: Setting input_directory to {input_directory}') if input_name and os.path.isdir(os.path.join(input_directory, input_name)): - logger.info(f'SEARCH: Found torrent directory {input_name} in input directory directory {input_directory}') + logger.info( + f'SEARCH: Found torrent directory {input_name} in input directory directory {input_directory}', + ) input_directory = os.path.join(input_directory, input_name) logger.info(f'SEARCH: Setting input_directory to {input_directory}') tordir = True - elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): + elif input_name and os.path.isdir( + os.path.join(input_directory, sanitize_name(input_name)), + ): logger.info( f'SEARCH: Found torrent directory {sanitize_name(input_name)} in input directory directory {input_directory}', ) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) + input_directory = os.path.join( + input_directory, sanitize_name(input_name), + ) logger.info(f'SEARCH: Setting input_directory to {input_directory}') tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): - logger.info(f'SEARCH: Found torrent file {input_name} in input directory directory {input_directory}') + elif input_name and os.path.isfile( + os.path.join(input_directory, input_name), + ): + logger.info( + f'SEARCH: Found torrent file {input_name} in input directory directory {input_directory}', + ) input_directory = os.path.join(input_directory, input_name) logger.info(f'SEARCH: Setting input_directory to {input_directory}') tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): + elif input_name and os.path.isfile( + os.path.join(input_directory, sanitize_name(input_name)), + ): logger.info( f'SEARCH: Found torrent file {sanitize_name(input_name)} in input directory directory {input_directory}', ) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) + input_directory = os.path.join( + input_directory, sanitize_name(input_name), + ) logger.info(f'SEARCH: Setting input_directory to {input_directory}') tordir = True elif input_name and os.path.isdir(input_directory): for file in os.listdir(input_directory): - if os.path.splitext(file)[0] in [input_name, sanitize_name(input_name)]: - logger.info(f'SEARCH: Found torrent file {file} in input directory directory {input_directory}') + if os.path.splitext(file)[0] in [ + input_name, + sanitize_name(input_name), + ]: + logger.info( + f'SEARCH: Found torrent file {file} in input directory directory {input_directory}', + ) input_directory = os.path.join(input_directory, file) - logger.info(f'SEARCH: Setting input_directory to {input_directory}') + logger.info( + f'SEARCH: Setting input_directory to {input_directory}', + ) input_name = file tordir = True break - imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. + imdbid = [ + item for item in pathlist if '.cp(tt' in item + ] # This looks for the .cp(tt imdb id in the path. if imdbid and '.cp(tt' not in input_name: - input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP + input_name = imdbid[ + 0 + ] # This ensures the imdb id is preserved and passed to CP tordir = True if input_category and not tordir: @@ -163,7 +210,9 @@ def category_search(input_directory, input_name, input_category, root, categorie index = pathlist.index(input_category) if index + 1 < len(pathlist): tordir = True - logger.info(f'SEARCH: Found a unique directory {pathlist[index + 1]} in the category directory') + logger.info( + f'SEARCH: Found a unique directory {pathlist[index + 1]} in the category directory', + ) if not input_name: input_name = pathlist[index + 1] except ValueError: @@ -171,7 +220,9 @@ def category_search(input_directory, input_name, input_category, root, categorie if input_name and not tordir: if input_name in pathlist or sanitize_name(input_name) in pathlist: - logger.info(f'SEARCH: Found torrent directory {input_name} in the directory structure') + logger.info( + f'SEARCH: Found torrent directory {input_name} in the directory structure', + ) tordir = True else: root = 1 @@ -179,7 +230,11 @@ def category_search(input_directory, input_name, input_category, root, categorie root = 2 if root > 0: - logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') - logger.info('SEARCH: We will try and determine which files to process, individually') + logger.info( + 'SEARCH: Could not find a unique directory for this download. Assume a common directory.', + ) + logger.info( + 'SEARCH: We will try and determine which files to process, individually', + ) return input_directory, input_name, input_category, root diff --git a/core/utils/links.py b/core/utils/links.py index 2075bcc6..b9333204 100644 --- a/core/utils/links.py +++ b/core/utils/links.py @@ -24,35 +24,57 @@ def copy_link(src, target_link, use_link): logger.info(f'TARGET FOLDER: [{os.path.dirname(target_link)}]', 'COPYLINK') if src != target_link and os.path.exists(target_link): - logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') + logger.info( + 'MEDIAFILE already exists in the TARGET folder, skipping ...', + 'COPYLINK', + ) return True - elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): - logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') + elif ( + src == target_link + and os.path.isfile(target_link) + and os.path.isfile(src) + ): + logger.info( + 'SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK', + ) return True elif src == os.path.dirname(target_link): - logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') + logger.info( + 'SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK', + ) return True make_dir(os.path.dirname(target_link)) try: if use_link == 'dir': - logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + logger.info( + 'Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK', + ) linktastic.dirlink(src, target_link) return True if use_link == 'junction': - logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + logger.info( + 'Directory junction linking SOURCE FOLDER -> TARGET FOLDER', + 'COPYLINK', + ) linktastic.dirlink(src, target_link) return True elif use_link == 'hard': - logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + logger.info( + 'Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK', + ) linktastic.link(src, target_link) return True elif use_link == 'sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + logger.info( + 'Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK', + ) linktastic.symlink(src, target_link) return True elif use_link == 'move-sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + logger.info( + 'Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK', + ) shutil.move(src, target_link) linktastic.symlink(target_link, src) return True @@ -81,9 +103,16 @@ def replace_links(link, max_depth=10): if not link_depth: logger.debug(f'{link} is not a link') - elif link_depth > max_depth or (link_depth == max_depth and islink(target)): - logger.warning(f'Exceeded maximum depth {max_depth} while following link {link}') + elif link_depth > max_depth or ( + link_depth == max_depth and islink(target) + ): + logger.warning( + f'Exceeded maximum depth {max_depth} while following link {link}', + ) else: - logger.info(f'Changing sym-link: {link} to point directly to file: {target}', 'COPYLINK') + logger.info( + f'Changing sym-link: {link} to point directly to file: {target}', + 'COPYLINK', + ) os.unlink(link) linktastic.symlink(target, link) diff --git a/core/utils/naming.py b/core/utils/naming.py index 6eb9362a..be66fe58 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -33,7 +33,9 @@ def clean_file_name(filename): space, but handles decimal numbers in string, for example: """ filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) - filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot + filename = re.sub( + r'(\d)\.(\d{4})', r'\1 \2', filename, + ) # if it ends in a year then don't keep the dot filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) filename = filename.replace('_', ' ') diff --git a/core/utils/network.py b/core/utils/network.py index 8ace77b9..4c0ed96b 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -12,10 +12,7 @@ from core import logger def make_wake_on_lan_packet(mac_address): """Build the Wake-On-LAN 'Magic Packet'.""" - address = ( - int(value, 16) - for value in mac_address.split(':') - ) + address = (int(value, 16) for value in mac_address.split(':')) fmt = 'BBBBBB' hardware_address = struct.pack(fmt, *address) broadcast_address = b'\xFF' * 6 # FF:FF:FF:FF:FF:FF @@ -115,7 +112,9 @@ def find_download(client_agent, download_id): 'value': download_id, } try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + r = requests.get( + url, params=params, verify=False, timeout=(30, 120), + ) except requests.ConnectionError: logger.error('Unable to open URL') return False # failure diff --git a/core/utils/parsers.py b/core/utils/parsers.py index aeaa2fce..3ff38e92 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -61,7 +61,11 @@ def parse_deluge(args): input_hash = args[1] input_id = args[1] try: - input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get(b'label').decode() + input_category = ( + core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']) + .get(b'label') + .decode() + ) except Exception: input_category = '' return input_directory, input_name, input_category, input_hash, input_id @@ -85,10 +89,16 @@ def parse_synods(args): input_name = os.getenv('TR_TORRENT_NAME') input_hash = os.getenv('TR_TORRENT_HASH') if not input_name: # No info passed. Assume manual download. - return input_directory, input_name, input_category, input_hash, input_id + return ( + input_directory, + input_name, + input_category, + input_hash, + input_id, + ) torrent_id = os.getenv('TR_TORRENT_ID') input_id = f'dbid_{torrent_id}' - #res = core.TORRENT_CLASS.tasks_list(additional_param='detail') + # res = core.TORRENT_CLASS.tasks_list(additional_param='detail') res = core.TORRENT_CLASS.tasks_info(input_id, additional_param='detail') logger.debug(f'result from syno {res}') if res['success']: diff --git a/core/utils/paths.py b/core/utils/paths.py index c1c0c1a5..46cb373e 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -105,7 +105,9 @@ def remove_read_only(filename): try: os.chmod(filename, stat.S_IWRITE) except Exception: - logger.warning(f'Cannot change permissions of {filename}', logger.WARNING) + logger.warning( + f'Cannot change permissions of {filename}', logger.WARNING, + ) def flatten_dir(destination, files): @@ -129,7 +131,9 @@ def flatten_dir(destination, files): def clean_directory(path, files): if not os.path.exists(path): - logger.info(f'Directory {path} has been processed and removed ...', 'CLEANDIR') + logger.info( + f'Directory {path} has been processed and removed ...', 'CLEANDIR', + ) return if core.FORCE_CLEAN and not core.FAILED: @@ -144,7 +148,9 @@ def clean_directory(path, files): ) return - logger.info(f'Directory {path} has been processed, removing ...', 'CLEANDIRS') + logger.info( + f'Directory {path} has been processed, removing ...', 'CLEANDIRS', + ) try: shutil.rmtree(path, onerror=onerror) except Exception: diff --git a/core/utils/processes.py b/core/utils/processes.py index 86b80ce4..92beb40e 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -20,7 +20,9 @@ if os.name == 'nt': class WindowsProcess: def __init__(self): self.mutex = None - self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' + self.mutexname = 'nzbtomedia_{pid}'.format( + pid=core.PID_FILE.replace('\\', '/'), + ) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' self.CreateMutex = CreateMutex self.CloseHandle = CloseHandle self.GetLastError = GetLastError diff --git a/core/version_check.py b/core/version_check.py index 5a513381..386f9cfc 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -62,7 +62,9 @@ class CheckVersion: force: if true the VERSION_NOTIFY setting will be ignored and a check will be forced """ if not core.VERSION_NOTIFY and not force: - logger.log('Version checking is disabled, not checking for the newest version') + logger.log( + 'Version checking is disabled, not checking for the newest version', + ) return False logger.log(f'Checking if {self.install_type} needs an update') @@ -118,8 +120,10 @@ class GitUpdateManager(UpdateManager): main_git = 'git' logger.log( - 'Checking if we can use git commands: {git} {cmd}'.format - (git=main_git, cmd=test_cmd), logger.DEBUG, + 'Checking if we can use git commands: {git} {cmd}'.format( + git=main_git, cmd=test_cmd, + ), + logger.DEBUG, ) output, err, exit_status = self._run_git(main_git, test_cmd) @@ -146,8 +150,10 @@ class GitUpdateManager(UpdateManager): for cur_git in alternative_git: logger.log( - 'Checking if we can use git commands: {git} {cmd}'.format - (git=cur_git, cmd=test_cmd), logger.DEBUG, + 'Checking if we can use git commands: {git} {cmd}'.format( + git=cur_git, cmd=test_cmd, + ), + logger.DEBUG, ) output, err, exit_status = self._run_git(cur_git, test_cmd) @@ -172,7 +178,9 @@ class GitUpdateManager(UpdateManager): err = None if not git_path: - logger.log('No git specified, can\'t use git commands', logger.DEBUG) + logger.log( + 'No git specified, can\'t use git commands', logger.DEBUG, + ) exit_status = 1 return output, err, exit_status @@ -180,12 +188,18 @@ class GitUpdateManager(UpdateManager): try: logger.log( - 'Executing {cmd} with your shell in {directory}'.format - (cmd=cmd, directory=core.APP_ROOT), logger.DEBUG, + 'Executing {cmd} with your shell in {directory}'.format( + cmd=cmd, directory=core.APP_ROOT, + ), + logger.DEBUG, ) p = subprocess.Popen( - cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - shell=True, cwd=core.APP_ROOT, + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + cwd=core.APP_ROOT, ) output, err = p.communicate() exit_status = p.returncode @@ -207,14 +221,16 @@ class GitUpdateManager(UpdateManager): exit_status = 0 elif core.LOG_GIT and exit_status in (1, 128): logger.log( - '{cmd} returned : {output}'.format - (cmd=cmd, output=output), logger.DEBUG, + f'{cmd} returned : {output}', + logger.DEBUG, ) else: if core.LOG_GIT: logger.log( - '{cmd} returned : {output}, treat as error for now'.format - (cmd=cmd, output=output), logger.DEBUG, + '{cmd} returned : {output}, treat as error for now'.format( + cmd=cmd, output=output, + ), + logger.DEBUG, ) exit_status = 1 @@ -228,12 +244,17 @@ class GitUpdateManager(UpdateManager): Returns: True for success or False for failure """ - output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable + output, err, exit_status = self._run_git( + self._git_path, 'rev-parse HEAD', + ) # @UnusedVariable if exit_status == 0 and output: cur_commit_hash = output.strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): - logger.log('Output doesn\'t look like a hash, not using it', logger.ERROR) + logger.log( + 'Output doesn\'t look like a hash, not using it', + logger.ERROR, + ) return False self._cur_commit_hash = cur_commit_hash if self._cur_commit_hash: @@ -244,7 +265,9 @@ class GitUpdateManager(UpdateManager): def _find_git_branch(self): core.NZBTOMEDIA_BRANCH = self.get_github_branch() - branch_info, err, exit_status = self._run_git(self._git_path, 'symbolic-ref -q HEAD') # @UnusedVariable + branch_info, err, exit_status = self._run_git( + self._git_path, 'symbolic-ref -q HEAD', + ) # @UnusedVariable if exit_status == 0 and branch_info: branch = branch_info.strip().replace('refs/heads/', '', 1) if branch: @@ -265,20 +288,30 @@ class GitUpdateManager(UpdateManager): self._num_commits_ahead = 0 # get all new info from github - output, err, exit_status = self._run_git(self._git_path, 'fetch origin') + output, err, exit_status = self._run_git( + self._git_path, 'fetch origin', + ) if not exit_status == 0: - logger.log('Unable to contact github, can\'t check for update', logger.ERROR) + logger.log( + 'Unable to contact github, can\'t check for update', + logger.ERROR, + ) return # get latest commit_hash from remote - output, err, exit_status = self._run_git(self._git_path, 'rev-parse --verify --quiet \'@{upstream}\'') + output, err, exit_status = self._run_git( + self._git_path, 'rev-parse --verify --quiet \'@{upstream}\'', + ) if exit_status == 0 and output: cur_commit_hash = output.strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): - logger.log('Output doesn\'t look like a hash, not using it', logger.DEBUG) + logger.log( + 'Output doesn\'t look like a hash, not using it', + logger.DEBUG, + ) return else: @@ -288,7 +321,9 @@ class GitUpdateManager(UpdateManager): return # get number of commits behind and ahead (option --count not supported git < 1.7.2) - output, err, exit_status = self._run_git(self._git_path, 'rev-list --left-right \'@{upstream}\'...HEAD') + output, err, exit_status = self._run_git( + self._git_path, 'rev-list --left-right \'@{upstream}\'...HEAD', + ) if exit_status == 0 and output: @@ -297,35 +332,47 @@ class GitUpdateManager(UpdateManager): self._num_commits_ahead = int(output.count('>')) except Exception: - logger.log('git didn\'t return numbers for behind and ahead, not using it', logger.DEBUG) + logger.log( + 'git didn\'t return numbers for behind and ahead, not using it', + logger.DEBUG, + ) return logger.log( 'cur_commit = {current} % (newest_commit)= {new}, ' - 'num_commits_behind = {x}, num_commits_ahead = {y}'.format - ( - current=self._cur_commit_hash, new=self._newest_commit_hash, - x=self._num_commits_behind, y=self._num_commits_ahead, - ), logger.DEBUG, + 'num_commits_behind = {x}, num_commits_ahead = {y}'.format( + current=self._cur_commit_hash, + new=self._newest_commit_hash, + x=self._num_commits_behind, + y=self._num_commits_ahead, + ), + logger.DEBUG, ) def set_newest_text(self): if self._num_commits_ahead: logger.log( - 'Local branch is ahead of {branch}. Automatic update not possible.'.format - (branch=self.branch), logger.ERROR, + 'Local branch is ahead of {branch}. Automatic update not possible.'.format( + branch=self.branch, + ), + logger.ERROR, ) elif self._num_commits_behind: logger.log( - 'There is a newer version available (you\'re {x} commit{s} behind)'.format - (x=self._num_commits_behind, s='s' if self._num_commits_behind > 1 else ''), logger.MESSAGE, + 'There is a newer version available (you\'re {x} commit{s} behind)'.format( + x=self._num_commits_behind, + s='s' if self._num_commits_behind > 1 else '', + ), + logger.MESSAGE, ) else: return def need_update(self): if not self._find_installed_version(): - logger.error('Unable to determine installed version via git, please check your logs!') + logger.error( + 'Unable to determine installed version via git, please check your logs!', + ) return False if not self._cur_commit_hash: @@ -334,7 +381,10 @@ class GitUpdateManager(UpdateManager): try: self._check_github_for_update() except Exception as error: - logger.log(f'Unable to contact github, can\'t check for update: {error!r}', logger.ERROR) + logger.log( + f'Unable to contact github, can\'t check for update: {error!r}', + logger.ERROR, + ) return False if self._num_commits_behind > 0: @@ -349,7 +399,9 @@ class GitUpdateManager(UpdateManager): Calls git pull origin in order to update Sick Beard. Returns a bool depending on the call's success. """ - output, err, exit_status = self._run_git(self._git_path, f'pull origin {self.branch}') # @UnusedVariable + output, err, exit_status = self._run_git( + self._git_path, f'pull origin {self.branch}', + ) # @UnusedVariable if exit_status == 0: return True @@ -379,7 +431,9 @@ class SourceUpdateManager(UpdateManager): with open(version_file) as fp: self._cur_commit_hash = fp.read().strip(' \n\r') except OSError as error: - logger.log(f'Unable to open \'version.txt\': {error}', logger.DEBUG) + logger.log( + f'Unable to open \'version.txt\': {error}', logger.DEBUG, + ) if not self._cur_commit_hash: self._cur_commit_hash = None @@ -393,7 +447,10 @@ class SourceUpdateManager(UpdateManager): try: self._check_github_for_update() except Exception as error: - logger.log(f'Unable to contact github, can\'t check for update: {error!r}', logger.ERROR) + logger.log( + f'Unable to contact github, can\'t check for update: {error!r}', + logger.ERROR, + ) return False if not self._cur_commit_hash or self._num_commits_behind > 0: @@ -414,14 +471,20 @@ class SourceUpdateManager(UpdateManager): self._num_commits_behind = 0 self._newest_commit_hash = None - gh = github.GitHub(self.github_repo_user, self.github_repo, self.branch) + gh = github.GitHub( + self.github_repo_user, self.github_repo, self.branch, + ) # try to get newest commit hash and commits behind directly by comparing branch and current commit if self._cur_commit_hash: - branch_compared = gh.compare(base=self.branch, head=self._cur_commit_hash) + branch_compared = gh.compare( + base=self.branch, head=self._cur_commit_hash, + ) if 'base_commit' in branch_compared: - self._newest_commit_hash = branch_compared['base_commit']['sha'] + self._newest_commit_hash = branch_compared['base_commit'][ + 'sha' + ] if 'behind_by' in branch_compared: self._num_commits_behind = int(branch_compared['behind_by']) @@ -442,8 +505,12 @@ class SourceUpdateManager(UpdateManager): self._num_commits_behind += 1 logger.log( - 'cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}'.format - (current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG, + 'cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}'.format( + current=self._cur_commit_hash, + new=self._newest_commit_hash, + x=self._num_commits_behind, + ), + logger.DEBUG, ) def set_newest_text(self): @@ -452,19 +519,29 @@ class SourceUpdateManager(UpdateManager): core.NEWEST_VERSION_STRING = None if not self._cur_commit_hash: - logger.log('Unknown current version number, don\'t know if we should update or not', logger.ERROR) + logger.log( + 'Unknown current version number, don\'t know if we should update or not', + logger.ERROR, + ) elif self._num_commits_behind > 0: logger.log( - 'There is a newer version available (you\'re {x} commit{s} behind)'.format - (x=self._num_commits_behind, s='s' if self._num_commits_behind > 1 else ''), logger.MESSAGE, + 'There is a newer version available (you\'re {x} commit{s} behind)'.format( + x=self._num_commits_behind, + s='s' if self._num_commits_behind > 1 else '', + ), + logger.MESSAGE, ) else: return def update(self): """Download and install latest source tarball from github.""" - tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format( - org=self.github_repo_user, repo=self.github_repo, branch=self.branch, + tar_download_url = ( + 'https://github.com/{org}/{repo}/tarball/{branch}'.format( + org=self.github_repo_user, + repo=self.github_repo, + branch=self.branch, + ) ) version_path = os.path.join(core.APP_ROOT, 'version.txt') @@ -473,28 +550,38 @@ class SourceUpdateManager(UpdateManager): sb_update_dir = os.path.join(core.APP_ROOT, 'sb-update') if os.path.isdir(sb_update_dir): - logger.log(f'Clearing out update folder {sb_update_dir} before extracting') + logger.log( + f'Clearing out update folder {sb_update_dir} before extracting', + ) shutil.rmtree(sb_update_dir) - logger.log(f'Creating update folder {sb_update_dir} before extracting') + logger.log( + f'Creating update folder {sb_update_dir} before extracting', + ) os.makedirs(sb_update_dir) # retrieve file logger.log(f'Downloading update from {tar_download_url!r}') - tar_download_path = os.path.join(sb_update_dir, 'nzbtomedia-update.tar') + tar_download_path = os.path.join( + sb_update_dir, 'nzbtomedia-update.tar', + ) urlretrieve(tar_download_url, tar_download_path) if not os.path.isfile(tar_download_path): logger.log( - 'Unable to retrieve new version from {url}, can\'t update'.format - (url=tar_download_url), logger.ERROR, + 'Unable to retrieve new version from {url}, can\'t update'.format( + url=tar_download_url, + ), + logger.ERROR, ) return False if not tarfile.is_tarfile(tar_download_path): logger.log( - 'Retrieved version from {url} is corrupt, can\'t update'.format - (url=tar_download_url), logger.ERROR, + 'Retrieved version from {url} is corrupt, can\'t update'.format( + url=tar_download_url, + ), + logger.ERROR, ) return False @@ -510,21 +597,28 @@ class SourceUpdateManager(UpdateManager): # find update dir name update_dir_contents = [ - x for x in os.listdir(sb_update_dir) if - os.path.isdir(os.path.join(sb_update_dir, x)) + x + for x in os.listdir(sb_update_dir) + if os.path.isdir(os.path.join(sb_update_dir, x)) ] if len(update_dir_contents) != 1: - logger.log(f'Invalid update data, update failed: {update_dir_contents}', logger.ERROR) + logger.log( + f'Invalid update data, update failed: {update_dir_contents}', + logger.ERROR, + ) return False content_dir = os.path.join(sb_update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder logger.log( - 'Moving files from {source} to {destination}'.format - (source=content_dir, destination=core.APP_ROOT), + 'Moving files from {source} to {destination}'.format( + source=content_dir, destination=core.APP_ROOT, + ), ) - for dirname, _, filenames in os.walk(content_dir): # @UnusedVariable - dirname = dirname[len(content_dir) + 1:] + for dirname, _, filenames in os.walk( + content_dir, + ): # @UnusedVariable + dirname = dirname[len(content_dir) + 1 :] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) new_path = os.path.join(core.APP_ROOT, dirname, curfile) @@ -539,10 +633,14 @@ class SourceUpdateManager(UpdateManager): os.renames(old_path, new_path) except Exception as error: logger.log( - 'Unable to update {path}: {msg}'.format - (path=new_path, msg=error), logger.DEBUG, + 'Unable to update {path}: {msg}'.format( + path=new_path, msg=error, + ), + logger.DEBUG, ) - os.remove(old_path) # Trash the updated file without moving in new path + os.remove( + old_path, + ) # Trash the updated file without moving in new path continue if os.path.isfile(new_path): @@ -555,15 +653,17 @@ class SourceUpdateManager(UpdateManager): ver_file.write(self._newest_commit_hash) except OSError as error: logger.log( - 'Unable to write version file, update not complete: {msg}'.format - (msg=error), logger.ERROR, + 'Unable to write version file, update not complete: {msg}'.format( + msg=error, + ), + logger.ERROR, ) return False except Exception as error: logger.log( - 'Error while trying to update: {msg}'.format - (msg=error), logger.ERROR, + f'Error while trying to update: {error}', + logger.ERROR, ) logger.log(f'Traceback: {traceback.format_exc()}', logger.DEBUG) return False diff --git a/requirements.txt b/requirements.txt index 6dd579cd..076933e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,12 @@ babelfish beets configobj +deluge-client@git+https://github.com/labrys/deluge.git@master guessit +jaraco-windows ; sys.platform == 'win32' linktastic python-qbittorrent +pywin32 ; sys.platform == 'win32' pyxdg rencode requests @@ -11,9 +14,6 @@ requests_oauthlib setuptools six subliminal != 2.1.0 -transmissionrpc -deluge-client@git+https://github.com/labrys/deluge.git@master syno@git+https://github.com/labrys/syno.git@master +transmissionrpc utorrent@git+https://github.com/labrys/utorrent.git@master -jaraco-windows ; sys.platform == 'win32' -pywin32 ; sys.platform == 'win32' diff --git a/tests/test_initialize.py b/tests/initialize_test.py old mode 100755 new mode 100644 similarity index 100% rename from tests/test_initialize.py rename to tests/initialize_test.py diff --git a/tests/test_transcoder.py b/tests/transcoder_test.py old mode 100755 new mode 100644 similarity index 100% rename from tests/test_transcoder.py rename to tests/transcoder_test.py