From 19c3e1fd8591e8876615578f67d058111955a3a7 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 15 Mar 2019 20:42:21 +1300 Subject: [PATCH 01/87] remove .encode which creates byte vs string comparison issues. Fixes #1582 --- TorrentToMedia.py | 48 ++++++++++++++++++------------------ core/utils/identification.py | 16 ++++++------ core/utils/naming.py | 8 +++--- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 74f37165..bd1166f4 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -60,14 +60,14 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass + #try: + # input_directory = input_directory.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format (input_directory, input_name, input_category)) @@ -125,10 +125,10 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - try: - output_destination = output_destination.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # output_destination = output_destination.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass if output_destination in input_directory: output_destination = input_directory @@ -170,10 +170,10 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - try: - target_file = target_file.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # target_file = target_file.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass if root == 1: if not found_file: logger.debug('Looking for {0} in: {1}'.format(input_name, inputFile)) @@ -350,15 +350,15 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - try: - dir_name = dir_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # dir_name = dir_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass input_name = os.path.basename(dir_name) - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) diff --git a/core/utils/identification.py b/core/utils/identification.py index c2dc0352..7a48ec87 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -90,14 +90,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - try: - input_name = input_name.encode(core.SYS_ENCODING) - except Exception: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except Exception: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except Exception: + # pass + #try: + # input_directory = input_directory.encode(core.SYS_ENCODING) + #except Exception: + # pass if input_directory is None: # =Nothing to process here. return input_directory, input_name, input_category, root diff --git a/core/utils/naming.py b/core/utils/naming.py index 352d51ba..8b3e6971 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -20,10 +20,10 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - try: - name = name.encode(core.SYS_ENCODING) - except Exception: - pass + #try: + # name = name.encode(core.SYS_ENCODING) + #except Exception: + # pass return name From a3db8fb4b657a733cba2abeb9609862ffadb6c9c Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Wed, 27 Mar 2019 10:09:47 +1300 Subject: [PATCH 02/87] Test 1 (#1586) * add transcoder tests --- azure-pipelines.yml | 3 +++ tests/test_transcoder.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100755 tests/test_transcoder.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3f857b99..a609703b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,6 +32,9 @@ jobs: - script: python -m pip install --upgrade pip displayName: 'Install dependencies' + - script: sudo apt-get install ffmpeg + displayName: 'Install ffmpeg' + - script: | pip install pytest pytest tests --doctest-modules --junitxml=junit/test-results.xml diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py new file mode 100755 index 00000000..929e7f23 --- /dev/null +++ b/tests/test_transcoder.py @@ -0,0 +1,14 @@ +#! /usr/bin/env python +from __future__ import print_function +import datetime +import os +import sys +import json +import time +import requests + +import core +from core import logger, transcoder + +def test_transcoder_check(): + assert transcoder.is_video_good(core.TEST_FILE, 0) == True From aee3b151c0f2a7f34c705d91d94af9c720a96e7b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 29 Mar 2019 09:50:43 +1300 Subject: [PATCH 03/87] Lazylib 1 (#1587) * add support for LazyLibrarian. Fixes #1223 --- TorrentToMedia.py | 5 +- autoProcessMedia.cfg.spec | 25 +++++++++ core/auto_process/books.py | 75 ++++++++++++++++++++++++++ core/configuration.py | 15 ++++++ nzbToCouchPotato.py | 2 +- nzbToGamez.py | 2 +- nzbToLazyLibrarian.py | 104 +++++++++++++++++++++++++++++++++++++ nzbToMedia.py | 40 +++++++++++++- 8 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 core/auto_process/books.py create mode 100755 nzbToLazyLibrarian.py diff --git a/TorrentToMedia.py b/TorrentToMedia.py index bd1166f4..06d36e1a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -13,7 +13,7 @@ import sys import core from core import logger, main_db -from core.auto_process import comics, games, movies, music, tv +from core.auto_process import comics, games, movies, music, tv, books from core.auto_process.common import ProcessResult from core.plugins.plex import plex_update from core.user_scripts import external_script @@ -256,6 +256,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp result = comics.process(section_name, output_destination, input_name, status, client_agent, input_category) elif section_name == 'Gamez': result = games.process(section_name, output_destination, input_name, status, client_agent, input_category) + elif section_name == 'LazyLibrarian': + result = books.process(section_name, output_destination, input_name, status, client_agent, input_category) + plex_update(input_category) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 767d937c..b54c640d 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -282,6 +282,31 @@ ##### Set to path where download client places completed downloads locally for this category watch_dir = +[LazyLibrarian] + #### autoProcessing for Games + #### games - category that gets called for post-processing with Gamez + [[books]] + enabled = 0 + apikey = + host = localhost + port = 5299 + ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### + ssl = 0 + web_root = + # Enable/Disable linking for Torrents + Torrent_NoLink = 0 + keep_archive = 1 + extract = 1 + # Set this to minimum required size to consider a media file valid (in MB) + minSize = 0 + # Enable/Disable deleting ignored files (samples and invalid media files) + delete_ignored = 0 + ##### Enable if LazyLibrarian is on a remote server for this category + remote_path = 0 + ##### Set to path where download client places completed downloads locally for this category + watch_dir = + + [Network] # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. MountPoints = /volume1/Public/,E:\|/volume2/share/,\\NAS\ diff --git a/core/auto_process/books.py b/core/auto_process/books.py new file mode 100644 index 00000000..b05032db --- /dev/null +++ b/core/auto_process/books.py @@ -0,0 +1,75 @@ +# coding=utf-8 + +import os +import shutil + +import requests + +import core +from core import logger +from core.auto_process.common import ProcessResult +from core.utils import convert_to_ascii, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + status = int(status) + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + protocol = 'https://' if ssl else 'http://' + + url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) + if not server_responding(url): + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + params = { + 'api_key': apikey, + 'cmd': 'forceProcess', + 'dir': dir_name, + } + + logger.debug('Opening URL: {0}'.format(url), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {1}'.format(section, section), + status_code=1, + ) + + result = r.json() + logger.postprocess('{0}'.format(result), section) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif result['success']: + logger.postprocess('SUCCESS: ForceProcess for {0} has been started in LazyLibrarian'.format(dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + else: + logger.error('FAILED: ForceProcess of {0} has Failed in LazyLibrarian'.format(dir_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, + ) diff --git a/core/configuration.py b/core/configuration.py index 31ea4852..216a1f05 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -383,6 +383,21 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new[section][os.environ[env_cat_key]][option] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + section = 'LazyLibrarian' + env_cat_key = 'NZBPO_LLCATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'REMOTE_PATH'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_LL{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + section = 'NzbDrone' env_cat_key = 'NZBPO_NDCATEGORY' env_keys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 12223106..be7fb4f3 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -4,7 +4,7 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to CouchPotato. # # This script sends the download to your automated media management servers. # diff --git a/nzbToGamez.py b/nzbToGamez.py index d1559e72..42d0a2f5 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -4,7 +4,7 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to Gamez. # # This script sends the download to your automated media management servers. # diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py new file mode 100755 index 00000000..96ab86c8 --- /dev/null +++ b/nzbToLazyLibrarian.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# coding=utf-8 +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Post-Process to LazyLibrarian. +# +# This script sends the download to your automated media management servers. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +# +### OPTIONS ### + +## General + +# Auto Update nzbToMedia (0, 1). +# +# Set to 1 if you want nzbToMedia to automatically check for and update to the latest version +#auto_update=0 + +# Safe Mode protection of DestDir (0, 1). +# +# Enable/Disable a safety check to ensure we don't process all downloads in the default_downloadDirectory by mistake. +#safe_mode=1 + +## LazyLibrarian + +# LazyLibrarian script category. +# +# category that gets called for post-processing with LazyLibrarian. +#llCategory=games + +# LazyLibrarian api key. +#llapikey= + +# LazyLibrarian host. +# +# The ipaddress for your LazyLibrarian server. e.g For the Same system use localhost or 127.0.0.1 +#llhost=localhost + +# LazyLibrarian port. +#llport=5299 + +# LazyLibrarian uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#llssl=0 + +# LazyLibrarian web_root +# +# set this if using a reverse proxy. +#llweb_root= + +# LazyLibrarian watch directory. +# +# set this to where your LazyLibrarian completed downloads are. +#llwatch_dir= + +## Posix + +# Niceness for external tasks Extractor and Transcoder. +# +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +#niceness=10 + +# ionice scheduling class (0, 1, 2, 3). +# +# Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. +#ionice_class=2 + +# ionice scheduling class data. +# +# Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. +#ionice_classdata=4 + +## WakeOnLan + +# use WOL (0, 1). +# +# set to 1 to send WOL broadcast to the mac and test the server (e.g. xbmc) on the host and port specified. +#wolwake=0 + +# WOL MAC +# +# enter the mac address of the system to be woken. +#wolmac=00:01:2e:2D:64:e1 + +# Set the Host and Port of a server to verify system has woken. +#wolhost=192.168.1.37 +#wolport=80 + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import sys + +import nzbToMedia + +section = 'LazyLibrarian' +result = nzbToMedia.main(sys.argv, section) +sys.exit(result) diff --git a/nzbToMedia.py b/nzbToMedia.py index d91f413f..c663916c 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -4,7 +4,8 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to CouchPotato, SickBeard, Sonarr, Mylar, Gamez, HeadPhones, +# LazyLibrarian, Radarr, Lidarr # # This script sends the download to your automated media management servers. # @@ -410,6 +411,39 @@ # Enable to replace local path with the path as per the mountPoints below. #gzremote_path=0 +## LazyLibrarian + +# LazyLibrarian script category. +# +# category that gets called for post-processing with LazyLibrarian. +#llCategory=games + +# LazyLibrarian api key. +#llapikey= + +# LazyLibrarian host. +# +# The ipaddress for your LazyLibrarian server. e.g For the Same system use localhost or 127.0.0.1 +#llhost=localhost + +# LazyLibrarian port. +#llport=5299 + +# LazyLibrarian uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#llssl=0 + +# LazyLibrarian web_root +# +# set this if using a reverse proxy. +#llweb_root= + +# LazyLibrarian watch directory. +# +# set this to where your LazyLibrarian completed downloads are. +#llwatch_dir= + ## Network # Network Mount Points (Needed for remote path above) @@ -635,7 +669,7 @@ import sys import core from core import logger, main_db -from core.auto_process import comics, games, movies, music, tv +from core.auto_process import comics, games, movies, music, tv, books from core.auto_process.common import ProcessResult from core.plugins.downloaders.nzb.utils import get_nzoid from core.plugins.plex import plex_update @@ -763,6 +797,8 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d result = comics.process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == 'Gamez': result = games.process(section_name, input_directory, input_name, status, client_agent, input_category) + elif section_name == 'LazyLibrarian': + result = books.process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == 'UserScript': result = external_script(input_directory, input_name, input_category, section[usercat]) else: From 1597763d3007dad4c013aefa93220f1febf1c082 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 29 Mar 2019 10:38:59 +1300 Subject: [PATCH 04/87] minor fix for LazyLibrarian api. --- autoProcessMedia.cfg.spec | 4 ++-- core/auto_process/books.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index b54c640d..30a399a0 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -283,8 +283,8 @@ watch_dir = [LazyLibrarian] - #### autoProcessing for Games - #### games - category that gets called for post-processing with Gamez + #### autoProcessing for LazyLibrarian + #### books - category that gets called for post-processing with LazyLibrarian [[books]] enabled = 0 apikey = diff --git a/core/auto_process/books.py b/core/auto_process/books.py index b05032db..c029d06f 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -36,12 +36,11 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_name, dir_name = convert_to_ascii(input_name, dir_name) params = { - 'api_key': apikey, + 'apikey': apikey, 'cmd': 'forceProcess', 'dir': dir_name, } - - logger.debug('Opening URL: {0}'.format(url), section) + logger.debug('Opening URL: {0} with params: {1}'.format(url, params), section) try: r = requests.get(url, params=params, verify=False, timeout=(30, 300)) @@ -52,8 +51,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', status_code=1, ) - result = r.json() - logger.postprocess('{0}'.format(result), section) + logger.postprocess('{0}'.format(r.text), section) if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) @@ -61,7 +59,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), status_code=1, ) - elif result['success']: + elif r.text == 'OK': logger.postprocess('SUCCESS: ForceProcess for {0} has been started in LazyLibrarian'.format(dir_name), section) return ProcessResult( message='{0}: Successfully post-processed {1}'.format(section, input_name), From 809e6420393464d65945931d5083d7de01c3c3c8 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 30 Mar 2019 08:47:20 +1300 Subject: [PATCH 05/87] fix LL default branch. --- nzbToLazyLibrarian.py | 2 +- nzbToMedia.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index 96ab86c8..0a26013b 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -31,7 +31,7 @@ # LazyLibrarian script category. # # category that gets called for post-processing with LazyLibrarian. -#llCategory=games +#llCategory=books # LazyLibrarian api key. #llapikey= diff --git a/nzbToMedia.py b/nzbToMedia.py index c663916c..8d969d6a 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -416,7 +416,7 @@ # LazyLibrarian script category. # # category that gets called for post-processing with LazyLibrarian. -#llCategory=games +#llCategory=books # LazyLibrarian api key. #llapikey= From f20e1e4f0d3366c35ff075735c5c802cef8c2b51 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 11:45:04 -0400 Subject: [PATCH 06/87] Add pywin32 to setup.py install_requires on Windows --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 803f430b..b2f78439 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,9 @@ setup( author_email='fock_wulf@hotmail.com', url='https://github.com/clinton-hall/nzbToMedia', packages=['core'], + install_requires=[ + 'pywin32;platform_system=="Windows"', + ], classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers 'Development Status :: 5 - Production/Stable', From a531f4480e33dc720568da6ac38487e8b62333c6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:13:11 -0400 Subject: [PATCH 07/87] Add source install cleanup test --- azure-pipelines.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a609703b..485f214f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,6 +40,13 @@ jobs: pytest tests --doctest-modules --junitxml=junit/test-results.xml displayName: 'pytest' + - script: + rm -rf .git + python cleanup.py + python TorrentToMedia.py + python nzbToMedia.py + displayName: 'Test source install cleanup' + - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-results.xml' From 02813a6eaf865157433dadfccaf9ee651c64e6d7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:39:12 -0400 Subject: [PATCH 08/87] Add source install cleanup test --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 485f214f..96321a5b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,7 @@ jobs: pytest tests --doctest-modules --junitxml=junit/test-results.xml displayName: 'pytest' - - script: + - script: | rm -rf .git python cleanup.py python TorrentToMedia.py From 16b7c1149511528972751d1b986558ead0b188be Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:45:07 -0400 Subject: [PATCH 09/87] Force cleanup errors for confirming CI test --- cleanup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cleanup.py b/cleanup.py index d313f2b4..43dc5775 100644 --- a/cleanup.py +++ b/cleanup.py @@ -15,10 +15,6 @@ FOLDER_STRUCTURE = { 'win', ], 'core': [ - 'auto_process', - 'extractor', - 'plugins', - 'utils', ], } From f5fdc145774b936b592d33ad523a43ec2a8eb204 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:49:32 -0400 Subject: [PATCH 10/87] Revert "Force cleanup errors for confirming CI test" This reverts commit 16b7c1149511528972751d1b986558ead0b188be. --- cleanup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cleanup.py b/cleanup.py index 43dc5775..d313f2b4 100644 --- a/cleanup.py +++ b/cleanup.py @@ -15,6 +15,10 @@ FOLDER_STRUCTURE = { 'win', ], 'core': [ + 'auto_process', + 'extractor', + 'plugins', + 'utils', ], } From 825b48a6c121a12f398dea3ea23410155383e737 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 4 Apr 2019 11:34:25 +1300 Subject: [PATCH 11/87] add h265 to MKV profile allow. Fixes #1592 --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 3087ff55..56f08bce 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -746,7 +746,7 @@ def configure_transcoder(): }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, From 822603d0211d6b2ac0c4daf9a20a8882b21128a1 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:10:38 -0400 Subject: [PATCH 12/87] Add tox.ini --- tox.ini | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..a18abb54 --- /dev/null +++ b/tox.ini @@ -0,0 +1,54 @@ +; a generative tox configuration, see: https://tox.readthedocs.io/en/latest/config.html#generative-envlist + +[tox] +envlist = + clean, + check, + {py27, py35, py36, py37}, + report + +[testenv] +basepython = + py27: {env:TOXPYTHON:python2.7} + py35: {env:TOXPYTHON:python3.5} + py36: {env:TOXPYTHON:python3.6} + py37: {env:TOXPYTHON:python3.7} + {clean,check,report,codecov}: {env:TOXPYTHON:python3} +setenv = + PYTHONPATH={toxinidir}/tests + PYTHONUNBUFFERED=yes +passenv = + * +usedevelop = false +skip_install = true +deps = + pytest + pytest-travis-fold + pytest-cov + pywin32 ; sys.platform == 'win32' +commands = + {posargs:pytest --cov --cov-report=term-missing tests} + +[coverage:run] +omit = + libs/* + +[testenv:codecov] +deps = + codecov +skip_install = true +commands = + coverage xml --ignore-errors + codecov [] + +[testenv:report] +deps = coverage +skip_install = true +commands = + coverage report + coverage html + +[testenv:clean] +commands = coverage erase +skip_install = true +deps = coverage From a8d1cc4fe9f02005443d73ef0097c635713d1181 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 17:11:16 -0400 Subject: [PATCH 13/87] Add flake8 quality checks to tox.ini --- tox.ini | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tox.ini b/tox.ini index a18abb54..2684e857 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,30 @@ deps = commands = {posargs:pytest --cov --cov-report=term-missing tests} +[flake8] +max-line-length = 79 +verbose = 2 +statistics = True +ignore = +; -- flake8 -- +; E501 line too long + E501 + +per-file-ignores = +; F401 imported but unused +; E402 module level import not at top of file + core/__init__.py: E402, F401 + core/utils/__init__.py: F401 + core/plugins/downloaders/configuration.py: F401 + core/plugins/downloaders/utils.py: F401 + +[testenv:check] +deps = + flake8 +skip_install = true +commands = + flake8 core tests setup.py + [coverage:run] omit = libs/* From 90090d7e02ed8b24306e219b38836f3456a1276a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:36:23 -0400 Subject: [PATCH 14/87] Fix flake8 E117 over-indented --- core/auto_process/movies.py | 32 ++++++++++++++++---------------- core/version_check.py | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index b7fe3a66..ff1222bf 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -373,22 +373,22 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', except Exception: pass elif scan_id: - url = '{0}/{1}'.format(base_url, scan_id) - command_status = command_complete(url, params, headers, section) - if command_status: - logger.debug('The Scan command return status: {0}'.format(command_status), section) - if command_status in ['completed']: - logger.debug('The Scan command has completed successfully. Renaming was successful.', section) - return ProcessResult( - message='{0}: Successfully post-processed {1}'.format(section, input_name), - status_code=0, - ) - elif command_status in ['failed']: - logger.debug('The Scan command has failed. Renaming was not successful.', section) - # return ProcessResult( - # message='{0}: Failed to post-process {1}'.format(section, input_name), - # status_code=1, - # ) + url = '{0}/{1}'.format(base_url, scan_id) + command_status = command_complete(url, params, headers, section) + if command_status: + logger.debug('The Scan command return status: {0}'.format(command_status), section) + if command_status in ['completed']: + logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status in ['failed']: + logger.debug('The Scan command has failed. Renaming was not successful.', section) + # return ProcessResult( + # message='{0}: Failed to post-process {1}'.format(section, input_name), + # status_code=1, + # ) if not os.path.isdir(dir_name): logger.postprocess('SUCCESS: Input Directory [{0}] has been processed and removed'.format( diff --git a/core/version_check.py b/core/version_check.py index 53945e1f..d865b9f3 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -199,8 +199,8 @@ class GitUpdateManager(UpdateManager): logger.log(u'{cmd} : returned successful'.format(cmd=cmd), logger.DEBUG) exit_status = 0 elif core.LOG_GIT and exit_status in (1, 128): - logger.log(u'{cmd} returned : {output}'.format - (cmd=cmd, output=output), logger.DEBUG) + logger.log(u'{cmd} returned : {output}'.format + (cmd=cmd, output=output), logger.DEBUG) else: if core.LOG_GIT: logger.log(u'{cmd} returned : {output}, treat as error for now'.format From 87e813f06280c5b46a15f367f5e148a2181c0227 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:04:50 -0400 Subject: [PATCH 15/87] Fix flake8 E126 continuation line over-indented for hanging indent --- core/forks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/forks.py b/core/forks.py index be81a965..897a1fd9 100644 --- a/core/forks.py +++ b/core/forks.py @@ -42,7 +42,8 @@ def auto_fork(section, input_category): logger.info('Attempting to verify {category} fork'.format (category=input_category)) url = '{protocol}{host}:{port}{root}/api/rootfolder'.format( - protocol=protocol, host=host, port=port, root=web_root) + protocol=protocol, host=host, port=port, root=web_root + ) headers = {'X-Api-Key': apikey} try: r = requests.get(url, headers=headers, stream=True, verify=False) @@ -65,10 +66,12 @@ def auto_fork(section, input_category): if apikey: url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess'.format( - protocol=protocol, host=host, port=port, root=web_root, apikey=apikey) + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey + ) else: url = '{protocol}{host}:{port}{root}/home/postprocess/'.format( - protocol=protocol, host=host, port=port, root=web_root) + protocol=protocol, host=host, port=port, root=web_root + ) # attempting to auto-detect fork try: From 07ad515b5066c6bbc53f639d4679647b01c02790 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:18:31 -0400 Subject: [PATCH 16/87] Fix flake8 E226 missing whitespace around arithmetic operator --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 56f08bce..0a224f94 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -870,7 +870,7 @@ def configure_utility_locations(): else: if SYS_PATH: - os.environ['PATH'] += ':'+SYS_PATH + os.environ['PATH'] += ':' + SYS_PATH try: SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: From 8a22f20a8b189e149a858547a97a807c1e1bf828 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:10:20 -0400 Subject: [PATCH 17/87] Fix flake8 E241 multiple spaces after ':' --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 0a224f94..40b9d54f 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -108,7 +108,7 @@ FORKS = { FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, - FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, + FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} } From 5f633b931aaa0508dae655872446429ce0531ac3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:27:39 -0400 Subject: [PATCH 18/87] Fix flake8 E261 at least two spaces before inline comment --- core/auto_process/movies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index ff1222bf..76d49b78 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -256,7 +256,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', return ProcessResult( message='{0}: Sending failed download back to {0}'.format(section), status_code=1, # Return as failed to flag this in the downloader. - ) # Return failed flag, but log the event as successful. + ) # Return failed flag, but log the event as successful. if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) From a571fc31224ef0a86fa54c110c32bafc3517d96f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:42:46 -0400 Subject: [PATCH 19/87] Fix flake8 E265 block comment should start with '# ' --- core/utils/identification.py | 8 ++++---- core/utils/naming.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 7a48ec87..5aada2c1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -90,13 +90,13 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass - #try: + # try: # input_directory = input_directory.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass if input_directory is None: # =Nothing to process here. diff --git a/core/utils/naming.py b/core/utils/naming.py index 8b3e6971..ab8d9174 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -20,9 +20,9 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - #try: + # try: # name = name.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass return name From 032f7456f9656072a510eacbe86f41014fba3984 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:52:41 -0400 Subject: [PATCH 20/87] Fix flake8 E302 expected 2 blank lines, found 1 --- tests/test_initialize.py | 8 ++++++++ tests/test_transcoder.py | 1 + 2 files changed, 9 insertions(+) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 612261d3..ecc11331 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -8,28 +8,35 @@ def test_eol(): import eol eol.check() + def test_cleanup(): import cleanup cleanup.clean(cleanup.FOLDER_STRUCTURE) + def test_import_core(): import core from core import logger, main_db + def test_import_core_auto_process(): from core.auto_process import comics, games, movies, music, tv from core.auto_process.common import ProcessResult + def test_import_core_plugins(): from core.plugins.downloaders.nzb.utils import get_nzoid from core.plugins.plex import plex_update + def test_import_core_user_scripts(): from core.user_scripts import external_script + def test_import_six(): from six import text_type + def test_import_core_utils(): from core.utils import ( char_replace, clean_dir, convert_to_ascii, @@ -44,5 +51,6 @@ def test_initial(): core.initialize() del core.MYAPP + def test_core_parameters(): assert core.CHECK_MEDIA == 1 diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 929e7f23..15b8f4aa 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -10,5 +10,6 @@ import requests import core from core import logger, transcoder + def test_transcoder_check(): assert transcoder.is_video_good(core.TEST_FILE, 0) == True From 8e6e2d16470c698929f1dfad70fc557989e8a389 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:28:36 -0400 Subject: [PATCH 21/87] Fix flake8 E305 expected 2 blank lines after class or function definition, found 1 --- tests/test_initialize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index ecc11331..90759dbd 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -44,6 +44,7 @@ def test_import_core_utils(): update_download_info_status, replace_links, ) + import core from core import logger, main_db From d20879843064f44acac0451b2fde991e8e8d0d55 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:50:11 -0400 Subject: [PATCH 22/87] Fix flake8 E402 module level import not at top of file --- tests/test_initialize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 90759dbd..ea25826e 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -4,6 +4,10 @@ import datetime import os import sys +import core +from core import logger, main_db + + def test_eol(): import eol eol.check() @@ -45,9 +49,6 @@ def test_import_core_utils(): ) -import core -from core import logger, main_db - def test_initial(): core.initialize() del core.MYAPP From faa378f7875f7f1cd20d99f6edd72a90efb888cd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:30:35 -0400 Subject: [PATCH 23/87] Fix flake8 E712 comparison to True should be 'if cond is True:' or 'if cond:' --- tests/test_transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 15b8f4aa..c027aee8 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -12,4 +12,4 @@ from core import logger, transcoder def test_transcoder_check(): - assert transcoder.is_video_good(core.TEST_FILE, 0) == True + assert transcoder.is_video_good(core.TEST_FILE, 0) is True From 644a11118c7d05e606234a1ca7ec6169b81c46f5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:07:24 -0400 Subject: [PATCH 24/87] Fix flake8 F401 imported but unused --- core/auto_process/books.py | 3 --- core/utils/identification.py | 1 - core/utils/naming.py | 1 - tests/test_initialize.py | 23 ++++++----------------- tests/test_transcoder.py | 8 +------- 5 files changed, 7 insertions(+), 29 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index c029d06f..7d843f5e 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -1,8 +1,5 @@ # coding=utf-8 -import os -import shutil - import requests import core diff --git a/core/utils/identification.py b/core/utils/identification.py index 5aada2c1..9029f7e4 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -5,7 +5,6 @@ import guessit import requests from six import text_type -import core from core import logger from core.utils.naming import sanitize_name diff --git a/core/utils/naming.py b/core/utils/naming.py index ab8d9174..b54e3258 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -1,5 +1,4 @@ import re -import core def sanitize_name(name): diff --git a/tests/test_initialize.py b/tests/test_initialize.py index ea25826e..dffcfe76 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -1,11 +1,7 @@ #! /usr/bin/env python from __future__ import print_function -import datetime -import os -import sys import core -from core import logger, main_db def test_eol(): @@ -19,34 +15,27 @@ def test_cleanup(): def test_import_core(): - import core - from core import logger, main_db + pass def test_import_core_auto_process(): - from core.auto_process import comics, games, movies, music, tv - from core.auto_process.common import ProcessResult + pass def test_import_core_plugins(): - from core.plugins.downloaders.nzb.utils import get_nzoid - from core.plugins.plex import plex_update + pass def test_import_core_user_scripts(): - from core.user_scripts import external_script + pass def test_import_six(): - from six import text_type + pass def test_import_core_utils(): - from core.utils import ( - char_replace, clean_dir, convert_to_ascii, - extract_files, get_dirs, get_download_info, - update_download_info_status, replace_links, - ) + pass def test_initial(): diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index c027aee8..148156b0 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -1,14 +1,8 @@ #! /usr/bin/env python from __future__ import print_function -import datetime -import os -import sys -import json -import time -import requests import core -from core import logger, transcoder +from core import transcoder def test_transcoder_check(): From 0350521b874c7d38340a8f19dbb1b19e53cbcaf6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:37:23 -0400 Subject: [PATCH 25/87] Fix flake8 W291 trailing whitespace --- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index f2c53c45..42473ab8 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -171,7 +171,7 @@ def par2(dirname): cmd = '' for item in command: cmd = '{cmd} {item}'.format(cmd=cmd, item=item) - logger.debug('calling command:{0}'.format(cmd), 'PAR2') + logger.debug('calling command:{0}'.format(cmd), 'PAR2') try: proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) proc.communicate() diff --git a/core/transcoder.py b/core/transcoder.py index c53a1211..edb98761 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -828,7 +828,7 @@ def transcode_directory(dir_name): if not os.listdir(text_type(new_dir)): # this is an empty directory and we didn't transcode into it. os.rmdir(new_dir) new_dir = dir_name - if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB + if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB new_dir = dir_name bitbucket.close() return final_result, new_dir diff --git a/setup.py b/setup.py index b2f78439..86d82b89 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( version='12.0.10', license='GPLv3', description='Efficient on demand post processing', - long_description=""" + long_description=""" nzbToMedia ========== From 697df555ec36367f718eec568ed9fd8976239f96 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:44:44 -0400 Subject: [PATCH 26/87] Fix flake8 W293 blank line contains whitespace --- core/auto_process/books.py | 2 +- core/logger.py | 2 +- setup.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index 7d843f5e..8ba74a43 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -49,7 +49,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', ) logger.postprocess('{0}'.format(r.text), section) - + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) return ProcessResult( diff --git a/core/logger.py b/core/logger.py index 3305a96e..d73b501b 100644 --- a/core/logger.py +++ b/core/logger.py @@ -132,7 +132,7 @@ class NTMRotatingLogHandler(object): """ Returns a numbered log file name depending on i. If i==0 it just uses logName, if not it appends it to the extension (blah.log.3 for i == 3) - + i: Log number to ues """ diff --git a/setup.py b/setup.py index 86d82b89..fed3a4be 100644 --- a/setup.py +++ b/setup.py @@ -24,18 +24,18 @@ setup( long_description=""" nzbToMedia ========== - + Efficient on demand post processing ----------------------------------- - + A PVR app needs to know when a download is ready for post-processing. There are two methods: - + 1. On-demand post-processing script (e.g. sabToSickBeard.py or nzbToMedia.py): A script in the downloader runs once at the end of the download job and notifies the PVR app that the download is complete. - + 2. Continuous folder scanning: The PVR app frequently polls download folder(s) for completed downloads. - + On-demand is superior, for several reasons: - + 1. The PVR app is notified only once, exactly when the download is ready for post-processing 2. The PVR app does not have to wait for the next poll interval before it starts processing 3. Continuous polling is not as efficient and is more stressful on low performance hardware @@ -46,7 +46,7 @@ setup( 8. On-demand scripts can be tweaked to allow for delays with slow hardware nzbToMedia is an on-demand post-processing script and was created out of a demand for more efficient post-processing on low-performance hardware. Many features have been added so higher performance hardware can benefit too. - + Many issues that users have with folder scanning can be fixed by switching to on-demand. A whole class of support issues can be eliminated by using nzbToMedia. """, author='Clinton Hall', From 0bcbabd6816719ac9dd1a9f3807f6273623d133e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:05:03 -0400 Subject: [PATCH 27/87] Add flake8-commas to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 2684e857..74cf8e25 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,7 @@ per-file-ignores = [testenv:check] deps = flake8 + flake8-commas skip_install = true commands = flake8 core tests setup.py From 14b2aa6bf46dc2b14db96ed6ac59e274196b8d6e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:03:25 -0400 Subject: [PATCH 28/87] Fix flake8-commas C812 missing trailing comma --- core/__init__.py | 36 +++++++++---------- core/auto_process/comics.py | 2 +- core/auto_process/common.py | 2 +- core/auto_process/games.py | 2 +- core/auto_process/music.py | 6 ++-- core/databases.py | 4 +-- core/forks.py | 6 ++-- core/logger.py | 4 +-- core/main_db.py | 8 ++--- .../downloaders/torrent/qbittorrent.py | 2 +- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- setup.py | 2 +- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 40b9d54f..6b4e40e9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -110,7 +110,7 @@ FORKS = { FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, - FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} + FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} @@ -653,7 +653,7 @@ def configure_transcoder(): codec_alias = { 'libx264': ['libx264', 'h264', 'h.264', 'AVC', 'MPEG-4'], 'libmp3lame': ['libmp3lame', 'mp3'], - 'libfaac': ['libfaac', 'aac', 'faac'] + 'libfaac': ['libfaac', 'aac', 'faac'], } transcode_defaults = { 'iPad': { @@ -662,7 +662,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPad-1080p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -670,7 +670,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPad-720p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -678,7 +678,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Apple-TV': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -686,7 +686,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPod': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -694,7 +694,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPhone': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -702,7 +702,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'PS3': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -710,7 +710,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'xbox': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -718,7 +718,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-480p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -726,7 +726,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-720p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -734,7 +734,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-1080p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -742,7 +742,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 160000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -750,7 +750,7 @@ def configure_transcoder(): 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'mp4-scene-release': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': 19, 'VLEVEL': '3.1', @@ -758,7 +758,7 @@ def configure_transcoder(): 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'MKV-SD': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': '1200k', 'VCRF': None, 'VLEVEL': None, @@ -766,8 +766,8 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' - } + 'SCODEC': 'mov_text', + }, } if DEFAULTS and DEFAULTS in transcode_defaults: VEXTENSION = transcode_defaults[DEFAULTS]['VEXTENSION'] @@ -957,7 +957,7 @@ def check_python(): major=sys.version_info[0], minor=sys.version_info[1], x=days_left, - ) + ), ) if days_left <= grace_period: logger.warning('Please upgrade to a more recent Python version.') diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 7049704b..1669d8d8 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -60,7 +60,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', logger.error('Unable to open URL', section) return ProcessResult( message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), - status_code=1 + status_code=1, ) if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) diff --git a/core/auto_process/common.py b/core/auto_process/common.py index 8da83485..8c3d9ef2 100644 --- a/core/auto_process/common.py +++ b/core/auto_process/common.py @@ -17,7 +17,7 @@ class ProcessResult(object): def __str__(self): return 'Processing {0}: {1}'.format( 'succeeded' if bool(self) else 'failed', - self.message + self.message, ) def __repr__(self): diff --git a/core/auto_process/games.py b/core/auto_process/games.py index c412a690..410b82a3 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -46,7 +46,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', 'api_key': apikey, 'mode': 'UPDATEREQUESTEDSTATUS', 'db_id': gamez_id, - 'status': download_status + 'status': download_status, } logger.debug('Opening URL: {0}'.format(url), section) diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 4f10c64e..2f614ee1 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -73,7 +73,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': remote_dir(dir_name) if remote_path else dir_name + 'dir': remote_dir(dir_name) if remote_path else dir_name, } res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) @@ -83,7 +83,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0] + 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0], } res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) @@ -187,7 +187,7 @@ def get_status(url, apikey, dir_name): params = { 'apikey': apikey, - 'cmd': 'getHistory' + 'cmd': 'getHistory', } logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params)) diff --git a/core/databases.py b/core/databases.py index 07a78a4d..0803b9fc 100644 --- a/core/databases.py +++ b/core/databases.py @@ -33,7 +33,7 @@ class InitialSchema(main_db.SchemaUpgrade): queries = [ 'CREATE TABLE db_version (db_version INTEGER);', 'CREATE TABLE downloads (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));', - 'INSERT INTO db_version (db_version) VALUES (2);' + 'INSERT INTO db_version (db_version) VALUES (2);', ] for query in queries: self.connection.action(query) @@ -59,7 +59,7 @@ class InitialSchema(main_db.SchemaUpgrade): 'INSERT INTO downloads2 SELECT * FROM downloads;', 'DROP TABLE IF EXISTS downloads;', 'ALTER TABLE downloads2 RENAME TO downloads;', - 'INSERT INTO db_version (db_version) VALUES (2);' + 'INSERT INTO db_version (db_version) VALUES (2);', ] for query in queries: self.connection.action(query) diff --git a/core/forks.py b/core/forks.py index 897a1fd9..400d7e50 100644 --- a/core/forks.py +++ b/core/forks.py @@ -42,7 +42,7 @@ def auto_fork(section, input_category): logger.info('Attempting to verify {category} fork'.format (category=input_category)) url = '{protocol}{host}:{port}{root}/api/rootfolder'.format( - protocol=protocol, host=host, port=port, root=web_root + protocol=protocol, host=host, port=port, root=web_root, ) headers = {'X-Api-Key': apikey} try: @@ -66,11 +66,11 @@ def auto_fork(section, input_category): if apikey: url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess'.format( - protocol=protocol, host=host, port=port, root=web_root, apikey=apikey + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey, ) else: url = '{protocol}{host}:{port}{root}/home/postprocess/'.format( - protocol=protocol, host=host, port=port, root=web_root + protocol=protocol, host=host, port=port, root=web_root, ) # attempting to auto-detect fork diff --git a/core/logger.py b/core/logger.py index d73b501b..fcc306e2 100644 --- a/core/logger.py +++ b/core/logger.py @@ -85,7 +85,7 @@ class NTMRotatingLogHandler(object): console.setFormatter(DispatchingFormatter( {'nzbtomedia': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), 'postprocess': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), - 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S') + 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), }, logging.Formatter('%(message)s'), )) @@ -122,7 +122,7 @@ class NTMRotatingLogHandler(object): file_handler.setFormatter(DispatchingFormatter( {'nzbtomedia': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), 'postprocess': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), - 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S') + 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), }, logging.Formatter('%(message)s'), )) diff --git a/core/main_db.py b/core/main_db.py index 12c21c7e..241b3adf 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -183,9 +183,9 @@ class DBConnection(object): 'WHERE {conditions}'.format( table=table_name, params=', '.join(gen_params(value_dict)), - conditions=' AND '.join(gen_params(key_dict)) + conditions=' AND '.join(gen_params(key_dict)), ), - items + items, ) if self.connection.total_changes == changes_before: @@ -194,9 +194,9 @@ class DBConnection(object): 'VALUES ({values})'.format( table=table_name, columns=', '.join(map(text_type, value_dict.keys())), - values=', '.join(['?'] * len(value_dict.values())) + values=', '.join(['?'] * len(value_dict.values())), ), - list(value_dict.values()) + list(value_dict.values()), ) def table_info(self, table_name): diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py index ff92512c..61e84a7d 100644 --- a/core/plugins/downloaders/torrent/qbittorrent.py +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -14,7 +14,7 @@ def configure_client(): password = core.QBITTORRENT_PASSWORD logger.debug( - 'Connecting to {0}: http://{1}:{2}'.format(agent, host, port) + 'Connecting to {0}: http://{1}:{2}'.format(agent, host, port), ) client = qBittorrentClient('http://{0}:{1}/'.format(host, port)) try: diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 42473ab8..d62df0ba 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -25,7 +25,7 @@ media_list = [r'\.s\d{2}e\d{2}\.', r'\.1080[pi]\.', r'\.720p\.', r'\.576[pi]', r r'\.internal\.', r'\bac3\b', r'\.ntsc\.', r'\.pal\.', r'\.secam\.', r'\bdivx\b', r'\bxvid\b'] media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) garbage_name = re.compile(r'^[a-zA-Z0-9]*$') -char_replace = [[r'(\w)1\.(\w)', r'\1i\2'] +char_replace = [[r'(\w)1\.(\w)', r'\1i\2'], ] diff --git a/core/transcoder.py b/core/transcoder.py index edb98761..6dc9ce32 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -672,7 +672,7 @@ def rip_iso(item, new_dir, bitbucket): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, ) new_files.append({item: {'name': name, 'files': concat}}) if core.CONCAT: diff --git a/setup.py b/setup.py index fed3a4be..36e97020 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ from setuptools import setup def read(*names, **kwargs): with io.open( os.path.join(os.path.dirname(__file__), *names), - encoding=kwargs.get('encoding', 'utf8') + encoding=kwargs.get('encoding', 'utf8'), ) as fh: return fh.read() From c5244df5104c9c4daa9bb4418ab94319a8e177cb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:04:46 -0400 Subject: [PATCH 29/87] Fix flake8-commas C819 trailing comma prohibited --- core/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/logger.py b/core/logger.py index fcc306e2..f7fbf9c4 100644 --- a/core/logger.py +++ b/core/logger.py @@ -87,7 +87,7 @@ class NTMRotatingLogHandler(object): 'postprocess': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), }, - logging.Formatter('%(message)s'), )) + logging.Formatter('%(message)s'))) # add the handler to the root logger logging.getLogger('nzbtomedia').addHandler(console) @@ -124,7 +124,7 @@ class NTMRotatingLogHandler(object): 'postprocess': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), }, - logging.Formatter('%(message)s'), )) + logging.Formatter('%(message)s'))) return file_handler From 2995c7f39171060b43069c11c01e0c9328b2749e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 19:04:11 -0400 Subject: [PATCH 30/87] Add flake8-quotes to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 74cf8e25..77e5ae63 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,7 @@ per-file-ignores = deps = flake8 flake8-commas + flake8-quotes skip_install = true commands = flake8 core tests setup.py From 94c42dbd8a89662b3891b368b1d269c004657882 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:21:26 -0400 Subject: [PATCH 31/87] Fix flake8-quotes Q000 Remove bad quotes --- core/__init__.py | 2 +- core/auto_process/tv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 6b4e40e9..85aeaf27 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -110,7 +110,7 @@ FORKS = { FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, - FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None}, + FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index c4a68f59..3f8f27c5 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -266,7 +266,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu if apikey: url = '{0}{1}:{2}{3}/api/{4}/?cmd=postprocess'.format(protocol, host, port, web_root, apikey) elif fork == 'Stheno': - url = "{0}{1}:{2}{3}/home/postprocess/process_episode".format(protocol, host, port, web_root) + url = '{0}{1}:{2}{3}/home/postprocess/process_episode'.format(protocol, host, port, web_root) else: url = '{0}{1}:{2}{3}/home/postprocess/processEpisode'.format(protocol, host, port, web_root) elif section == 'NzbDrone': From f2964296c5b83c225a6e148bd940a9e02eb3c82a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 19:19:11 -0400 Subject: [PATCH 32/87] Add flake8-comprehensions to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 77e5ae63..56d21272 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,7 @@ per-file-ignores = deps = flake8 flake8-commas + flake8-comprehensions flake8-quotes skip_install = true commands = From b9c7eec834e6a90402b68accf99137c9444ceea2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:34:46 -0400 Subject: [PATCH 33/87] Fix flake8-comprehensions C403 Unnecessary list comprehension --- core/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/transcoder.py b/core/transcoder.py index 6dc9ce32..d4b7e2f2 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -713,7 +713,7 @@ def combine_vts(vts_path): def combine_cd(combine): new_files = [] - for item in set([re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine]): + for item in {re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine}: concat = '' for n in range(99): files = [file for file in combine if From 169fcaae4a565765bcb70080d376ff4bef705817 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:35:23 -0400 Subject: [PATCH 34/87] Fix flake8-comprehensions C407 Unnecessary list comprehension --- core/utils/paths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/paths.py b/core/utils/paths.py index dba119f5..fe049ae7 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -67,10 +67,10 @@ def remote_dir(path): def get_dir_size(input_path): prepend = partial(os.path.join, input_path) - return sum([ + return sum( (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) for f in map(prepend, os.listdir(text_type(input_path))) - ]) + ) def remove_empty_folders(path, remove_root=True): From a8043d025948fd54d13dda0fb7e42e9cfa996e87 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 22:59:51 -0400 Subject: [PATCH 35/87] Add flake8-docstrings to tox.ini --- tox.ini | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tox.ini b/tox.ini index 56d21272..2111ddc6 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,22 @@ ignore = ; E501 line too long E501 +; -- flake8-docstrings -- +; D100 Missing docstring in public module +; D101 Missing docstring in public class +; D102 Missing docstring in public method +; D103 Missing docstring in public function +; D104 Missing docstring in public package +; D105 Missing docstring in magic method +; D107 Missing docstring in __init__ +; D200 One-line docstring should fit on one line with quotes +; D202 No blank lines allowed after function docstring +; D205 1 blank line required between summary line and description +; D400 First line should end with a period +; D401 First line should be in imperative mood +; D402 First line should not be the function's "signature" + D100, D101, D102, D103, D104, D105, D107 + per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file @@ -51,6 +67,7 @@ deps = flake8 flake8-commas flake8-comprehensions + flake8-docstrings flake8-quotes skip_install = true commands = From 4dd58afaf60ac4b8c9cc42a7b7459084ded60521 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:45:48 -0400 Subject: [PATCH 36/87] Fix flake8-docstrings D200 One-line docstring should fit on one line with quotes --- core/github_api.py | 8 ++------ core/logger.py | 4 +--- core/version_check.py | 8 ++------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/core/github_api.py b/core/github_api.py index 6e44f9f3..b6fb0856 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -4,9 +4,7 @@ import requests class GitHub(object): - """ - Simple api wrapper for the Github API v3. - """ + """Simple api wrapper for the Github API v3.""" def __init__(self, github_repo_user, github_repo, branch='master'): @@ -15,9 +13,7 @@ class GitHub(object): self.branch = branch def _access_api(self, path, params=None): - """ - Access the API at the path given and with the optional params given. - """ + """Access API at given an API path and optional parameters.""" url = 'https://api.github.com/{path}'.format(path='/'.join(path)) data = requests.get(url, params=params, verify=False) return data.json() if data.ok else [] diff --git a/core/logger.py b/core/logger.py index f7fbf9c4..701881cf 100644 --- a/core/logger.py +++ b/core/logger.py @@ -111,9 +111,7 @@ class NTMRotatingLogHandler(object): self.close_log(old_handler) def _config_handler(self): - """ - Configure a file handler to log at file_name and return it. - """ + """Configure a file handler to log at file_name and return it.""" file_handler = logging.FileHandler(self.log_file_path, encoding='utf-8') diff --git a/core/version_check.py b/core/version_check.py index d865b9f3..b39a8b08 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -19,9 +19,7 @@ from core import github_api as github, logger class CheckVersion(object): - """ - Version check class meant to run as a thread object with the SB scheduler. - """ + """Version checker that runs in a thread with the SB scheduler.""" def __init__(self): self.install_type = self.find_install_type() @@ -435,9 +433,7 @@ class SourceUpdateManager(UpdateManager): return def update(self): - """ - Downloads the latest source tarball from github and installs it over the existing version. - """ + """Download and install latest source tarball from github.""" tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format( org=self.github_repo_user, repo=self.github_repo, branch=self.branch) version_path = os.path.join(core.APP_ROOT, u'version.txt') From 777bc7e35d821cb26c89c2828d54c64aae805d5a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:48:25 -0400 Subject: [PATCH 37/87] Fix flake8-docstrings D202 No blank lines allowed after function docstring --- core/logger.py | 3 --- core/utils/naming.py | 2 -- core/version_check.py | 6 ------ 3 files changed, 11 deletions(-) diff --git a/core/logger.py b/core/logger.py index 701881cf..243e3bde 100644 --- a/core/logger.py +++ b/core/logger.py @@ -112,7 +112,6 @@ class NTMRotatingLogHandler(object): def _config_handler(self): """Configure a file handler to log at file_name and return it.""" - file_handler = logging.FileHandler(self.log_file_path, encoding='utf-8') file_handler.setLevel(DB) @@ -133,7 +132,6 @@ class NTMRotatingLogHandler(object): i: Log number to ues """ - return self.log_file_path + ('.{0}'.format(i) if i else '') def _num_logs(self): @@ -142,7 +140,6 @@ class NTMRotatingLogHandler(object): Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ - cur_log = 0 while os.path.isfile(self._log_file_name(cur_log)): cur_log += 1 diff --git a/core/utils/naming.py b/core/utils/naming.py index b54e3258..27ea8c02 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -12,7 +12,6 @@ def sanitize_name(name): >>> sanitize_name('.a.b..') 'a.b' """ - # remove bad chars from the filename name = re.sub(r'[\\/*]', '-', name) name = re.sub(r'[:\'<>|?]', '', name) @@ -34,7 +33,6 @@ def clean_file_name(filename): Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: """ - filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) diff --git a/core/version_check.py b/core/version_check.py index b39a8b08..a5c98abf 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -45,7 +45,6 @@ class CheckVersion(object): 'git': running from source using git 'source': running from source without git """ - # check if we're a windows build if os.path.isdir(os.path.join(core.APP_ROOT, u'.git')): install_type = 'git' @@ -62,7 +61,6 @@ class CheckVersion(object): force: if true the VERSION_NOTIFY setting will be ignored and a check will be forced """ - if not core.VERSION_NOTIFY and not force: logger.log(u'Version checking is disabled, not checking for the newest version') return False @@ -215,7 +213,6 @@ class GitUpdateManager(UpdateManager): Returns: True for success or False for failure """ - output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable if exit_status == 0 and output: @@ -245,7 +242,6 @@ class GitUpdateManager(UpdateManager): Uses git commands to check if there is a newer version that the provided commit hash. If there is a newer version it sets _num_commits_behind. """ - self._newest_commit_hash = None self._num_commits_behind = 0 self._num_commits_ahead = 0 @@ -325,7 +321,6 @@ class GitUpdateManager(UpdateManager): Calls git pull origin in order to update Sick Beard. Returns a bool depending on the call's success. """ - output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable if exit_status == 0: @@ -385,7 +380,6 @@ class SourceUpdateManager(UpdateManager): commit_hash: hash that we're checking against """ - self._num_commits_behind = 0 self._newest_commit_hash = None From 1d7dba8aebb9078e3c68aecbd6fd63bc3be94266 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:02:50 -0400 Subject: [PATCH 38/87] Fix flake8-docstrings D205 1 blank line required between summary line and description --- core/logger.py | 6 ++++-- core/main_db.py | 2 ++ core/utils/naming.py | 7 ++++--- core/version_check.py | 20 ++++++++++++++------ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/core/logger.py b/core/logger.py index 243e3bde..c569bfb3 100644 --- a/core/logger.py +++ b/core/logger.py @@ -127,8 +127,10 @@ class NTMRotatingLogHandler(object): def _log_file_name(self, i): """ - Returns a numbered log file name depending on i. If i==0 it just uses logName, if not it appends - it to the extension (blah.log.3 for i == 3) + Returns a numbered log file name depending on i. + + If i==0 it just uses logName, if not it appends it to the extension + e.g. (blah.log.3 for i == 3) i: Log number to ues """ diff --git a/core/main_db.py b/core/main_db.py index 241b3adf..14a75d21 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -14,6 +14,8 @@ from core import logger def db_filename(filename='nzbtomedia.db', suffix=None): """ + Return the correct location of the database file. + @param filename: The sqlite database filename to use. If not specified, will be made to be nzbtomedia.db @param suffix: The suffix to append to the filename. A '.' will be added diff --git a/core/utils/naming.py b/core/utils/naming.py index 27ea8c02..964fb6fd 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -3,6 +3,8 @@ import re def sanitize_name(name): """ + Remove bad chars from the filename. + >>> sanitize_name('a/b/c') 'a-b-c' >>> sanitize_name('abc') @@ -12,7 +14,6 @@ def sanitize_name(name): >>> sanitize_name('.a.b..') 'a.b' """ - # remove bad chars from the filename name = re.sub(r'[\\/*]', '-', name) name = re.sub(r'[:\'<>|?]', '', name) @@ -27,8 +28,8 @@ def sanitize_name(name): def clean_file_name(filename): - """Cleans up nzb name by removing any . and _ - characters, along with any trailing hyphens. + """ + Cleans up nzb name by removing any . and _ characters and trailing hyphens. Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: diff --git a/core/version_check.py b/core/version_check.py index a5c98abf..ac7fdd6d 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -239,8 +239,11 @@ class GitUpdateManager(UpdateManager): def _check_github_for_update(self): """ - Uses git commands to check if there is a newer version that the provided - commit hash. If there is a newer version it sets _num_commits_behind. + Check Github for a new version. + + Uses git commands to check if there is a newer version than + the provided commit hash. If there is a newer version it + sets _num_commits_behind. """ self._newest_commit_hash = None self._num_commits_behind = 0 @@ -318,8 +321,10 @@ class GitUpdateManager(UpdateManager): def update(self): """ - Calls git pull origin in order to update Sick Beard. Returns a bool depending - on the call's success. + Check git for a new version. + + Calls git pull origin in order to update Sick Beard. + Returns a bool depending on the call's success. """ output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable @@ -375,8 +380,11 @@ class SourceUpdateManager(UpdateManager): def _check_github_for_update(self): """ - Uses pygithub to ask github if there is a newer version that the provided - commit hash. If there is a newer version it sets Sick Beard's version text. + Check Github for a new version. + + Uses pygithub to ask github if there is a newer version than + the provided commit hash. If there is a newer version it sets + Sick Beard's version text. commit_hash: hash that we're checking against """ From 6f6c9bcc9d5732d543550b7c9eb57e0068564faa Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:16:06 -0400 Subject: [PATCH 39/87] Fix flake8-docstrings D400 First line should end with a period --- core/logger.py | 2 +- core/utils/files.py | 2 +- core/utils/paths.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/logger.py b/core/logger.py index c569bfb3..dba98947 100644 --- a/core/logger.py +++ b/core/logger.py @@ -138,7 +138,7 @@ class NTMRotatingLogHandler(object): def _num_logs(self): """ - Scans the log folder and figures out how many log files there are already on disk + Scans the log folder and figures out how many log files there are already on disk. Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ diff --git a/core/utils/files.py b/core/utils/files.py index 895125e1..edb968ae 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -88,7 +88,7 @@ def is_min_size(input_name, min_size): def is_archive_file(filename): - """Check if the filename is allowed for the Archive""" + """Check if the filename is allowed for the Archive.""" for regext in core.COMPRESSED_CONTAINER: if regext.search(filename): return regext.split(filename)[0] diff --git a/core/utils/paths.py b/core/utils/paths.py index fe049ae7..cdb2c3ff 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -74,7 +74,7 @@ def get_dir_size(input_path): def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders""" + """Function to remove empty folders.""" if not os.path.isdir(path): return From 267d8d16322f168b01464a373d18e1e0c5247af2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 23:04:04 -0400 Subject: [PATCH 40/87] Add flake8-bugbear to tox.ini --- tox.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tox.ini b/tox.ini index 2111ddc6..c614d8d3 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,12 @@ commands = max-line-length = 79 verbose = 2 statistics = True +select = +; -- flake8-bugbear -- +; B902 Invalid first argument used for instance method. +; B903 Data class should either be immutable or use __slots__ to save memory. + B902, B903 + ignore = ; -- flake8 -- ; E501 line too long @@ -65,6 +71,7 @@ per-file-ignores = [testenv:check] deps = flake8 + flake8-bugbear flake8-commas flake8-comprehensions flake8-docstrings From e00b5cc1952808d87c8647f1216ac1f1215f4e7a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:43:56 -0400 Subject: [PATCH 41/87] Fix flake8-bugbear B010 Do not call setattr with a constant attribute value, it is not any safer than normal property access. --- core/logger.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/logger.py b/core/logger.py index df5a5120..bdb7e683 100644 --- a/core/logger.py +++ b/core/logger.py @@ -6,6 +6,7 @@ import sys import threading import core +import functools # number of log files to keep NUM_LOGS = 3 @@ -199,9 +200,8 @@ class NTMRotatingLogHandler(object): ntm_logger = logging.getLogger('nzbtomedia') pp_logger = logging.getLogger('postprocess') db_logger = logging.getLogger('db') - setattr(pp_logger, 'postprocess', lambda *args: pp_logger.log(POSTPROCESS, *args)) - setattr(db_logger, 'db', lambda *args: db_logger.log(DB, *args)) - + pp_logger.postprocess = functools.partial(pp_logger.log, POSTPROCESS) + db_logger.db = functools.partial(db_logger.log, DB) try: if log_level == DEBUG: if core.LOG_DEBUG == 1: From 4c8e896bbb9843232f949a2d4ff351e9325e4e10 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 16:02:23 -0400 Subject: [PATCH 42/87] Fix flake8-bugbear B007 Loop control variable not used within the loop body. --- core/auto_process/movies.py | 2 +- core/configuration.py | 4 ++-- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- core/user_scripts.py | 4 ++-- core/utils/encoding.py | 4 ++-- core/version_check.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 76d49b78..4d2a4b2f 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -504,7 +504,7 @@ def get_release(base_url, imdb_id=None, download_id=None, release_id=None): # Narrow results by removing old releases by comparing their last_edit field if len(results) > 1: for id1, x1 in results.items(): - for id2, x2 in results.items(): + for x2 in results.values(): try: if x2['last_edit'] > x1['last_edit']: results.pop(id1) diff --git a/core/configuration.py b/core/configuration.py index 216a1f05..387f9b72 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -136,10 +136,10 @@ class ConfigObj(configobj.ConfigObj, Section): subsections = {} # gather all new-style and old-style sub-sections - for newsection, newitems in CFG_NEW.items(): + for newsection in CFG_NEW: if CFG_NEW[newsection].sections: subsections.update({newsection: CFG_NEW[newsection].sections}) - for section, items in CFG_OLD.items(): + for section in CFG_OLD: if CFG_OLD[section].sections: subsections.update({section: CFG_OLD[section].sections}) for option, value in CFG_OLD[section].items(): diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index d62df0ba..0b70c52c 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -121,7 +121,7 @@ def reverse_filename(filename, dirname, name): def rename_script(dirname): rename_file = '' - for directory, directories, files in os.walk(dirname): + for directory, _, files in os.walk(dirname): for file in files: if re.search(r'(rename\S*\.(sh|bat)$)', file, re.IGNORECASE): rename_file = os.path.join(directory, file) diff --git a/core/transcoder.py b/core/transcoder.py index d4b7e2f2..f0047dad 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -519,7 +519,7 @@ def get_subs(file): sub_ext = ['.srt', '.sub', '.idx'] name = os.path.splitext(os.path.split(file)[1])[0] path = os.path.split(file)[0] - for directory, directories, filenames in os.walk(path): + for directory, _, filenames in os.walk(path): for filename in filenames: filepaths.extend([os.path.join(directory, filename)]) subfiles = [item for item in filepaths if os.path.splitext(item)[1] in sub_ext and name in item] diff --git a/core/user_scripts.py b/core/user_scripts.py index 5d447670..fbc58614 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -47,7 +47,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): logger.info('Corrupt video file found {0}. Deleting.'.format(video), 'USERSCRIPT') os.unlink(video) - for dirpath, dirnames, filenames in os.walk(output_destination): + for dirpath, _, filenames in os.walk(output_destination): for file in filenames: file_path = core.os.path.join(dirpath, file) @@ -102,7 +102,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): final_result += result num_files_new = 0 - for dirpath, dirnames, filenames in os.walk(output_destination): + for _, _, filenames in os.walk(output_destination): for file in filenames: file_name, file_extension = os.path.splitext(file) diff --git a/core/utils/encoding.py b/core/utils/encoding.py index ca19e054..cbcc3113 100644 --- a/core/utils/encoding.py +++ b/core/utils/encoding.py @@ -68,14 +68,14 @@ def convert_to_ascii(input_name, dir_name): if 'NZBOP_SCRIPTDIR' in os.environ: print('[NZB] DIRECTORY={0}'.format(dir_name)) - for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): + for dirname, dirnames, _ in os.walk(dir_name, topdown=False): for subdirname in dirnames: encoded, subdirname2 = char_replace(subdirname) if encoded: logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) - for dirname, dirnames, filenames in os.walk(dir_name): + for dirname, _, filenames in os.walk(dir_name): for filename in filenames: encoded, filename2 = char_replace(filename) if encoded: diff --git a/core/version_check.py b/core/version_check.py index 23384a44..e7dc5578 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -487,7 +487,7 @@ class SourceUpdateManager(UpdateManager): # walk temp folder and move files to main folder logger.log(u'Moving files from {source} to {destination}'.format (source=content_dir, destination=core.APP_ROOT)) - for dirname, dirnames, filenames in os.walk(content_dir): # @UnusedVariable + for dirname, _, filenames in os.walk(content_dir): # @UnusedVariable dirname = dirname[len(content_dir) + 1:] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) From 10b2eab3c571b42fef29b3f5531a23375f980075 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:20:23 -0400 Subject: [PATCH 43/87] Fix flake8-docstrings D401 First line should be in imperative mood --- core/github_api.py | 4 ++-- core/logger.py | 4 ++-- core/utils/naming.py | 2 +- core/utils/paths.py | 2 +- core/version_check.py | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/github_api.py b/core/github_api.py index b6fb0856..b1a4f566 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -20,7 +20,7 @@ class GitHub(object): def commits(self): """ - Uses the API to get a list of the 100 most recent commits from the specified user/repo/branch, starting from HEAD. + Get the 100 most recent commits from the specified user/repo/branch, starting from HEAD. user: The github username of the person whose repo you're querying repo: The repo name to query @@ -35,7 +35,7 @@ class GitHub(object): def compare(self, base, head, per_page=1): """ - Uses the API to get a list of compares between base and head. + Get compares between base and head. user: The github username of the person whose repo you're querying repo: The repo name to query diff --git a/core/logger.py b/core/logger.py index dba98947..df5a5120 100644 --- a/core/logger.py +++ b/core/logger.py @@ -127,7 +127,7 @@ class NTMRotatingLogHandler(object): def _log_file_name(self, i): """ - Returns a numbered log file name depending on i. + Return a numbered log file name depending on i. If i==0 it just uses logName, if not it appends it to the extension e.g. (blah.log.3 for i == 3) @@ -138,7 +138,7 @@ class NTMRotatingLogHandler(object): def _num_logs(self): """ - Scans the log folder and figures out how many log files there are already on disk. + Scan the log folder and figure out how many log files there are already on disk. Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ diff --git a/core/utils/naming.py b/core/utils/naming.py index 964fb6fd..64cd6d70 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -29,7 +29,7 @@ def sanitize_name(name): def clean_file_name(filename): """ - Cleans up nzb name by removing any . and _ characters and trailing hyphens. + Clean up nzb name by removing any . and _ characters and trailing hyphens. Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: diff --git a/core/utils/paths.py b/core/utils/paths.py index cdb2c3ff..a2a96996 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -74,7 +74,7 @@ def get_dir_size(input_path): def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders.""" + """Remove empty folders.""" if not os.path.isdir(path): return diff --git a/core/version_check.py b/core/version_check.py index ac7fdd6d..23384a44 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -38,7 +38,7 @@ class CheckVersion(object): def find_install_type(self): """ - Determines how this copy of SB was installed. + Determine how this copy of SB was installed. returns: type of installation. Possible values are: 'win': any compiled windows build @@ -55,7 +55,7 @@ class CheckVersion(object): def check_for_new_version(self, force=False): """ - Checks the internet for a newer version. + Check the internet for a newer version. returns: bool, True for new version or False for no new version. @@ -207,7 +207,7 @@ class GitUpdateManager(UpdateManager): def _find_installed_version(self): """ - Attempts to find the currently installed version of Sick Beard. + Attempt to find the currently installed version of Sick Beard. Uses git show to get commit version. From 72140e939ced3b85bcdc7d52f1d0357849ff307b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 16:15:21 -0400 Subject: [PATCH 44/87] Fix flake8-bugbear B902 Invalid first argument used for instance method. --- core/configuration.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/configuration.py b/core/configuration.py index 387f9b72..2d34c8bc 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -13,17 +13,17 @@ from core import logger class Section(configobj.Section, object): - def isenabled(section): + def isenabled(self): # checks if subsection enabled, returns true/false if subsection specified otherwise returns true/false in {} - if not section.sections: + if not self.sections: try: - value = list(ConfigObj.find_key(section, 'enabled'))[0] + value = list(ConfigObj.find_key(self, 'enabled'))[0] except Exception: value = 0 if int(value) == 1: - return section + return self else: - to_return = copy.deepcopy(section) + to_return = copy.deepcopy(self) for section_name, subsections in to_return.items(): for subsection in subsections: try: @@ -40,8 +40,8 @@ class Section(configobj.Section, object): return to_return - def findsection(section, key): - to_return = copy.deepcopy(section) + def findsection(self, key): + to_return = copy.deepcopy(self) for subsection in to_return: try: value = list(ConfigObj.find_key(to_return[subsection], key))[0] From 0788a754cbd9d66c3be95e0e74edd06dff9d93cc Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:15:07 -0400 Subject: [PATCH 45/87] Fix code quality checks to run all desired tests Fixes #1602 --- tox.ini | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index c614d8d3..cf22519c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,12 +33,6 @@ commands = max-line-length = 79 verbose = 2 statistics = True -select = -; -- flake8-bugbear -- -; B902 Invalid first argument used for instance method. -; B903 Data class should either be immutable or use __slots__ to save memory. - B902, B903 - ignore = ; -- flake8 -- ; E501 line too long @@ -78,7 +72,15 @@ deps = flake8-quotes skip_install = true commands = +; ** PRIMARY TESTS ** +; Run flake8 tests (with plugins) using default test selections flake8 core tests setup.py +; ** SELECTIVE TESTS ** +; Run flake8 tests (with plugins) for specific optional codes defined below +; -- flake8-bugbear -- +; B902 Invalid first argument used for instance method. +; B903 Data class should be immutable or use __slots__ to save memory. + flake8 core tests setup.py --select=B902,B903 [coverage:run] omit = From e7179dde1c8115233edadb36dc61776ea793d79a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:38:43 -0400 Subject: [PATCH 46/87] Temporarily disable some flake8 ignores for testing --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf22519c..089c95f8 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ statistics = True ignore = ; -- flake8 -- ; E501 line too long - E501 +; E501 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -57,7 +57,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - core/__init__.py: E402, F401 +; core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 From 28ff74d0c8e8ce585e1452b58e84f745e5737fe4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:42:18 -0400 Subject: [PATCH 47/87] Revert "Temporarily disable some flake8 ignores for testing" This reverts commit e7179dde1c8115233edadb36dc61776ea793d79a. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 089c95f8..cf22519c 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ statistics = True ignore = ; -- flake8 -- ; E501 line too long -; E501 + E501 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -57,7 +57,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file -; core/__init__.py: E402, F401 + core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 From 8736642e78929be4ff00a3ef3c5189b83f4d1b11 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:19:35 -0400 Subject: [PATCH 48/87] Fix code quality checks to run on project root and custom libs Fixes #1600 Fixes #1601 --- tox.ini | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf22519c..e9a5bcc1 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,15 @@ commands = max-line-length = 79 verbose = 2 statistics = True +exclude = + .github/ + .tox/ + .pytest_cache/ + htmlcov/ + logs/ + libs/common + libs/win + libs/py2 ignore = ; -- flake8 -- ; E501 line too long @@ -74,13 +83,13 @@ skip_install = true commands = ; ** PRIMARY TESTS ** ; Run flake8 tests (with plugins) using default test selections - flake8 core tests setup.py + flake8 ; ** SELECTIVE TESTS ** ; Run flake8 tests (with plugins) for specific optional codes defined below ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. - flake8 core tests setup.py --select=B902,B903 + flake8 --select=B902,B903 [coverage:run] omit = From 077f04bc53aadcc193c8032d0e15ef57357f725e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:56:50 -0400 Subject: [PATCH 49/87] Fix flake8 E265 block comment should start with '# ' Ignore for NZBGET scripts --- TorrentToMedia.py | 24 ++++++++++++------------ tox.ini | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 06d36e1a..dd7c363a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -60,13 +60,13 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass - #try: + # try: # input_directory = input_directory.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format @@ -125,9 +125,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - #try: + # try: # output_destination = output_destination.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass if output_destination in input_directory: @@ -170,9 +170,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - #try: + # try: # target_file = target_file.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass if root == 1: if not found_file: @@ -353,14 +353,14 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - #try: + # try: # dir_name = dir_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass input_name = os.path.basename(dir_name) - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, diff --git a/tox.ini b/tox.ini index e9a5bcc1..f9ca222c 100644 --- a/tox.ini +++ b/tox.ini @@ -66,6 +66,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file + nzbTo*.py: E265 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From 9dd25f96b2485794d0c8e66cd3b9c4cf5b62b514 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:58:31 -0400 Subject: [PATCH 50/87] Fix flake8 E266 too many leading '#' for block comment Ignore for NZBGET scripts --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f9ca222c..97f17b47 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - nzbTo*.py: E265 + nzbTo*.py: E265, E266 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From daa9819798b4747383d06e6f3a6172a989eabdac Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:06:25 -0400 Subject: [PATCH 51/87] Fix flake8 F401 item imported but unused --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 97f17b47..2230329e 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,7 @@ per-file-ignores = core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 + libs/custom/synchronousdeluge/__init__.py: F401 [testenv:check] deps = From 98e8fd581ae9eff7d634a466b9961cece524376a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:08:31 -0400 Subject: [PATCH 52/87] Fix flake8 E303 too many blank lines --- TorrentToMedia.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index dd7c363a..555b101d 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -259,7 +259,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp elif section_name == 'LazyLibrarian': result = books.process(section_name, output_destination, input_name, status, client_agent, input_category) - plex_update(input_category) if result.status_code != 0: From 9527a2bd677c400d4b5259db1c5625a469f20afe Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:16:35 -0400 Subject: [PATCH 53/87] Fix flake8 E402 module level import not at top of file --- TorrentToMedia.py | 17 ++++++++++------- nzbToMedia.py | 12 ++++++------ tox.ini | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 555b101d..fcd3bac5 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -1,16 +1,15 @@ #!/usr/bin/env python # coding=utf-8 -import eol -eol.check() - -import cleanup -cleanup.clean(cleanup.FOLDER_STRUCTURE) - import datetime import os import sys +import eol +import cleanup +eol.check() +cleanup.clean(cleanup.FOLDER_STRUCTURE) + import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv, books @@ -18,7 +17,11 @@ from core.auto_process.common import ProcessResult from core.plugins.plex import plex_update from core.user_scripts import external_script from core.utils import char_replace, convert_to_ascii, replace_links -from six import text_type + +try: + text_type = unicode +except NameError: + text_type = str def process_torrent(input_directory, input_name, input_category, input_hash, input_id, client_agent): diff --git a/nzbToMedia.py b/nzbToMedia.py index 8d969d6a..4f43329c 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -657,16 +657,16 @@ from __future__ import print_function -import eol -eol.check() - -import cleanup -cleanup.clean(cleanup.FOLDER_STRUCTURE) - import datetime import os import sys +import eol +import cleanup + +eol.check() +cleanup.clean(cleanup.FOLDER_STRUCTURE) + import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv, books diff --git a/tox.ini b/tox.ini index 2230329e..fd7a997b 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,8 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - nzbTo*.py: E265, E266 + nzbTo*.py: E265, E266, E402 + TorrentToMedia.py: E402 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From 90602bf154aa20355466f2bc7d4bcb3d7d866a56 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:17:55 -0400 Subject: [PATCH 54/87] Fix flake8 W293 blank line contains whitespace --- libs/custom/synchronousdeluge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index fbafe6fe..e658debc 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -1,7 +1,7 @@ # coding=utf-8 """A synchronous implementation of the Deluge RPC protocol based on gevent-deluge by Christopher Rosell. - + https://github.com/chrippa/gevent-deluge Example usage: From 181675722d7dad38bb08751c896de8ac75253e9e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:23:24 -0400 Subject: [PATCH 55/87] Fix flake8 W291 trailing whitespace --- libs/custom/utorrent/upload.py | 2 +- nzbToCouchPotato.py | 8 ++++---- nzbToHeadPhones.py | 2 +- nzbToLidarr.py | 14 +++++++------- nzbToMedia.py | 8 ++++---- nzbToMylar.py | 2 +- nzbToNzbDrone.py | 14 +++++++------- nzbToRadarr.py | 14 +++++++------- nzbToSickBeard.py | 8 ++++---- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/libs/custom/utorrent/upload.py b/libs/custom/utorrent/upload.py index 03d58bcc..1ccbcd37 100644 --- a/libs/custom/utorrent/upload.py +++ b/libs/custom/utorrent/upload.py @@ -36,7 +36,7 @@ class MultiPartForm(object): # Build a list of lists, each containing "lines" of the # request. Each part is separated by a boundary string. # Once the list is built, return a string where each - # line is separated by '\r\n'. + # line is separated by '\r\n'. parts = [] part_boundary = '--' + self.boundary diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index be7fb4f3..9869b0e1 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -99,7 +99,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -134,7 +134,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -216,7 +216,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -227,7 +227,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index 34103b0c..34fac2e3 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -101,7 +101,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## WakeOnLan diff --git a/nzbToLidarr.py b/nzbToLidarr.py index d8a20dc8..230d8b9d 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -84,7 +84,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -119,7 +119,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -145,7 +145,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -201,20 +201,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToMedia.py b/nzbToMedia.py index 4f43329c..1d479d95 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -450,7 +450,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -485,7 +485,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -567,7 +567,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -578,7 +578,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame diff --git a/nzbToMylar.py b/nzbToMylar.py index 8fe166e3..d16b357a 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -92,7 +92,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## WakeOnLan diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index b7d0ac33..df551472 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -89,7 +89,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -124,7 +124,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -150,7 +150,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -206,20 +206,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToRadarr.py b/nzbToRadarr.py index 446585bc..fa2aa26e 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -94,7 +94,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -129,7 +129,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -155,7 +155,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -211,20 +211,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 3f1b300d..d342cf0d 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -100,7 +100,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -135,7 +135,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -217,7 +217,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -228,7 +228,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame From 70fa47394ee3abffdf0ce0985d358550737886d7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:26:13 -0400 Subject: [PATCH 56/87] Fix flake8-docstrings D202 No blank lines allowed after function docstring --- libs/custom/synchronousdeluge/client.py | 1 - libs/custom/utorrent/client.py | 1 - 2 files changed, 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index aa180bf7..ec519687 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -141,7 +141,6 @@ class DelugeClient(object): :param username: str, the username to login with :param password: str, the password to login with """ - # Connect transport self.transfer.connect((host, port)) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 2be51c6d..613d94a7 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -32,7 +32,6 @@ class UTorrentClient(object): def _make_opener(self, realm, base_url, username, password): """uTorrent API need HTTP Basic Auth and cookie support for token verify.""" - auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm=realm, uri=base_url, From f98b39cdbb0d5bdaa2f49651e21ccc446f155f00 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:27:31 -0400 Subject: [PATCH 57/87] Fix flake8-docstrings D204 1 blank line required after class docstring --- cleanup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cleanup.py b/cleanup.py index d313f2b4..b4cf650e 100644 --- a/cleanup.py +++ b/cleanup.py @@ -25,6 +25,7 @@ FOLDER_STRUCTURE = { class WorkingDirectory(object): """Context manager for changing current working directory.""" + def __init__(self, new, original=None): self.working_directory = new self.original_directory = os.getcwd() if original is None else original From 73e47466b4b86ffc19ec117da2184b40c99419eb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:30:40 -0400 Subject: [PATCH 58/87] Fix flake8-docstrings D205 1 blank line required between summary line and description --- libs/custom/synchronousdeluge/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index e658debc..b0283e83 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -1,7 +1,8 @@ # coding=utf-8 -"""A synchronous implementation of the Deluge RPC protocol - based on gevent-deluge by Christopher Rosell. +""" +A synchronous implementation of the Deluge RPC protocol. +Based on gevent-deluge by Christopher Rosell: https://github.com/chrippa/gevent-deluge Example usage: From 093f49d5aab84d0319dde02cbeb47887329f6af2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:32:06 -0400 Subject: [PATCH 59/87] Fix flake8-docstrings D401 First line should be in imperative mood --- libs/custom/synchronousdeluge/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index ec519687..817aeba1 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -18,7 +18,7 @@ RPC_EVENT = 3 class DelugeClient(object): def __init__(self): - """A deluge client session.""" + """Create a deluge client session.""" self.transfer = DelugeTransfer() self.modules = [] self._request_counter = 0 @@ -134,7 +134,7 @@ class DelugeClient(object): return response def connect(self, host="127.0.0.1", port=58846, username="", password=""): - """Connects to a daemon process. + """Connect to a daemon process. :param host: str, the hostname of the daemon :param port: int, the port of the daemon From eec977d9096becd66b58afb257a165ad79f8d30e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:33:20 -0400 Subject: [PATCH 60/87] Fix flake8-docstrings D403 First word of the first line should be properly capitalized --- libs/custom/utorrent/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 613d94a7..fe7e53da 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -31,7 +31,7 @@ class UTorrentClient(object): # TODO refresh token, when necessary def _make_opener(self, realm, base_url, username, password): - """uTorrent API need HTTP Basic Auth and cookie support for token verify.""" + """HTTP Basic Auth and cookie support for token verification.""" auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm=realm, uri=base_url, From 81c50efcd6b22651a2b8109a9d1b9367964d77b7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:37:17 -0400 Subject: [PATCH 61/87] Fix flake8-commas C813 missing trailing comma in Python 3 --- cleanup.py | 2 +- eol.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cleanup.py b/cleanup.py index b4cf650e..f0d4e6d1 100644 --- a/cleanup.py +++ b/cleanup.py @@ -44,7 +44,7 @@ class WorkingDirectory(object): original_directory=self.original_directory, error=error, working_directory=self.working_directory, - ) + ), ) diff --git a/eol.py b/eol.py index a67bfc9e..5504df7e 100644 --- a/eol.py +++ b/eol.py @@ -157,7 +157,7 @@ def print_statuses(show_expired=False): major=python_version[0], minor=python_version[1], remaining=days_left, - ) + ), ) if not show_expired: return @@ -171,7 +171,7 @@ def print_statuses(show_expired=False): major=python_version[0], minor=python_version[1], remaining=-days_left, - ) + ), ) From d608000345f8d6f8a899ca394e0865c8e514bb91 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:38:27 -0400 Subject: [PATCH 62/87] Fix flake8-commas C819 trailing comma prohibited --- libs/custom/utorrent/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index fe7e53da..5d3e1fec 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -60,25 +60,25 @@ class UTorrentClient(object): return self._action(params) def start(self, *hashes): - params = [('action', 'start'), ] + params = [('action', 'start')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def stop(self, *hashes): - params = [('action', 'stop'), ] + params = [('action', 'stop')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def pause(self, *hashes): - params = [('action', 'pause'), ] + params = [('action', 'pause')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def forcestart(self, *hashes): - params = [('action', 'forcestart'), ] + params = [('action', 'forcestart')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) @@ -124,13 +124,13 @@ class UTorrentClient(object): self._action(params) def remove(self, *hashes): - params = [('action', 'remove'), ] + params = [('action', 'remove')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def removedata(self, *hashes): - params = [('action', 'removedata'), ] + params = [('action', 'removedata')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) From 99159acd8033a22dc518479cc7aaf31c480ce840 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:39:48 -0400 Subject: [PATCH 63/87] Fix flake8-bugbear B007 Loop control variable not used within the loop body. --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index fcd3bac5..e3665c5e 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -281,7 +281,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # remove torrent if core.USE_LINK == 'move-sym' and not core.DELETE_ORIGINAL == 1: logger.debug('Checking for sym-links to re-direct in: {0}'.format(input_directory)) - for dirpath, dirs, files in os.walk(input_directory): + for dirpath, _, files in os.walk(input_directory): for file in files: logger.debug('Checking symlink: {0}'.format(os.path.join(dirpath, file))) replace_links(os.path.join(dirpath, file)) From 9f52406d45c4a9830190f87b13ea95ef6baf411c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:44:33 -0400 Subject: [PATCH 64/87] Fix flake8-quotes Q000 Remove bad quotes --- libs/custom/synchronousdeluge/__init__.py | 6 ++-- libs/custom/synchronousdeluge/client.py | 36 ++++++++++----------- libs/custom/synchronousdeluge/exceptions.py | 2 +- libs/custom/synchronousdeluge/transfer.py | 4 +-- libs/custom/utorrent/client.py | 4 +-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index b0283e83..ad90cfe8 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -19,6 +19,6 @@ Example usage: from .exceptions import DelugeRPCError -__title__ = "synchronous-deluge" -__version__ = "0.1" -__author__ = "Christian Dale" +__title__ = 'synchronous-deluge' +__version__ = '0.1' +__author__ = 'Christian Dale' diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 817aeba1..9194ebe8 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -9,7 +9,7 @@ from .exceptions import DelugeRPCError from .protocol import DelugeRPCRequest, DelugeRPCResponse from .transfer import DelugeTransfer -__all__ = ["DelugeClient"] +__all__ = ['DelugeClient'] RPC_RESPONSE = 1 RPC_ERROR = 2 @@ -24,35 +24,35 @@ class DelugeClient(object): self._request_counter = 0 def _get_local_auth(self): - username = password = "" + username = password = '' if platform.system() in ('Windows', 'Microsoft'): - app_data_path = os.environ.get("APPDATA") + app_data_path = os.environ.get('APPDATA') if not app_data_path: from six.moves import winreg hkey = winreg.OpenKey( winreg.HKEY_CURRENT_USER, - "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders", + 'Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders', ) - app_data_reg = winreg.QueryValueEx(hkey, "AppData") + app_data_reg = winreg.QueryValueEx(hkey, 'AppData') app_data_path = app_data_reg[0] winreg.CloseKey(hkey) - auth_file = os.path.join(app_data_path, "deluge", "auth") + auth_file = os.path.join(app_data_path, 'deluge', 'auth') else: from xdg.BaseDirectory import save_config_path try: - auth_file = os.path.join(save_config_path("deluge"), "auth") + auth_file = os.path.join(save_config_path('deluge'), 'auth') except OSError: return username, password if os.path.exists(auth_file): for line in open(auth_file): - if line.startswith("#"): + if line.startswith('#'): # This is a comment line continue line = line.strip() try: - lsplit = line.split(":") + lsplit = line.split(':') except Exception: continue @@ -63,13 +63,13 @@ class DelugeClient(object): else: continue - if username == "localclient": + if username == 'localclient': return username, password - return "", "" + return '', '' def _create_module_method(self, module, method): - fullname = "{0}.{1}".format(module, method) + fullname = '{0}.{1}'.format(module, method) def func(obj, *args, **kwargs): return self.remote_call(fullname, *args, **kwargs) @@ -80,18 +80,18 @@ class DelugeClient(object): def _introspect(self): def splitter(value): - return value.split(".") + return value.split('.') self.modules = [] - methods = self.remote_call("daemon.get_method_list").get() + methods = self.remote_call('daemon.get_method_list').get() methodmap = defaultdict(dict) for module, method in imap(splitter, methods): methodmap[module][method] = self._create_module_method(module, method) for module, methods in methodmap.items(): - clsname = "DelugeModule{0}".format(module.capitalize()) + clsname = 'DelugeModule{0}'.format(module.capitalize()) cls = type(clsname, (), methods) setattr(self, module, cls()) self.modules.append(module) @@ -133,7 +133,7 @@ class DelugeClient(object): self._request_counter += 1 return response - def connect(self, host="127.0.0.1", port=58846, username="", password=""): + def connect(self, host='127.0.0.1', port=58846, username='', password=''): """Connect to a daemon process. :param host: str, the hostname of the daemon @@ -145,11 +145,11 @@ class DelugeClient(object): self.transfer.connect((host, port)) # Attempt to fetch local auth info if needed - if not username and host in ("127.0.0.1", "localhost"): + if not username and host in ('127.0.0.1', 'localhost'): username, password = self._get_local_auth() # Authenticate - self.remote_call("daemon.login", username, password).get() + self.remote_call('daemon.login', username, password).get() # Introspect available methods self._introspect() diff --git a/libs/custom/synchronousdeluge/exceptions.py b/libs/custom/synchronousdeluge/exceptions.py index 6da4bdde..dcb23009 100644 --- a/libs/custom/synchronousdeluge/exceptions.py +++ b/libs/custom/synchronousdeluge/exceptions.py @@ -8,4 +8,4 @@ class DelugeRPCError(Exception): self.traceback = traceback def __str__(self): - return "{0}: {1}: {2}".format(self.__class__.__name__, self.name, self.msg) + return '{0}: {1}: {2}'.format(self.__class__.__name__, self.name, self.msg) diff --git a/libs/custom/synchronousdeluge/transfer.py b/libs/custom/synchronousdeluge/transfer.py index 42863b83..cbf49100 100644 --- a/libs/custom/synchronousdeluge/transfer.py +++ b/libs/custom/synchronousdeluge/transfer.py @@ -6,7 +6,7 @@ import zlib import rencode -__all__ = ["DelugeTransfer"] +__all__ = ['DelugeTransfer'] class DelugeTransfer(object): @@ -33,7 +33,7 @@ class DelugeTransfer(object): payload = zlib.compress(rencode.dumps(data)) self.conn.sendall(payload) - buf = b"" + buf = b'' while True: data = self.conn.recv(1024) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 5d3e1fec..ee9ef983 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -94,8 +94,8 @@ class UTorrentClient(object): def setprops(self, cur_hash, **kvpairs): params = [('action', 'setprops'), ('hash', cur_hash)] for k, v in iteritems(kvpairs): - params.append(("s", k)) - params.append(("v", v)) + params.append(('s', k)) + params.append(('v', v)) return self._action(params) From c6e35bd2db42fb9e3e9914602fb85da391d56ceb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 14:20:20 -0400 Subject: [PATCH 65/87] Add optional flake8 tests to selective testing Ignore W505 (doc string length) for now --- tox.ini | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fd7a997b..b1074483 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,7 @@ commands = [flake8] max-line-length = 79 +max-doc-length = 79 verbose = 2 statistics = True exclude = @@ -45,7 +46,8 @@ exclude = ignore = ; -- flake8 -- ; E501 line too long - E501 +; W505 doc line too long + E501, W505 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -89,10 +91,18 @@ commands = flake8 ; ** SELECTIVE TESTS ** ; Run flake8 tests (with plugins) for specific optional codes defined below +; -- flake8 -- +; E123 closing bracket does not match indentation of opening bracket’s line +; E226 missing whitespace around arithmetic operator +; E241 multiple spaces after ‘,’ +; E242 tab after ‘,’ +; E704 multiple statements on one line +; W504 line break after binary operator +; W505 doc line too long ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. - flake8 --select=B902,B903 +flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 [coverage:run] omit = From f42cc020ea2a7f1f440a7def925b7a1a0813f095 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 23:19:14 -0400 Subject: [PATCH 66/87] Add flake8-future-import to tox.ini --- tox.ini | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b1074483..cb6b6642 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,8 @@ max-line-length = 79 max-doc-length = 79 verbose = 2 statistics = True +min-version = 2.7 +require-code = True exclude = .github/ .tox/ @@ -65,6 +67,20 @@ ignore = ; D402 First line should not be the function's "signature" D100, D101, D102, D103, D104, D105, D107 +; -- flake8-future-import -- +; x = 1 for missing, 5 for present +; FIx6 nested_scopes 2.2 +; FIx7 generators 2.3 +; FIx2 with_statement 2.6 +; FIx1 absolute_import 3.0 +; FIx0 division 3.0 +; FIx3 print_function 3.0 +; FIx4 unicode_literals 3.0 +; FIx5 generator_stop 3.7 +; ???? annotations 4.0 +; FI90 __future__ import does not exist + FI50, FI51, FI53, FI54 + per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file @@ -83,6 +99,7 @@ deps = flake8-commas flake8-comprehensions flake8-docstrings + flake8-future-import flake8-quotes skip_install = true commands = @@ -102,7 +119,7 @@ commands = ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. -flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 + flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 [coverage:run] omit = From 424879e4b6ab60953199b30cd78c8dfad6761dfb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 17:44:25 -0400 Subject: [PATCH 67/87] Add future imports --- TorrentToMedia.py | 7 +++++++ cleanup.py | 7 ++++++- core/__init__.py | 7 ++++++- core/auto_process/books.py | 7 +++++++ core/auto_process/comics.py | 7 +++++++ core/auto_process/common.py | 7 +++++++ core/auto_process/games.py | 7 +++++++ core/auto_process/movies.py | 7 +++++++ core/auto_process/music.py | 7 +++++++ core/auto_process/tv.py | 7 +++++++ core/configuration.py | 7 +++++++ core/databases.py | 7 +++++++ core/extractor/__init__.py | 7 +++++++ core/forks.py | 7 +++++++ core/github_api.py | 7 +++++++ core/logger.py | 7 +++++++ core/main_db.py | 7 ++++++- core/plugins/downloaders/nzb/configuration.py | 7 +++++++ core/plugins/downloaders/nzb/utils.py | 7 +++++++ core/plugins/downloaders/torrent/configuration.py | 7 +++++++ core/plugins/downloaders/torrent/deluge.py | 7 +++++++ core/plugins/downloaders/torrent/qbittorrent.py | 8 +++++++- core/plugins/downloaders/torrent/transmission.py | 7 +++++++ core/plugins/downloaders/torrent/utils.py | 7 +++++++ core/plugins/downloaders/torrent/utorrent.py | 7 ++++++- core/plugins/plex.py | 7 +++++++ core/plugins/subtitles.py | 7 +++++++ core/scene_exceptions.py | 7 +++++++ core/transcoder.py | 7 +++++++ core/user_scripts.py | 7 +++++++ core/utils/__init__.py | 7 +++++++ core/utils/common.py | 6 ++++++ core/utils/download_info.py | 7 +++++++ core/utils/encoding.py | 7 +++++++ core/utils/files.py | 7 +++++++ core/utils/identification.py | 7 +++++++ core/utils/links.py | 7 +++++++ core/utils/naming.py | 7 +++++++ core/utils/network.py | 7 +++++++ core/utils/parsers.py | 7 +++++++ core/utils/paths.py | 6 ++++++ core/utils/processes.py | 7 +++++++ core/utils/shutil_custom.py | 7 +++++++ core/version_check.py | 7 +++++++ eol.py | 7 +++++++ libs/__init__.py | 7 +++++++ libs/__main__.py | 7 +++++++ libs/autoload.py | 7 +++++++ libs/custom/synchronousdeluge/__init__.py | 7 +++++++ libs/custom/synchronousdeluge/client.py | 8 ++++++++ libs/custom/synchronousdeluge/exceptions.py | 7 +++++++ libs/custom/synchronousdeluge/protocol.py | 7 +++++++ libs/custom/synchronousdeluge/transfer.py | 8 ++++++++ libs/custom/utorrent/client.py | 8 ++++++++ libs/custom/utorrent/upload.py | 7 +++++++ libs/util.py | 7 +++++++ nzbToCouchPotato.py | 7 +++++++ nzbToGamez.py | 7 +++++++ nzbToHeadPhones.py | 7 +++++++ nzbToLazyLibrarian.py | 7 +++++++ nzbToLidarr.py | 7 +++++++ nzbToMedia.py | 7 ++++++- nzbToMylar.py | 7 +++++++ nzbToNzbDrone.py | 7 +++++++ nzbToRadarr.py | 7 +++++++ nzbToSickBeard.py | 7 +++++++ setup.py | 7 ++++++- tests/__init__.py | 7 +++++++ tests/test_initialize.py | 7 ++++++- tests/test_transcoder.py | 7 ++++++- 70 files changed, 483 insertions(+), 9 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index e3665c5e..51a257c8 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -1,6 +1,13 @@ #!/usr/bin/env python # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime import os import sys diff --git a/cleanup.py b/cleanup.py index f0d4e6d1..70c9ce29 100644 --- a/cleanup.py +++ b/cleanup.py @@ -1,6 +1,11 @@ #!/usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import os import subprocess diff --git a/core/__init__.py b/core/__init__.py index 85aeaf27..4621ecfb 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,6 +1,11 @@ # coding=utf-8 -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import itertools import locale diff --git a/core/auto_process/books.py b/core/auto_process/books.py index 8ba74a43..c746d785 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests import core diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 1669d8d8..bb884404 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import requests diff --git a/core/auto_process/common.py b/core/auto_process/common.py index 8c3d9ef2..c083414c 100644 --- a/core/auto_process/common.py +++ b/core/auto_process/common.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from core import logger diff --git a/core/auto_process/games.py b/core/auto_process/games.py index 410b82a3..73907d25 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import shutil diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 4d2a4b2f..2661edd1 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import os import time diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 2f614ee1..e72d9b8c 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import os import time diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 3f8f27c5..57fe2a1e 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import copy import errno import json diff --git a/core/configuration.py b/core/configuration.py index 2d34c8bc..c8e2a465 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import copy import os import shutil diff --git a/core/databases.py b/core/databases.py index 0803b9fc..50e025f3 100644 --- a/core/databases.py +++ b/core/databases.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from core import logger, main_db from core.utils import backup_versioned_file diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index 5d1c51a0..44be3102 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import shutil diff --git a/core/forks.py b/core/forks.py index 400d7e50..c1721aaf 100644 --- a/core/forks.py +++ b/core/forks.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from six import iteritems diff --git a/core/github_api.py b/core/github_api.py index b1a4f566..d61def2f 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests diff --git a/core/logger.py b/core/logger.py index bdb7e683..98b15432 100644 --- a/core/logger.py +++ b/core/logger.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os import sys diff --git a/core/main_db.py b/core/main_db.py index 14a75d21..4be8aa3c 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -1,6 +1,11 @@ # coding=utf-8 -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import re import sqlite3 diff --git a/core/plugins/downloaders/nzb/configuration.py b/core/plugins/downloaders/nzb/configuration.py index d0883d01..6acd4fc4 100644 --- a/core/plugins/downloaders/nzb/configuration.py +++ b/core/plugins/downloaders/nzb/configuration.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import core diff --git a/core/plugins/downloaders/nzb/utils.py b/core/plugins/downloaders/nzb/utils.py index 23061c96..f5dd29ba 100644 --- a/core/plugins/downloaders/nzb/utils.py +++ b/core/plugins/downloaders/nzb/utils.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import requests diff --git a/core/plugins/downloaders/torrent/configuration.py b/core/plugins/downloaders/torrent/configuration.py index 92001636..dd8e8ffa 100644 --- a/core/plugins/downloaders/torrent/configuration.py +++ b/core/plugins/downloaders/torrent/configuration.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import core from core.plugins.downloaders.torrent.utils import create_torrent_class diff --git a/core/plugins/downloaders/torrent/deluge.py b/core/plugins/downloaders/torrent/deluge.py index 69ae2000..86542a8a 100644 --- a/core/plugins/downloaders/torrent/deluge.py +++ b/core/plugins/downloaders/torrent/deluge.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from synchronousdeluge.client import DelugeClient import core diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py index 61e84a7d..0532b9a7 100644 --- a/core/plugins/downloaders/torrent/qbittorrent.py +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -1,4 +1,10 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from qbittorrent import Client as qBittorrentClient diff --git a/core/plugins/downloaders/torrent/transmission.py b/core/plugins/downloaders/torrent/transmission.py index d122aaf0..8c58eae2 100644 --- a/core/plugins/downloaders/torrent/transmission.py +++ b/core/plugins/downloaders/torrent/transmission.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from transmissionrpc.client import Client as TransmissionClient import core diff --git a/core/plugins/downloaders/torrent/utils.py b/core/plugins/downloaders/torrent/utils.py index be2db4e8..0d2896d7 100644 --- a/core/plugins/downloaders/torrent/utils.py +++ b/core/plugins/downloaders/torrent/utils.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import time import core diff --git a/core/plugins/downloaders/torrent/utorrent.py b/core/plugins/downloaders/torrent/utorrent.py index 634e8ca5..e591c46b 100644 --- a/core/plugins/downloaders/torrent/utorrent.py +++ b/core/plugins/downloaders/torrent/utorrent.py @@ -1,4 +1,9 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from utorrent.client import UTorrentClient diff --git a/core/plugins/plex.py b/core/plugins/plex.py index 6d851e85..5c3f0e8b 100644 --- a/core/plugins/plex.py +++ b/core/plugins/plex.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests import core diff --git a/core/plugins/subtitles.py b/core/plugins/subtitles.py index f62f90de..df37e532 100644 --- a/core/plugins/subtitles.py +++ b/core/plugins/subtitles.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from babelfish import Language import subliminal diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 0b70c52c..f00eccd3 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import re diff --git a/core/transcoder.py b/core/transcoder.py index f0047dad..3228c810 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import errno import json import os diff --git a/core/user_scripts.py b/core/user_scripts.py index fbc58614..45adb45b 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os from subprocess import Popen diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 470fe4fd..202e7d19 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from core.utils import shutil_custom diff --git a/core/utils/common.py b/core/utils/common.py index 5ecdf830..2174644c 100644 --- a/core/utils/common.py +++ b/core/utils/common.py @@ -1,3 +1,9 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import os.path diff --git a/core/utils/download_info.py b/core/utils/download_info.py index ce6e6717..2e30d35c 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime from six import text_type diff --git a/core/utils/encoding.py b/core/utils/encoding.py index cbcc3113..bcc4994f 100644 --- a/core/utils/encoding.py +++ b/core/utils/encoding.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os from six import text_type diff --git a/core/utils/files.py b/core/utils/files.py index edb968ae..726d4209 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import re import shutil diff --git a/core/utils/identification.py b/core/utils/identification.py index 9029f7e4..365ea790 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import re diff --git a/core/utils/links.py b/core/utils/links.py index ab558e9b..1daa2441 100644 --- a/core/utils/links.py +++ b/core/utils/links.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import shutil diff --git a/core/utils/naming.py b/core/utils/naming.py index 64cd6d70..4d664036 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import re diff --git a/core/utils/network.py b/core/utils/network.py index 5a7a5758..bfb54381 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import socket import struct import time diff --git a/core/utils/parsers.py b/core/utils/parsers.py index d5f6d933..981de146 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import core diff --git a/core/utils/paths.py b/core/utils/paths.py index a2a96996..ee0a48db 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -1,3 +1,9 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from functools import partial import os diff --git a/core/utils/processes.py b/core/utils/processes.py index 6edcfa44..23bb7a92 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import socket import subprocess diff --git a/core/utils/shutil_custom.py b/core/utils/shutil_custom.py index 5525df1f..a6cb3af8 100644 --- a/core/utils/shutil_custom.py +++ b/core/utils/shutil_custom.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from functools import partial import shutil from six import PY2 diff --git a/core/version_check.py b/core/version_check.py index e7dc5578..f1c05ba9 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -2,6 +2,13 @@ # Author: Nic Wolfe # Modified by: echel0n +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import re diff --git a/eol.py b/eol.py index 5504df7e..7310cfd6 100644 --- a/eol.py +++ b/eol.py @@ -1,5 +1,12 @@ #!/usr/bin/env python +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime import sys import warnings diff --git a/libs/__init__.py b/libs/__init__.py index 4405201d..6c9b28b4 100644 --- a/libs/__init__.py +++ b/libs/__init__.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import site import sys diff --git a/libs/__main__.py b/libs/__main__.py index 767f0d6e..b3d3d8e2 100644 --- a/libs/__main__.py +++ b/libs/__main__.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import shutil import os import time diff --git a/libs/autoload.py b/libs/autoload.py index 23205448..a44ac0fb 100644 --- a/libs/autoload.py +++ b/libs/autoload.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import libs __all__ = ['completed'] diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index ad90cfe8..fef43298 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -16,6 +16,13 @@ Example usage: download_location = client.core.get_config_value("download_location").get() """ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from .exceptions import DelugeRPCError diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 9194ebe8..191c70f6 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -1,4 +1,12 @@ # coding=utf-8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform from collections import defaultdict diff --git a/libs/custom/synchronousdeluge/exceptions.py b/libs/custom/synchronousdeluge/exceptions.py index dcb23009..1a8693e6 100644 --- a/libs/custom/synchronousdeluge/exceptions.py +++ b/libs/custom/synchronousdeluge/exceptions.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + class DelugeRPCError(Exception): def __init__(self, name, msg, traceback): diff --git a/libs/custom/synchronousdeluge/protocol.py b/libs/custom/synchronousdeluge/protocol.py index 6dcdb2a1..b71dafb4 100644 --- a/libs/custom/synchronousdeluge/protocol.py +++ b/libs/custom/synchronousdeluge/protocol.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + class DelugeRPCRequest(object): def __init__(self, request_id, method, *args, **kwargs): diff --git a/libs/custom/synchronousdeluge/transfer.py b/libs/custom/synchronousdeluge/transfer.py index cbf49100..75228032 100644 --- a/libs/custom/synchronousdeluge/transfer.py +++ b/libs/custom/synchronousdeluge/transfer.py @@ -1,4 +1,12 @@ # coding=utf-8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import socket import ssl import struct diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index ee9ef983..f5eaf93b 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -1,4 +1,12 @@ # coding=utf8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import re from six import StringIO, iteritems diff --git a/libs/custom/utorrent/upload.py b/libs/custom/utorrent/upload.py index 1ccbcd37..6aa829a4 100644 --- a/libs/custom/utorrent/upload.py +++ b/libs/custom/utorrent/upload.py @@ -1,6 +1,13 @@ # coding=utf-8 # code copied from http://www.doughellmann.com/PyMOTW/urllib2/ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import itertools import mimetypes from email.generator import _make_boundary as choose_boundary diff --git a/libs/util.py b/libs/util.py index 5ac9f943..354b14cf 100644 --- a/libs/util.py +++ b/libs/util.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import subprocess import sys import os diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 9869b0e1..16785b46 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -255,6 +255,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToGamez.py b/nzbToGamez.py index 42d0a2f5..39023421 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -100,6 +100,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index 34fac2e3..a1917f10 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -122,6 +122,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index 0a26013b..a8b88067 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -95,6 +95,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToLidarr.py b/nzbToLidarr.py index 230d8b9d..9681a408 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -237,6 +237,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToMedia.py b/nzbToMedia.py index 1d479d95..1eed58f9 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -655,7 +655,12 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import datetime import os diff --git a/nzbToMylar.py b/nzbToMylar.py index d16b357a..e26cfe6d 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -113,6 +113,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index df551472..fbc3de0c 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -242,6 +242,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToRadarr.py b/nzbToRadarr.py index fa2aa26e..504bbe52 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -247,6 +247,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index d342cf0d..43d23d3d 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -256,6 +256,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/setup.py b/setup.py index 36e97020..bac19586 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,11 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import, print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import io import os.path diff --git a/tests/__init__.py b/tests/__init__.py index 1f47cffe..13e499a5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,8 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + __author__ = 'Justin' diff --git a/tests/test_initialize.py b/tests/test_initialize.py index dffcfe76..9e672f4f 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -1,5 +1,10 @@ #! /usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import core diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 148156b0..448ceeaf 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -1,5 +1,10 @@ #! /usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import core from core import transcoder From d3bbcb6b635c686f440b52475345720cad6cb807 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 8 Apr 2019 19:23:21 -0400 Subject: [PATCH 68/87] Remove unnecessary dict factory for database. --- core/main_db.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/core/main_db.py b/core/main_db.py index 4be8aa3c..220b44ca 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -37,10 +37,7 @@ class DBConnection(object): self.filename = filename self.connection = sqlite3.connect(db_filename(filename), 20) - if row_type == 'dict': - self.connection.row_factory = self._dict_factory - else: - self.connection.row_factory = sqlite3.Row + self.connection.row_factory = sqlite3.Row def check_db_version(self): result = None @@ -214,13 +211,6 @@ class DBConnection(object): for column in cursor } - # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query - def _dict_factory(self, cursor, row): - return { - col[0]: row[idx] - for idx, col in enumerate(cursor.description) - } - def sanity_check_database(connection, sanity_check): sanity_check(connection).check() From 455915907b97fd80a4666291134ad10257c45e18 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 8 Apr 2019 19:22:16 -0400 Subject: [PATCH 69/87] Fix key access for sqlite3.Row on Python 2.7 Fixes #1607 --- core/main_db.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/core/main_db.py b/core/main_db.py index 220b44ca..d9d7c1b9 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -11,11 +11,37 @@ import re import sqlite3 import time -from six import text_type +from six import text_type, PY2 import core from core import logger +if PY2: + class Row(sqlite3.Row, object): + """ + Row factory that uses Byte Strings for keys. + + The sqlite3.Row in Python 2 does not support unicode keys. + This overrides __getitem__ to attempt to encode the key to bytes first. + """ + + def __getitem__(self, item): + """ + Get an item from the row by index or key. + + :param item: Index or Key of item to return. + :return: An item from the sqlite3.Row. + """ + try: + # sqlite3.Row column names should be Bytes in Python 2 + item = item.encode() + except AttributeError: + pass # assume item is a numeric index + + return super(Row, self).__getitem__(item) +else: + from sqlite3 import Row + def db_filename(filename='nzbtomedia.db', suffix=None): """ @@ -37,7 +63,7 @@ class DBConnection(object): self.filename = filename self.connection = sqlite3.connect(db_filename(filename), 20) - self.connection.row_factory = sqlite3.Row + self.connection.row_factory = Row def check_db_version(self): result = None From 52cae37609e3cf9db2a19d1d9d01147bf57629b6 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 18 Apr 2019 08:40:11 +1200 Subject: [PATCH 70/87] Fix crash of remote_path exception. #1223 --- nzbToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMedia.py b/nzbToMedia.py index 1eed58f9..4435ca04 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -782,7 +782,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d ) except Exception: logger.error('Remote Path {0} is not valid for {1}:{2} Please set this to either 0 to disable or 1 to enable!'.format( - core.get('remote_path'), section_name, input_category)) + cfg.get('remote_path'), section_name, input_category)) input_name, input_directory = convert_to_ascii(input_name, input_directory) From 5375d46c327ee16c90aed4f29ead9c0adad46f92 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 18 Apr 2019 21:46:32 +1200 Subject: [PATCH 71/87] add remote path handling for LazyLibrarian #1223 --- core/auto_process/books.py | 10 ++++++++-- nzbToLazyLibrarian.py | 13 +++++++++++++ nzbToMedia.py | 5 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index c746d785..9c908a94 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -12,7 +12,12 @@ import requests import core from core import logger from core.auto_process.common import ProcessResult -from core.utils import convert_to_ascii, server_responding +from core.utils import ( + convert_to_ascii, + remote_dir, + server_responding, +) + requests.packages.urllib3.disable_warnings() @@ -28,6 +33,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', ssl = int(cfg.get('ssl', 0)) web_root = cfg.get('web_root', '') protocol = 'https://' if ssl else 'http://' + remote_path = int(cfg.get('remote_path', 0)) url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) if not server_responding(url): @@ -42,7 +48,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': dir_name, + 'dir': remote_dir(dir_name) if remote_path else dir_name, } logger.debug('Opening URL: {0} with params: {1}'.format(url, params), section) diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index a8b88067..dd760709 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -59,6 +59,19 @@ # set this to where your LazyLibrarian completed downloads are. #llwatch_dir= +# LazyLibrarian and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#llremote_path=0 + +## Network + +# Network Mount Points (Needed for remote path above) +# +# Enter Mount points as LocalPath,RemotePath and separate each pair with '|' +# e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ +#mountPoints= + ## Posix # Niceness for external tasks Extractor and Transcoder. diff --git a/nzbToMedia.py b/nzbToMedia.py index 4435ca04..f4fb0759 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -444,6 +444,11 @@ # set this to where your LazyLibrarian completed downloads are. #llwatch_dir= +# LazyLibrarian and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#llremote_path=0 + ## Network # Network Mount Points (Needed for remote path above) From 5ff056844c10c5d8905700727c3c503c7a35f928 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Mon, 20 May 2019 21:17:54 +1200 Subject: [PATCH 72/87] Fix NoExtractFailed usage. Fixes #1618 --- nzbToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMedia.py b/nzbToMedia.py index f4fb0759..03f19a8d 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -791,7 +791,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d input_name, input_directory = convert_to_ascii(input_name, input_directory) - if extract == 1: + if extract == 1 and not (status > 0 and core.NOEXTRACTFAILED): logger.debug('Checking for archives to extract in directory: {0}'.format(input_directory)) extract_files(input_directory) From 8c45e7650774421cbdd9781f01f55edfcf8ee955 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 31 May 2019 14:06:25 +1200 Subject: [PATCH 73/87] Bluray 1 (#1620) * added code to extract bluray images and folder structure. #1588 * add Mounting of iso files as fall-back * add new mkv-bluray default. * clean-up fall-back for ffmpeg not accepting -show error --- autoProcessMedia.cfg.spec | 2 +- core/__init__.py | 13 ++- core/transcoder.py | 222 +++++++++++++++++++++++++++++++------- nzbToCouchPotato.py | 2 +- nzbToLidarr.py | 2 +- nzbToMedia.py | 2 +- nzbToNzbDrone.py | 2 +- nzbToRadarr.py | 2 +- nzbToSickBeard.py | 2 +- 9 files changed, 205 insertions(+), 44 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 30a399a0..6829d6b2 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -418,7 +418,7 @@ generalOptions = # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, leave this blank and set the remaining options below. - # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release + # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release outputDefault = #### Define custom settings below. outputVideoExtension = .mp4 diff --git a/core/__init__.py b/core/__init__.py index 4621ecfb..7fb32140 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -199,6 +199,7 @@ META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] +MOUNTED = None GETSUBS = False TRANSCODE = None CONCAT = None @@ -504,6 +505,7 @@ def configure_containers(): def configure_transcoder(): + global MOUNTED global GETSUBS global TRANSCODE global DUPLICATE @@ -548,6 +550,7 @@ def configure_transcoder(): global ALLOWSUBS global DEFAULTS + MOUNTED = None GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) DUPLICATE = int(CFG['Transcoder']['duplicate']) @@ -751,7 +754,15 @@ def configure_transcoder(): }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'SCODEC': 'mov_text' + }, + 'mkv-bluray': { + 'VEXTENSION': '.mkv', 'VCODEC': 'libx265', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'hevc', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, diff --git a/core/transcoder.py b/core/transcoder.py index 3228c810..e1850de2 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -9,7 +9,9 @@ from __future__ import ( import errno import json +import sys import os +import time import platform import re import shutil @@ -73,7 +75,10 @@ def is_video_good(videofile, status): def zip_out(file, img, bitbucket): procin = None - cmd = [core.SEVENZIP, '-so', 'e', img, file] + if os.path.isfile(file): + cmd = ['cat', file] + else: + cmd = [core.SEVENZIP, '-so', 'e', img, file] try: procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) except Exception: @@ -104,12 +109,11 @@ def get_video_details(videofile, img=None, bitbucket=None): result = proc.returncode video_details = json.loads(out.decode()) except Exception: - pass - if not video_details: - try: + try: # try this again without -show error in case of ffmpeg limitation command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] + print_cmd(command) if img: - procin = zip_out(file, img) + procin = zip_out(file, img, bitbucket) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) procin.stdout.close() else: @@ -122,6 +126,21 @@ def get_video_details(videofile, img=None, bitbucket=None): return video_details, result +def check_vid_file(video_details, result): + if result != 0: + return False + if video_details.get('error'): + return False + if not video_details.get('streams'): + return False + video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] + audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + if len(video_streams) > 0 and len(audio_streams) > 0: + return True + else: + return False + + def build_commands(file, new_dir, movie_name, bitbucket): if isinstance(file, string_types): input_file = file @@ -139,9 +158,18 @@ def build_commands(file, new_dir, movie_name, bitbucket): name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext' + new_file = file else: img, data = next(iteritems(file)) name = data['name'] + new_file = [] + rem_vid = [] + for vid in data['files']: + video_details, result = get_video_details(vid, img, bitbucket) + if not check_vid_file(video_details, result): #lets not transcode menu or other clips that don't have audio and video. + rem_vid.append(vid) + data['files'] = [ f for f in data['files'] if f not in rem_vid ] + new_file = {img: {'name': data['name'], 'files': data['files']}} video_details, result = get_video_details(data['files'][0], img, bitbucket) input_file = '-' file = '-' @@ -518,7 +546,7 @@ def build_commands(file, new_dir, movie_name, bitbucket): command.append(newfile_path) if platform.system() != 'Windows': command = core.NICENESS + command - return command + return command, new_file def get_subs(file): @@ -597,6 +625,7 @@ def process_list(it, new_dir, bitbucket): new_list = [] combine = [] vts_path = None + mts_path = None success = True for item in it: ext = os.path.splitext(item)[1].lower() @@ -612,6 +641,14 @@ def process_list(it, new_dir, bitbucket): except Exception: vts_path = os.path.split(item)[0] rem_list.append(item) + elif re.match('.+BDMV[/\\]SOURCE[/\\][0-9]+[0-9].[Mm][Tt][Ss]', item) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(item), 'TRANSCODER') + if not mts_path: + try: + mts_path = re.match('(.+BDMV[/\\]SOURCE)', item).groups()[0] + except Exception: + mts_path = os.path.split(item)[0] + rem_list.append(item) elif re.match('.+VIDEO_TS.', item) or re.match('.+VTS_[0-9][0-9]_[0-9].', item): rem_list.append(item) elif core.CONCAT and re.match('.+[cC][dD][0-9].', item): @@ -621,6 +658,8 @@ def process_list(it, new_dir, bitbucket): continue if vts_path: new_list.extend(combine_vts(vts_path)) + if mts_path: + new_list.extend(combine_mts(mts_path)) if combine: new_list.extend(combine_cd(combine)) for file in new_list: @@ -639,17 +678,53 @@ def process_list(it, new_dir, bitbucket): return it, rem_list, new_list, success +def mount_iso(item, new_dir, bitbucket): #Currently only supports Linux Mount when permissions allow. + if platform.system() == 'Windows': + logger.error('No mounting options available under Windows for image file {0}'.format(item), 'TRANSCODER') + return [] + mount_point = os.path.join(os.path.dirname(os.path.abspath(item)),'temp') + make_dir(mount_point) + cmd = ['mount', '-o', 'loop', item, mount_point] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + core.MOUNTED = mount_point # Allows us to verify this has been done and then cleanup. + for root, dirs, files in os.walk(mount_point): + for file in files: + full_path = os.path.join(root, file) + if re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', full_path) and '.vob' not in core.IGNOREEXTENSIONS: + logger.debug('Found VIDEO_TS image file: {0}'.format(full_path), 'TRANSCODER') + try: + vts_path = re.match('(.+VIDEO_TS)', full_path).groups()[0] + except Exception: + vts_path = os.path.split(full_path)[0] + return combine_vts(vts_path) + elif re.match('.+BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]', full_path) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(full_path), 'TRANSCODER') + try: + mts_path = re.match('(.+BDMV[/\\]STREAM)', full_path).groups()[0] + except Exception: + mts_path = os.path.split(full_path)[0] + return combine_mts(mts_path) + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file {0}'.format(mount_point), 'TRANSCODER') + return ['failure'] # If we got here, nothing matched our criteria + + def rip_iso(item, new_dir, bitbucket): new_files = [] failure_dir = 'failure' # Mount the ISO in your OS and call combineVTS. if not core.SEVENZIP: - logger.error('No 7zip installed. Can\'t extract image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.debug('No 7zip installed. Attempting to mount image file {0}'.format(item), 'TRANSCODER') + try: + new_files = mount_iso(item, new_dir, bitbucket) # Currently only works for Linux. + except Exception: + logger.error('Failed to mount and extract from image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] return new_files cmd = [core.SEVENZIP, 'l', item] try: - logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') + logger.debug('Attempting to extract .vob or .mts from image file {0}'.format(item), 'TRANSCODER') print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() @@ -663,31 +738,58 @@ def rip_iso(item, new_dir, bitbucket): if file_match ] combined = [] - for n in range(99): - concat = [] - m = 1 - while True: - vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) - if vts_name in file_list: - concat.append(vts_name) - m += 1 - else: + if file_list: # handle DVD + for n in range(99): + concat = [] + m = 1 + while True: + vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) + if vts_name in file_list: + concat.append(vts_name) + m += 1 + else: + break + if not concat: break - if not concat: - break - if core.CONCAT: - combined.extend(concat) - continue - name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + ) + new_files.append({item: {'name': name, 'files': concat}}) + else: #check BlueRay for BDMV/STREAM/XXXX.MTS + mts_list_gen = ( + re.match(r'.+(BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]).', line) + for line in out.decode().splitlines() ) - new_files.append({item: {'name': name, 'files': concat}}) - if core.CONCAT: + mts_list = [ + file_match.groups()[0] + for file_match in mts_list_gen + if file_match + ] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) # Sort all .mts files in numerical order + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + n = 0 + for mts_name in mts_list: + concat = [] + n += 1 + concat.append(mts_name) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + ) + new_files.append({item: {'name': name, 'files': concat}}) + if core.CONCAT and combined: name = os.path.splitext(os.path.split(item)[1])[0] new_files.append({item: {'name': name, 'files': combined}}) if not new_files: - logger.error('No VIDEO_TS folder found in image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {0}'.format(item), 'TRANSCODER') + new_files = mount_iso(item, new_dir, bitbucket) except Exception: logger.error('Failed to extract from image file {0}'.format(item), 'TRANSCODER') new_files = [failure_dir] @@ -696,25 +798,63 @@ def rip_iso(item, new_dir, bitbucket): def combine_vts(vts_path): new_files = [] - combined = '' + combined = [] + name = re.match(r'(.+)[/\\]VIDEO_TS', vts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) for n in range(99): - concat = '' + concat = [] m = 1 while True: vts_name = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m) if os.path.isfile(os.path.join(vts_path, vts_name)): - concat += '{file}|'.format(file=os.path.join(vts_path, vts_name)) + concat.append(os.path.join(vts_path, vts_name)) m += 1 else: break if not concat: break if core.CONCAT: - combined += '{files}|'.format(files=concat) + combined.extend(concat) continue - new_files.append('concat:{0}'.format(concat[:-1])) + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({vts_path: {'name': name, 'files': concat}}) if core.CONCAT: - new_files.append('concat:{0}'.format(combined[:-1])) + new_files.append({vts_path: {'name': name, 'files': combined}}) + return new_files + + +def combine_mts(mts_path): + new_files = [] + combined = [] + name = re.match(r'(.+)[/\\]BDMV[/\\]STREAM', mts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) + n = 0 + mts_list = [f for f in os.listdir(mts_path) if os.path.isfile(os.path.join(mts_path, f))] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + for mts_name in mts_list: ### need to sort all files [1 - 998].mts in order + concat = [] + concat.append(os.path.join(mts_path, mts_name)) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({mts_path: {'name': name, 'files': concat}}) + n += 1 + if core.CONCAT: + new_files.append({mts_path: {'name': name, 'files': combined}}) return new_files @@ -768,7 +908,7 @@ def transcode_directory(dir_name): for file in file_list: if isinstance(file, string_types) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: continue - command = build_commands(file, new_dir, movie_name, bitbucket) + command, file = build_commands(file, new_dir, movie_name, bitbucket) newfile_path = command[-1] # transcoding files may remove the original file, so make sure to extract subtitles first @@ -795,6 +935,7 @@ def transcode_directory(dir_name): for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: + logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() proc.communicate() @@ -826,6 +967,15 @@ def transcode_directory(dir_name): logger.error('Transcoding of video to {0} failed with result {1}'.format(newfile_path, result)) # this will be 0 (successful) it all are successful, else will return a positive integer for failure. final_result = final_result + result + if core.MOUNTED: # In case we mounted an .iso file, unmount here. + time.sleep(5) # play it safe and avoid failing to unmount. + cmd = ['umount', '-l', core.MOUNTED] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + time.sleep(5) + os.rmdir(core.MOUNTED) + core.MOUNTED = None if final_result == 0 and not core.DUPLICATE: for file in rem_list: try: diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 16785b46..862e7f78 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -202,7 +202,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToLidarr.py b/nzbToLidarr.py index 9681a408..c38f5975 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -187,7 +187,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToMedia.py b/nzbToMedia.py index 03f19a8d..12a72ae6 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -558,7 +558,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index fbc3de0c..680074cd 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -192,7 +192,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToRadarr.py b/nzbToRadarr.py index 504bbe52..f75ca350 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -197,7 +197,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 43d23d3d..00f18077 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -203,7 +203,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. From fd1149aea1e0b1e524d821900d586c05bc9ce639 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 6 Jun 2019 21:46:56 +1200 Subject: [PATCH 74/87] add additional options to pass into ffmpeg. #1619 --- autoProcessMedia.cfg.spec | 4 +++- core/__init__.py | 7 +++++++ core/transcoder.py | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 6829d6b2..7969831d 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -414,8 +414,10 @@ externalSubDir = # hwAccel. 1 will set ffmpeg to enable hardware acceleration (this requires a recent ffmpeg) hwAccel = 0 - # generalOptions. Enter your additional ffmpeg options here with commas to separate each option/value (i.e replace spaces with commas). + # generalOptions. Enter your additional ffmpeg options (these insert before the '-i' input files) here with commas to separate each option/value (i.e replace spaces with commas). generalOptions = + # otherOptions. Enter your additional ffmpeg options (these insert after the '-i' input files and before the output file) here with commas to separate each option/value (i.e replace spaces with commas). + otherOptions = # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, leave this blank and set the remaining options below. # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release diff --git a/core/__init__.py b/core/__init__.py index 7fb32140..b3c643fb 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -211,6 +211,7 @@ VEXTENSION = None OUTPUTVIDEOPATH = None PROCESSOUTPUT = False GENERALOPTS = [] +OTHEROPTS = [] ALANGUAGE = None AINCLUDE = False SLANGUAGES = [] @@ -513,6 +514,7 @@ def configure_transcoder(): global IGNOREEXTENSIONS global OUTPUTFASTSTART global GENERALOPTS + global OTHEROPTS global OUTPUTQUALITYPERCENT global OUTPUTVIDEOPATH global PROCESSOUTPUT @@ -568,6 +570,11 @@ def configure_transcoder(): GENERALOPTS.append('-fflags') if '+genpts' not in GENERALOPTS: GENERALOPTS.append('+genpts') + OTHEROPTS = (CFG['Transcoder']['otherOptions']) + if isinstance(OTHEROPTS, str): + OTHEROPTS = OTHEROPTS.split(',') + if OTHEROPTS == ['']: + OTHEROPTS = [] try: OUTPUTQUALITYPERCENT = int(CFG['Transcoder']['outputQualityPercent']) except Exception: diff --git a/core/transcoder.py b/core/transcoder.py index e1850de2..fcbeb43f 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -493,6 +493,8 @@ def build_commands(file, new_dir, movie_name, bitbucket): if core.OUTPUTFASTSTART: other_cmd.extend(['-movflags', '+faststart']) + if core.OTHEROPTS: + other_cmd.extend(core.OTHEROPTS) command = [core.FFMPEG, '-loglevel', 'warning'] From 323733677572579bcb2e2122d8f5672e78b9516d Mon Sep 17 00:00:00 2001 From: TheHolyRoger <39387497+TheHolyRoger@users.noreply.github.com> Date: Sat, 8 Jun 2019 00:20:12 +0100 Subject: [PATCH 75/87] Don't replace apostrophes in qBittorrent input_name Don't replace apostrophes in qBittorrent input_name - only trim if found at beginning/end of string. This stops nzbtomedia processing the entire download folder when asked to process a folder with apostrophes in the title --- core/utils/parsers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/utils/parsers.py b/core/utils/parsers.py index 981de146..fd88ec63 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -127,7 +127,11 @@ def parse_qbittorrent(args): except Exception: input_directory = '' try: - input_name = cur_input[1].replace('\'', '') + input_name = cur_input[1] + if input_name[0] == '\'': + input_name = input_name[1:] + if input_name[-1] == '\'': + input_name = input_name[:-1] except Exception: input_name = '' try: From d39a7dd23497943dd853d585e7976cdb46fc7877 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 18 Jun 2019 20:52:19 +1200 Subject: [PATCH 76/87] fix to make deluge client py 2 and 3 compatible. Fixes #1626 --- libs/custom/synchronousdeluge/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 191c70f6..0418e806 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -82,13 +82,13 @@ class DelugeClient(object): def func(obj, *args, **kwargs): return self.remote_call(fullname, *args, **kwargs) - func.__name__ = method + func.__name__ = str(method) return func def _introspect(self): def splitter(value): - return value.split('.') + return value.split(b'.') self.modules = [] @@ -100,8 +100,8 @@ class DelugeClient(object): for module, methods in methodmap.items(): clsname = 'DelugeModule{0}'.format(module.capitalize()) - cls = type(clsname, (), methods) - setattr(self, module, cls()) + cls = type(str(clsname), (), methods) + setattr(self, str(module), cls()) self.modules.append(module) def remote_call(self, method, *args, **kwargs): From f1dc6720564ab7c4d255fdf54c87a164e7eb6dcf Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 19 Jun 2019 22:50:36 +1200 Subject: [PATCH 77/87] fix deluge client for python3. Fixes #1626 --- libs/custom/synchronousdeluge/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 0418e806..f446cede 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -88,11 +88,12 @@ class DelugeClient(object): def _introspect(self): def splitter(value): - return value.split(b'.') + return value.split('.') self.modules = [] methods = self.remote_call('daemon.get_method_list').get() + methods = (x.decode() for x in methods) methodmap = defaultdict(dict) for module, method in imap(splitter, methods): From ce50a1c27da5ebe3b38adfdc363eda5f7254941f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 19 Jun 2019 21:37:42 +0000 Subject: [PATCH 78/87] Fix allready running handling for Python3. #1626 --- core/utils/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/processes.py b/core/utils/processes.py index 23bb7a92..56882757 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -54,7 +54,7 @@ class PosixProcess(object): self.lasterror = False return self.lasterror except socket.error as e: - if 'Address already in use' in e: + if 'Address already in use' in str(e): self.lasterror = True return self.lasterror except AttributeError: From 9f6c068cdec1a31943a8f1d785f84d1476ebb749 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 20 Jun 2019 12:56:02 +1200 Subject: [PATCH 79/87] Transcode patch 1 (#1627) * Add Piping of stderr to capture transcoding failures. #1619 * Allow passing absolute nice command. #1619 * Change .cfg description for niceness * Fix errors due to VM packages out of date (ffmpeg) * Fix Sqlite import error on tests * Fix Azure issues https://developercommunity.visualstudio.com/content/problem/598264/known-issue-azure-pipelines-images-missing-sqlite3.html --- autoProcessMedia.cfg.spec | 4 +++- azure-pipelines.yml | 44 +++++++++++++++++++++++++++++++++++---- core/__init__.py | 5 ++++- core/transcoder.py | 10 +++++---- nzbToCouchPotato.py | 6 ++++-- nzbToGamez.py | 6 ++++-- nzbToHeadPhones.py | 6 ++++-- nzbToLazyLibrarian.py | 6 ++++-- nzbToLidarr.py | 6 ++++-- nzbToMedia.py | 6 ++++-- nzbToMylar.py | 6 ++++-- nzbToNzbDrone.py | 6 ++++-- nzbToRadarr.py | 6 ++++-- nzbToSickBeard.py | 6 ++++-- 14 files changed, 93 insertions(+), 30 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 7969831d..e3d11cb6 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -36,7 +36,9 @@ [Posix] ### Process priority setting for External commands (Extractor and Transcoder) on Posix (Unix/Linux/OSX) systems. # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). - niceness = 0 + # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). + # If entering a comma separated list e.g. 'niceness = nice,4' this will be passed as 'nice 4' (Safer). + niceness = nice,-n0 # Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. ionice_class = 0 # Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 96321a5b..bc64c2d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,42 @@ jobs: maxParallel: 4 steps: + - script: | + # Make sure all packages are pulled from latest + sudo apt-get update + + # Fail out if any setups fail + set -e + + # Delete old Pythons + rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + + # Download new Pythons + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + + # Install new Pythons + original_directory=$PWD + setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + for setup in $setups; do + chmod +x $setup; + cd $(dirname $setup); + ./$(basename $setup); + cd $original_directory; + done; + displayName: 'Workaround: update apt and roll back Python versions' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' @@ -41,10 +77,10 @@ jobs: displayName: 'pytest' - script: | - rm -rf .git - python cleanup.py - python TorrentToMedia.py - python nzbToMedia.py + rm -rf .git + python cleanup.py + python TorrentToMedia.py + python nzbToMedia.py displayName: 'Test source install cleanup' - task: PublishTestResults@2 diff --git a/core/__init__.py b/core/__init__.py index b3c643fb..a7a4ce24 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -457,7 +457,10 @@ def configure_niceness(): with open(os.devnull, 'w') as devnull: try: subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) + if len(CFG['Posix']['niceness'].split(',')) > 1: #Allow passing of absolute command, not just value. + NICENESS.extend(CFG['Posix']['niceness'].split(',')) + else: + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) except Exception: pass try: diff --git a/core/transcoder.py b/core/transcoder.py index fcbeb43f..f0da8dd4 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -607,7 +607,7 @@ def extract_subs(file, newfile_path, bitbucket): result = 1 # set result to failed in case call fails. try: proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - proc.communicate() + out, err = proc.communicate() result = proc.returncode except Exception: logger.error('Extracting subtitle has failed') @@ -930,17 +930,19 @@ def transcode_directory(dir_name): result = 1 # set result to failed in case call fails. try: if isinstance(file, string_types): - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE) else: img, data = next(iteritems(file)) - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE) for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() - proc.communicate() + out, err = proc.communicate() + if err: + logger.error('Transcoder returned:{0} has failed'.format(err)) result = proc.returncode except Exception: logger.error('Transcoding of video {0} has failed'.format(newfile_path)) diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 862e7f78..60638200 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -112,8 +112,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToGamez.py b/nzbToGamez.py index 39023421..92237f07 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -68,8 +68,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index a1917f10..d875d151 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -82,8 +82,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index dd760709..ad8d2534 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -76,8 +76,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLidarr.py b/nzbToLidarr.py index c38f5975..da89ab46 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -97,8 +97,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMedia.py b/nzbToMedia.py index 12a72ae6..f2a5a57f 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -468,8 +468,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMylar.py b/nzbToMylar.py index e26cfe6d..cd408dcb 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -73,8 +73,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index 680074cd..b99ed190 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -102,8 +102,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToRadarr.py b/nzbToRadarr.py index f75ca350..a7e02f93 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -107,8 +107,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 00f18077..b6d8b42e 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -113,8 +113,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # From 9a958afac8053c0e579d9094b9c469681245bb5b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 12 Jul 2019 19:39:55 +1200 Subject: [PATCH 80/87] don't crash when no optionalParameters. Fixes #1630 (#1632) --- core/forks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/forks.py b/core/forks.py index c1721aaf..b5e8c840 100644 --- a/core/forks.py +++ b/core/forks.py @@ -114,11 +114,14 @@ def auto_fork(section, input_category): else: json_data = json_data.get('data', json_data) - optional_parameters = json_data['optionalParameters'].keys() - # Find excess parameters - excess_parameters = set(params).difference(optional_parameters) - logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) - rem_params.extend(excess_parameters) + try: + optional_parameters = json_data['optionalParameters'].keys() + # Find excess parameters + excess_parameters = set(params).difference(optional_parameters) + logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) + rem_params.extend(excess_parameters) + except: + logger.error('Failed to identify optionalParameters') else: # Find excess parameters rem_params.extend( From f21e18b1bfd07dc2a4e20ff66fb020034340572d Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 16 Jul 2019 14:36:00 +1200 Subject: [PATCH 81/87] Fix2 (#1636) add chnaged api handling for SickGear. Fixes #1630 --- core/forks.py | 63 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/core/forks.py b/core/forks.py index b5e8c840..97be31da 100644 --- a/core/forks.py +++ b/core/forks.py @@ -13,6 +13,34 @@ from six import iteritems import core from core import logger +def api_check(r, params, rem_params): + try: + json_data = r.json() + except ValueError: + logger.error('Failed to get JSON data from response') + logger.debug('Response received') + raise + + try: + json_data = json_data['data'] + except KeyError: + logger.error('Failed to get data from JSON') + logger.debug('Response received: {}'.format(json_data)) + raise + else: + json_data = json_data.get('data', json_data) + + try: + optional_parameters = json_data['optionalParameters'].keys() + # Find excess parameters + excess_parameters = set(params).difference(optional_parameters) + logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) + rem_params.extend(excess_parameters) + return rem_params, True + except: + logger.error('Failed to identify optionalParameters') + return rem_params, False + def auto_fork(section, input_category): # auto-detect correct section @@ -98,30 +126,17 @@ def auto_fork(section, input_category): r = [] if r and r.ok: if apikey: - try: - json_data = r.json() - except ValueError: - logger.error('Failed to get JSON data from response') - logger.debug('Response received') - raise - - try: - json_data = json_data['data'] - except KeyError: - logger.error('Failed to get data from JSON') - logger.debug('Response received: {}'.format(json_data)) - raise - else: - json_data = json_data.get('data', json_data) - - try: - optional_parameters = json_data['optionalParameters'].keys() - # Find excess parameters - excess_parameters = set(params).difference(optional_parameters) - logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) - rem_params.extend(excess_parameters) - except: - logger.error('Failed to identify optionalParameters') + rem_params, found = api_check(r, params, rem_params) + if not found: # try different api set for SickGear. + url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=postprocess&help=1'.format( + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey, + ) + try: + r = s.get(url, auth=(username, password), verify=False) + except requests.ConnectionError: + logger.info('Could not connect to {section}:{category} to perform auto-fork detection!'.format + (section=section, category=input_category)) + rem_params, found = api_check(r, params, rem_params) else: # Find excess parameters rem_params.extend( From 8ba8caf02155e311dff6137129fe46a48209f64a Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 22 Jul 2019 12:35:01 +1200 Subject: [PATCH 82/87] Fix3 (#1637) * add singular fork detection for multiple runs. Fixes #1637 * Add newly identified fork variants #1630 #1637 * remove encoding of paths. #1637 #1582 --- TorrentToMedia.py | 24 ------------------------ core/__init__.py | 5 +++++ core/forks.py | 5 +++++ core/utils/identification.py | 9 --------- core/utils/naming.py | 4 ---- nzbToMedia.py | 8 -------- 6 files changed, 10 insertions(+), 45 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 51a257c8..42707b1a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -70,14 +70,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass - # try: - # input_directory = input_directory.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format (input_directory, input_name, input_category)) @@ -135,10 +127,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - # try: - # output_destination = output_destination.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass if output_destination in input_directory: output_destination = input_directory @@ -180,10 +168,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - # try: - # target_file = target_file.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass if root == 1: if not found_file: logger.debug('Looking for {0} in: {1}'.format(input_name, inputFile)) @@ -362,15 +346,7 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - # try: - # dir_name = dir_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass input_name = os.path.basename(dir_name) - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) diff --git a/core/__init__.py b/core/__init__.py index a7a4ce24..e2436182 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -99,10 +99,12 @@ FORK_FAILED = 'failed' FORK_FAILED_TORRENT = 'failed-torrent' FORK_SICKRAGE = 'SickRage' FORK_SICKCHILL = 'SickChill' +FORK_SICKCHILL_API = 'SickChill-api' FORK_SICKBEARD_API = 'SickBeard-api' FORK_MEDUSA = 'Medusa' FORK_MEDUSA_API = 'Medusa-api' FORK_SICKGEAR = 'SickGear' +FORK_SICKGEAR_API = 'SickGear-api' FORK_STHENO = 'Stheno' FORKS = { @@ -111,10 +113,12 @@ FORKS = { FORK_FAILED_TORRENT: {'dir': None, 'failed': None, 'process_method': None}, FORK_SICKRAGE: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None}, FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, + FORK_SICKCHILL_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'is_priority': None}, FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, + FORK_SICKGEAR_API: {'path': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'is priority': None}, FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} @@ -198,6 +202,7 @@ META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] +FORK_SET = [] MOUNTED = None GETSUBS = False diff --git a/core/forks.py b/core/forks.py index 97be31da..f49ed5a9 100644 --- a/core/forks.py +++ b/core/forks.py @@ -45,6 +45,10 @@ def api_check(r, params, rem_params): def auto_fork(section, input_category): # auto-detect correct section # config settings + if core.FORK_SET: # keep using determined fork for multiple (manual) post-processing + logger.info('{section}:{category} fork already set to {fork}'.format + (section=section, category=input_category, fork=core.FORK_SET[0])) + return core.FORK_SET[0], core.FORK_SET[1] cfg = dict(core.CFG[section][input_category]) @@ -168,4 +172,5 @@ def auto_fork(section, input_category): logger.info('{section}:{category} fork set to {fork}'.format (section=section, category=input_category, fork=fork[0])) + core.FORK_SET = fork return fork[0], fork[1] diff --git a/core/utils/identification.py b/core/utils/identification.py index 365ea790..7ea2fdc1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -96,15 +96,6 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except Exception: - # pass - # try: - # input_directory = input_directory.encode(core.SYS_ENCODING) - # except Exception: - # pass - if input_directory is None: # =Nothing to process here. return input_directory, input_name, input_category, root diff --git a/core/utils/naming.py b/core/utils/naming.py index 4d664036..dfabec3f 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -26,10 +26,6 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - # try: - # name = name.encode(core.SYS_ENCODING) - # except Exception: - # pass return name diff --git a/nzbToMedia.py b/nzbToMedia.py index f2a5a57f..a6139fe3 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -973,15 +973,7 @@ def main(args, section=None): if client_agent and client_agent.lower() not in core.NZB_CLIENTS: continue - try: - dir_name = dir_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass input_name = os.path.basename(dir_name) - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass results = process(dir_name, input_name, 0, client_agent=client_agent, download_id=download_id or None, input_category=subsection) From 5a18ee9a27ba9fabfab44a90e3320071c54d06f3 Mon Sep 17 00:00:00 2001 From: currently-off-my-rocker Date: Mon, 22 Jul 2019 13:07:09 +0200 Subject: [PATCH 83/87] identify imdb ids with 8 digits --- core/utils/identification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 7ea2fdc1..621907a1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -23,14 +23,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): # find imdbid in dirName logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7})', dir_name + input_name) + m = re.search(r'(tt\d{7,8})', dir_name + input_name) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}]'.format(imdbid)) return imdbid if os.path.isdir(dir_name): for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7})', file) + m = re.search(r'(tt\d{7,8})', file) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}] via file name'.format(imdbid)) From d7eab5d2d39575d006d23b06f6f12977b9388cca Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 23 Jul 2019 14:24:31 +1200 Subject: [PATCH 84/87] Add word boundary to imdb match. #1639 Prevents matching (and truncating) longer ids. Thanks @currently-off-my-rocker --- core/utils/identification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 621907a1..84fa834a 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -23,14 +23,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): # find imdbid in dirName logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7,8})', dir_name + input_name) + m = re.search(r'\b(tt\d{7,8})\b', dir_name + input_name) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}]'.format(imdbid)) return imdbid if os.path.isdir(dir_name): for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7,8})', file) + m = re.search(r'\b(tt\d{7,8})\b', file) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}] via file name'.format(imdbid)) From 5714540949b183372b038d8ebf05911f2db93932 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 2 Aug 2019 13:02:46 +1200 Subject: [PATCH 85/87] Fix uTorrent with Python3 (#1644) * Remove temp workaround for Microsoft Azure python issues. --- azure-pipelines.yml | 50 +++++++++++++++++----------------- libs/custom/utorrent/client.py | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc64c2d4..db2d4f2b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,42 +24,42 @@ jobs: maxParallel: 4 steps: - - script: | + #- script: | # Make sure all packages are pulled from latest - sudo apt-get update + #sudo apt-get update # Fail out if any setups fail - set -e + #set -e # Delete old Pythons - rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 - rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 - rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 # Download new Pythons - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ - --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ - --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ - --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 # Install new Pythons - original_directory=$PWD - setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) - for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; - done; - displayName: 'Workaround: update apt and roll back Python versions' - + #original_directory=$PWD + #setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + #for setup in $setups; do + #chmod +x $setup; + #cd $(dirname $setup); + #./$(basename $setup); + #cd $original_directory; + #done; + #displayName: 'Workaround: update apt and roll back Python versions' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index f5eaf93b..ac1a9beb 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -59,7 +59,7 @@ class UTorrentClient(object): url = urljoin(self.base_url, 'token.html') response = self.opener.open(url) token_re = "" - match = re.search(token_re, response.read()) + match = re.search(token_re, str(response.read())) return match.group(1) def list(self, **kwargs): From bde5a15f660c82eb2b827710042b02158078fae6 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 6 Aug 2019 09:04:45 +1200 Subject: [PATCH 86/87] Fixes for user_script categories (#1645) Fixes for user_script categories. #1643 --- TorrentToMedia.py | 25 ++++++++++++++----------- core/user_scripts.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 42707b1a..39b08329 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -76,16 +76,19 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # auto-detect section section = core.CFG.findsection(input_category).isenabled() - if section is None: - section = core.CFG.findsection('ALL').isenabled() - if section is None: - logger.error('Category:[{0}] is not defined or is not enabled. ' - 'Please rename it or ensure it is enabled for the appropriate section ' - 'in your autoProcessMedia.cfg and try again.'.format - (input_category)) - return [-1, ''] + if section is None: #Check for user_scripts for 'ALL' and 'UNCAT' + if usercat in core.CATEGORIES: + section = core.CFG.findsection('ALL').isenabled() + usercat = 'ALL' else: - usercat = 'ALL' + section = core.CFG.findsection('UNCAT').isenabled() + usercat = 'UNCAT' + if section is None: # We haven't found any categories to process. + logger.error('Category:[{0}] is not defined or is not enabled. ' + 'Please rename it or ensure it is enabled for the appropriate section ' + 'in your autoProcessMedia.cfg and try again.'.format + (input_category)) + return [-1, ''] if len(section) > 1: logger.error('Category:[{0}] is not unique, {1} are using it. ' @@ -108,7 +111,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp torrent_no_link = int(section.get('Torrent_NoLink', 0)) keep_archive = int(section.get('keep_archive', 0)) extract = int(section.get('extract', 0)) - extensions = section.get('user_script_mediaExtensions', '').lower().split(',') + extensions = section.get('user_script_mediaExtensions', '') unique_path = int(section.get('unique_path', 1)) if client_agent != 'manual': @@ -278,7 +281,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp replace_links(os.path.join(dirpath, file)) core.remove_torrent(client_agent, input_hash, input_id, input_name) - if not section_name == 'UserScript': + if section_name != 'UserScript': # for user script, we assume this is cleaned by the script or option USER_SCRIPT_CLEAN # cleanup our processing folders of any misc unwanted files and empty directories core.clean_dir(output_destination, section_name, input_category) diff --git a/core/user_scripts.py b/core/user_scripts.py index 45adb45b..eaeb1b0f 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -14,33 +14,46 @@ import core from core import logger, transcoder from core.plugins.subtitles import import_subs from core.utils import list_media_files, remove_dir +from core.auto_process.common import ( + ProcessResult, +) + def external_script(output_destination, torrent_name, torrent_label, settings): final_result = 0 # start at 0. num_files = 0 + core.USER_SCRIPT_MEDIAEXTENSIONS = settings.get('user_script_mediaExtensions', '') try: - core.USER_SCRIPT_MEDIAEXTENSIONS = settings['user_script_mediaExtensions'].lower() if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): - core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',') + core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.lower().split(',') except Exception: + logger.error('user_script_mediaExtensions could not be set', 'USERSCRIPT') core.USER_SCRIPT_MEDIAEXTENSIONS = [] - core.USER_SCRIPT = settings.get('user_script_path') + core.USER_SCRIPT = settings.get('user_script_path', '') - if not core.USER_SCRIPT or core.USER_SCRIPT == 'None': # do nothing and return success. - return [0, ''] + if not core.USER_SCRIPT or core.USER_SCRIPT == 'None': + # do nothing and return success. This allows the user an option to Link files only and not run a script. + return ProcessResult( + status_code=0, + message='No user script defined', + ) + + core.USER_SCRIPT_PARAM = settings.get('user_script_param', '') try: - core.USER_SCRIPT_PARAM = settings['user_script_param'] if isinstance(core.USER_SCRIPT_PARAM, str): core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',') except Exception: + logger.error('user_script_params could not be set', 'USERSCRIPT') core.USER_SCRIPT_PARAM = [] + + core.USER_SCRIPT_SUCCESSCODES = settings.get('user_script_successCodes', 0) try: - core.USER_SCRIPT_SUCCESSCODES = settings['user_script_successCodes'] if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',') except Exception: + logger.error('user_script_successCodes could not be set', 'USERSCRIPT') core.USER_SCRIPT_SUCCESSCODES = 0 core.USER_SCRIPT_CLEAN = int(settings.get('user_script_clean', 1)) @@ -59,6 +72,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): file_path = core.os.path.join(dirpath, file) file_name, file_extension = os.path.splitext(file) + logger.debug('Checking file {0} to see if this should be processed.'.format(file), 'USERSCRIPT') if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or 'all' in core.USER_SCRIPT_MEDIAEXTENSIONS: num_files += 1 @@ -122,4 +136,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0: logger.info('{0} files were processed, but {1} still remain. outputDirectory will not be cleaned.'.format( num_files, num_files_new)) - return [final_result, ''] + return ProcessResult( + status_code=final_result, + message='User Script Completed', + ) From dc5d43b028e28aa025b57a2d63bc3009d4c6d0a4 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 6 Aug 2019 13:16:25 +1200 Subject: [PATCH 87/87] update to version 12.1.00 --- .bumpversion.cfg | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 31885a91..cc1a1057 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.10 +current_version = 12.1.00 commit = True tag = False diff --git a/core/__init__.py b/core/__init__.py index e2436182..55b8acba 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -83,7 +83,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.10' +__version__ = '12.1.00' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index bac19586..a115dfa2 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.10', + version='12.1.00', license='GPLv3', description='Efficient on demand post processing', long_description="""