From e386eaaec2e421e0fcdab5002756c28139deb454 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 00:46:37 -0500 Subject: [PATCH 001/406] =?UTF-8?q?Bump=20version:=2012.0.1=20=E2=86=92=20?= =?UTF-8?q?12.0.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0567b58c..e5b82b40 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.1 +current_version = 12.0.2 commit = True tag = False diff --git a/README.md b/README.md index 7528d0d3..db6b38a4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.1 +nzbToMedia v12.0.2 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 723b52f3..39611d06 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -52,7 +52,7 @@ from core.utils import ( resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -__version__ = '12.0.1' +__version__ = '12.0.2' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 9f19320a..1307c1f4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.1', + version='12.0.2', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 6d7dacf11455e462070653429726c32539dead77 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 5 Jan 2019 23:30:11 +1300 Subject: [PATCH 002/406] update a Readme to reflect recent chnages. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db6b38a4..21e1fe38 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficien when using one of the popular NZB download clients like [SABnzbd](http://sabnzbd.org/ "SABnzbd") and [NZBGet](http://nzbget.sourceforge.net/ "NZBGet") on low performance systems like a NAS. This script is based on sabToSickBeard (written by Nic Wolfe and supplied with SickBeard), with the support for NZBGet being added by [thorli](https://github.com/thorli "thorli") and further contributions by [schumi2004](https://github.com/schumi2004 "schumi2004") and [hugbug](https://sourceforge.net/apps/phpbb/nzbget/memberlist.php?mode=viewprofile&u=67 "hugbug"). Torrent suport added by [jkaberg](https://github.com/jkaberg "jkaberg") and [berkona](https://github.com/berkona "berkona") -Corrupt video checking, auto SickBeard fork determination and a whole lot of code improvement was done by [echel0n](https://github.com/echel0n "echel0n") +Corrupt video checking, auto SickBeard fork determination and a whole lot of code improvement was done by [Labrys of Knossos](https://github.com/echel0n "Labrys of Knossos") +Python3 compatibility, and much cleaner code base has been contributed by [echel0n](https://github.com/labrys "echel0n") + Introduction ------------ @@ -17,7 +19,7 @@ Failed download handling for SickBeard is available by using Tolstyak's fork [Si To use this feature, in autoProcessTV.cfg set the parameter "fork=failed". Default is "fork=default" and will work with the standard version of SickBeard and just ignores failed downloads. Development of Tolstyak's fork ended in 2013, but newer forks exist with significant feature updates such as [Mr-Orange TPB](https://github.com/coach0742/Sick-Beard) (discontinued), [SickRageTV](https://github.com/SiCKRAGETV/SickRage) and [SickRage](https://github.com/SickRage/SickRage) (active). See [SickBeard Forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks "SickBeard Forks") for a list of known forks. -Full support is provided for [SickRageTV](https://github.com/SiCKRAGETV/SickRage), [SickRage](https://github.com/SickRage/SickRage), and [SickGear](https://github.com/SickGear/SickGear). +Full support is provided for [SickChill](https://github.com/SickChill/SickChill), [SiCKRAGE](https://github.com/SiCKRAGE/SiCKRAGE), [Medusa](https://github.com/pymedusa/Medusa), and [SickGear](https://github.com/SickGear/SickGear). Torrent support has been added with the assistance of jkaberg and berkona. Currently supports uTorrent, Transmission, Deluge and possibly more. To enable Torrent extraction, on Windows, you need to install [7-zip](http://www.7-zip.org/ "7-zip") or on *nix you need to install the following packages/commands. From f1c4c6e84092a2c19e77b629a3aa40ccd2302014 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 5 Jan 2019 23:33:30 +1300 Subject: [PATCH 003/406] and that is why we don't make chnages using vi while on holiday! --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21e1fe38..f86cbb93 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficien when using one of the popular NZB download clients like [SABnzbd](http://sabnzbd.org/ "SABnzbd") and [NZBGet](http://nzbget.sourceforge.net/ "NZBGet") on low performance systems like a NAS. This script is based on sabToSickBeard (written by Nic Wolfe and supplied with SickBeard), with the support for NZBGet being added by [thorli](https://github.com/thorli "thorli") and further contributions by [schumi2004](https://github.com/schumi2004 "schumi2004") and [hugbug](https://sourceforge.net/apps/phpbb/nzbget/memberlist.php?mode=viewprofile&u=67 "hugbug"). Torrent suport added by [jkaberg](https://github.com/jkaberg "jkaberg") and [berkona](https://github.com/berkona "berkona") -Corrupt video checking, auto SickBeard fork determination and a whole lot of code improvement was done by [Labrys of Knossos](https://github.com/echel0n "Labrys of Knossos") -Python3 compatibility, and much cleaner code base has been contributed by [echel0n](https://github.com/labrys "echel0n") +Corrupt video checking, auto SickBeard fork determination and a whole lot of code improvement was done by [echel0n](https://github.com/echel0n "echel0n") +Python3 compatibility, and much cleaner code base has been contributed by [Labrys of Knossos](https://github.com/labrys "Labrys of Knossos") Introduction From d250e45c7b4347d58f16dea2ee6f7b0401e3b34d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 21:26:56 -0500 Subject: [PATCH 004/406] Hotfix cleaning for source installs --- cleanup.py | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/cleanup.py b/cleanup.py index 213aca99..8c2d54d2 100644 --- a/cleanup.py +++ b/cleanup.py @@ -5,6 +5,20 @@ from __future__ import print_function import os import subprocess import sys +import shutil + +FOLDER_STRUCTURE = { + 'libs': [ + 'common', + 'custom', + 'py2', + 'win', + ], + 'core': [ + 'auto_process', + 'extractor', + ], +} class WorkingDirectory(object): @@ -106,7 +120,22 @@ def clean_folders(*paths): return result -def clean(*paths): +def force_clean_folder(path, required): + root, dirs, files = next(os.walk(path)) + required = sorted(required) + if required: + print('Skipping required subfolders', required) + remove = sorted(set(dirs).difference(required)) + missing = sorted(set(required).difference(dirs)) + for path in remove: + pathname = os.path.join(root, path) + print('Removing', pathname) + shutil.rmtree(pathname) + if missing: + raise Exception('Required subfolders missing:', missing) + + +def clean(paths): """Clean up bytecode and obsolete folders.""" with WorkingDirectory(module_path()) as cwd: if cwd.working_directory != cwd.original_directory: @@ -121,7 +150,7 @@ def clean(*paths): print(result or 'No bytecode to clean') if paths and os.path.exists('.git'): - print('\n-- Cleaning folders: {} --'.format(paths)) + print('\n-- Cleaning folders: {} --'.format(list(paths))) try: result = clean_folders(*paths) except SystemExit as error: @@ -129,7 +158,15 @@ def clean(*paths): else: print(result or 'No folders to clean\n') else: - print('Directory is not a git repository') + print('\nDirectory is not a git repository') + try: + items = paths.items() + except AttributeError: + print('Failed to clean, no subfolder structure given') + else: + for folder, subfolders in items: + print('\nForce cleaning folder:', folder) + force_clean_folder(folder, subfolders) if cwd.working_directory != cwd.original_directory: print('Returning to directory: ', cwd.original_directory) @@ -138,4 +175,4 @@ def clean(*paths): if __name__ == '__main__': - clean('libs', 'core') + clean(FOLDER_STRUCTURE) From 6616801c382f9c1f5cce0daba9d29966f7830d34 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 21:27:14 -0500 Subject: [PATCH 005/406] =?UTF-8?q?Bump=20version:=2012.0.2=20=E2=86=92=20?= =?UTF-8?q?12.0.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e5b82b40..e11b54a8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.2 +current_version = 12.0.3 commit = True tag = False diff --git a/README.md b/README.md index db6b38a4..67f6c02e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.2 +nzbToMedia v12.0.3 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 39611d06..9fa7c320 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -52,7 +52,7 @@ from core.utils import ( resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -__version__ = '12.0.2' +__version__ = '12.0.3' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 1307c1f4..ccd9786c 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.2', + version='12.0.3', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 29171baaa311972892b3bd75add86c0cf4502074 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 22:34:12 -0500 Subject: [PATCH 006/406] Add extra logging for fork detection. --- core/forks.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/forks.py b/core/forks.py index 8f8fcb26..2e07bd74 100644 --- a/core/forks.py +++ b/core/forks.py @@ -81,9 +81,23 @@ def auto_fork(section, input_category): if apikey: optional_parameters = [] try: - optional_parameters = r.json()['data']['optionalParameters'].keys() - except Exception: - optional_parameters = r.json()['data']['data']['optionalParameters'].keys() + json_data = r.json() + except ValueError: + logger.error('Failed to get JSON data from response') + logger.debug('Response received') + raise + + try: + json_data = json_data['data'] + except KeyError: + logger.error('Failed to get data from JSON') + logger.debug('Response received: {}'.format(json_data)) + raise + else: + json_data = json_data.get('data', json_data) + + optional_parameters = json_data['optionalParameters'].keys() + for param in params: if param not in optional_parameters: rem_params.append(param) From f514eecf6c7462237d9fb0afc66aff220924b12e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 22:37:05 -0500 Subject: [PATCH 007/406] Fix excess parameter detection --- core/forks.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/forks.py b/core/forks.py index 2e07bd74..9aff79e5 100644 --- a/core/forks.py +++ b/core/forks.py @@ -97,14 +97,16 @@ def auto_fork(section, input_category): json_data = json_data.get('data', json_data) optional_parameters = json_data['optionalParameters'].keys() - - for param in params: - if param not in optional_parameters: - rem_params.append(param) + # Find excess parameters + excess_parameters = set(params).difference(optional_parameters) + logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) + rem_params.extend(excess_parameters) else: - for param in params: - if 'name="{param}"'.format(param=param) not in r.text: - rem_params.append(param) + rem_params.extend( + param + for param in params + if 'name="{param}"'.format(param=param) not in r.text + ) for param in rem_params: params.pop(param) for fork in sorted(iteritems(core.FORKS), reverse=False): From 656957f1fc85ce9a1351d40844161408bceaeb9c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 22:42:49 -0500 Subject: [PATCH 008/406] Fix excess parameter detection --- core/forks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/forks.py b/core/forks.py index 9aff79e5..8ff389fe 100644 --- a/core/forks.py +++ b/core/forks.py @@ -102,17 +102,22 @@ def auto_fork(section, input_category): logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) rem_params.extend(excess_parameters) else: + # Find excess parameters rem_params.extend( param for param in params if 'name="{param}"'.format(param=param) not in r.text ) + + # Remove excess params for param in rem_params: params.pop(param) + for fork in sorted(iteritems(core.FORKS), reverse=False): if params == fork[1]: detected = True break + if detected: logger.info('{section}:{category} fork auto-detection successful ...'.format (section=section, category=input_category)) From f83b37d80b212a6b860389d6f0f5921394ab22e6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 22:59:20 -0500 Subject: [PATCH 009/406] Fix missed commit for source cleaner --- core/version_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version_check.py b/core/version_check.py index a414f4b9..8fac2d37 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -81,7 +81,7 @@ class CheckVersion(object): def update(self): if self.updater.need_update(): result = self.updater.update() - cleanup.clean('core', 'libs') + cleanup.clean(cleanup.FOLDER_STRUCTURE) return result From 84a7011973018c8e147951f90be1d9da6befda3e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 22:59:31 -0500 Subject: [PATCH 010/406] =?UTF-8?q?Bump=20version:=2012.0.3=20=E2=86=92=20?= =?UTF-8?q?12.0.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e11b54a8..c26fb34f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.3 +current_version = 12.0.4 commit = True tag = False diff --git a/README.md b/README.md index 67f6c02e..3ddaccd5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.3 +nzbToMedia v12.0.4 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 9fa7c320..43799161 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -52,7 +52,7 @@ from core.utils import ( resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -__version__ = '12.0.3' +__version__ = '12.0.4' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index ccd9786c..e61e2fc3 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.3', + version='12.0.4', license='GPLv3', description='Efficient on demand post processing', long_description=""" From ada78a14f85136b4ad88c0161405d0e6200a2e93 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 00:47:01 -0500 Subject: [PATCH 011/406] hotfix --- TorrentToMedia.py | 2 +- nzbToMedia.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 5536bbab..0e14e60d 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -2,7 +2,7 @@ # coding=utf-8 import cleanup -cleanup.clean('core', 'libs') +cleanup.clean(cleanup.FOLDER_STRUCTURE) import datetime import os diff --git a/nzbToMedia.py b/nzbToMedia.py index 2fc8c80c..fceed9cd 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -624,7 +624,7 @@ from __future__ import print_function import cleanup -cleanup.clean('core', 'libs') +cleanup.clean(cleanup.FOLDER_STRUCTURE) import datetime import os From 0fd570cd85959d2a78e84e615f464706b1a21925 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 00:48:08 -0500 Subject: [PATCH 012/406] =?UTF-8?q?Bump=20version:=2012.0.4=20=E2=86=92=20?= =?UTF-8?q?12.0.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c26fb34f..feb03ca3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.4 +current_version = 12.0.5 commit = True tag = False diff --git a/README.md b/README.md index 3ddaccd5..a9bc945d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.4 +nzbToMedia v12.0.5 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 43799161..b55ea684 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -52,7 +52,7 @@ from core.utils import ( resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -__version__ = '12.0.4' +__version__ = '12.0.5' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index e61e2fc3..e8cb7477 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.4', + version='12.0.5', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 6d0d2d3f7e9e4ccbf7f2681d83ac2bea95cc31ac Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 31 Dec 2018 12:21:16 -0500 Subject: [PATCH 013/406] Use dict literal or comprehension for dict creation --- core/__init__.py | 22 ++++++++++++---------- core/auto_process/movies.py | 9 +++++---- core/forks.py | 12 ++++++++++-- core/main_db.py | 16 ++++++++-------- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index b55ea684..6487e716 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -63,7 +63,6 @@ SABNZB_NO_OF_ARGUMENTS = 8 SABNZB_0717_NO_OF_ARGUMENTS = 9 # sickbeard fork/branch constants -FORKS = {} FORK_DEFAULT = 'default' FORK_FAILED = 'failed' FORK_FAILED_TORRENT = 'failed-torrent' @@ -73,15 +72,18 @@ FORK_SICKBEARD_API = 'SickBeard-api' FORK_MEDUSA = 'Medusa' FORK_SICKGEAR = 'SickGear' FORK_STHENO = 'Stheno' -FORKS[FORK_DEFAULT] = {'dir': None} -FORKS[FORK_FAILED] = {'dirName': None, 'failed': None} -FORKS[FORK_FAILED_TORRENT] = {'dir': None, 'failed': None, 'process_method': None} -FORKS[FORK_SICKRAGE] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None} -FORKS[FORK_SICKCHILL] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None} -FORKS[FORK_SICKBEARD_API] = {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None} -FORKS[FORK_MEDUSA] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None} -FORKS[FORK_SICKGEAR] = {'dir': None, 'failed': None, 'process_method': None, 'force': None} -FORKS[FORK_STHENO] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} + +FORKS = { + FORK_DEFAULT: {'dir': None}, + FORK_FAILED: {'dirName': None, 'failed': None}, + FORK_FAILED_TORRENT: {'dir': None, 'failed': None, 'process_method': None}, + FORK_SICKRAGE: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None}, + FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, + FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, + FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, + FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, + FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} +} ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} # NZBGet Exit Codes diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index f3097b6f..7c7b8f02 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -163,17 +163,18 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', status_code=0, ) - params = {} + params = { + 'media_folder': remote_dir(dir_name) if remote_path else dir_name, + } + if download_id and release_id: params['downloader'] = downloader or client_agent params['download_id'] = download_id - params['media_folder'] = remote_dir(dir_name) if remote_path else dir_name - if section == 'CouchPotato': if method == 'manage': command = 'manage.update' - params = {} + params.clear() else: command = 'renamer.scan' diff --git a/core/forks.py b/core/forks.py index 8ff389fe..65610ad0 100644 --- a/core/forks.py +++ b/core/forks.py @@ -20,8 +20,16 @@ def auto_fork(section, input_category): apikey = cfg.get('apikey') ssl = int(cfg.get('ssl', 0)) web_root = cfg.get('web_root', '') - replace = {'sickrage': 'SickRage', 'sickchill': 'SickChill', 'sickgear': 'SickGear', 'medusa': 'Medusa', 'sickbeard-api': 'SickBeard-api', 'stheno': 'Stheno'} - f1 = replace[cfg.get('fork', 'auto')] if cfg.get('fork', 'auto') in replace else cfg.get('fork', 'auto') + replace = { + 'medusa': 'Medusa', + 'sickbeard-api': 'SickBeard-api', + 'sickgear': 'SickGear', + 'sickchill': 'SickChill', + 'sickrage': 'SickRage', + 'stheno': 'Stheno', + } + _val = cfg.get('fork', 'auto') + f1 = replace.get(_val, _val) try: fork = f1, core.FORKS[f1] except KeyError: diff --git a/core/main_db.py b/core/main_db.py index 6d2b6b95..12c21c7e 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -202,17 +202,17 @@ class DBConnection(object): def table_info(self, table_name): # FIXME ? binding is not supported here, but I cannot find a way to escape a string manually cursor = self.connection.execute('PRAGMA table_info({0})'.format(table_name)) - columns = {} - for column in cursor: - columns[column['name']] = {'type': column['type']} - return columns + return { + column['name']: {'type': column['type']} + for column in cursor + } # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query def _dict_factory(self, cursor, row): - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - return d + return { + col[0]: row[idx] + for idx, col in enumerate(cursor.description) + } def sanity_check_database(connection, sanity_check): From 93ec74f1c738bc5b8a93bdf5e35c5d8c35161ca5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 31 Dec 2018 12:28:15 -0500 Subject: [PATCH 014/406] Fix conditional assignment --- core/transcoder.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/transcoder.py b/core/transcoder.py index 38973a33..e2bfb33d 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -80,10 +80,7 @@ def get_video_details(videofile, img=None, bitbucket=None): file = videofile if not core.FFPROBE: return video_details, result - if 'avprobe' in core.FFPROBE: - print_format = '-of' - else: - print_format = '-print_format' + print_format = '-of' if 'avprobe' in core.FFPROBE else '-print_format' try: if img: videofile = '-' From a289eef88ec47ec8c4e9ac8fe94dbdf0d507ac17 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 31 Dec 2018 12:23:01 -0500 Subject: [PATCH 015/406] Remove unused variable --- core/__init__.py | 1 - core/auto_process/movies.py | 1 - core/auto_process/music.py | 6 ------ core/forks.py | 1 - core/scene_exceptions.py | 1 - 5 files changed, 10 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 6487e716..68d4a95c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -745,7 +745,6 @@ def initialize(section=None): if codec in codec_alias: extra = [item for item in codec_alias[codec] if item not in ACODEC3_ALLOW] ACODEC3_ALLOW.extend(extra) - codec_alias = {} # clear memory PASSWORDSFILE = CFG['passwords']['PassWordFile'] diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 7c7b8f02..0c2f5880 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -221,7 +221,6 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', res = json.loads(r.content) scan_id = int(res['id']) logger.debug('Scan started with id: {0}'.format(scan_id), section) - started = True except Exception as e: logger.warning('No scan id was returned due to: {0}'.format(e), section) scan_id = None diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 9c5f7048..58e63714 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -117,18 +117,12 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', status_code=1, ) - success = False - queued = False - started = False try: res = json.loads(r.content) scan_id = int(res['id']) logger.debug('Scan started with id: {0}'.format(scan_id), section) - started = True except Exception as e: logger.warning('No scan id was returned due to: {0}'.format(e), section) - scan_id = None - started = False return ProcessResult( message='{0}: Failed to post-process - Unable to start scan'.format(section), status_code=1, diff --git a/core/forks.py b/core/forks.py index 65610ad0..ede32f9c 100644 --- a/core/forks.py +++ b/core/forks.py @@ -87,7 +87,6 @@ def auto_fork(section, input_category): r = [] if r and r.ok: if apikey: - optional_parameters = [] try: json_data = r.json() except ValueError: diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 5d830012..f2c53c45 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -148,7 +148,6 @@ def rename_script(dirname): def par2(dirname): - newlist = [] sofar = 0 parfile = '' objects = [] From 3b670b895bb0ccfe67764719cb25eae42fe5d284 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 13:44:25 -0500 Subject: [PATCH 016/406] Refactor utils module to package --- cleanup.py | 1 + core/{utils.py => utils/__init__.py} | 0 2 files changed, 1 insertion(+) rename core/{utils.py => utils/__init__.py} (100%) diff --git a/cleanup.py b/cleanup.py index 8c2d54d2..02bdf648 100644 --- a/cleanup.py +++ b/cleanup.py @@ -17,6 +17,7 @@ FOLDER_STRUCTURE = { 'core': [ 'auto_process', 'extractor', + 'utils', ], } diff --git a/core/utils.py b/core/utils/__init__.py similarity index 100% rename from core/utils.py rename to core/utils/__init__.py From 21fa4e38965764ebc91e6da4cf1a5e2873e0e2f3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 13:57:46 -0500 Subject: [PATCH 017/406] Refactor utils.*Process -> utils.processes.*Process --- core/utils/__init__.py | 92 +---------------------------------------- core/utils/processes.py | 92 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 91 deletions(-) create mode 100644 core/utils/processes.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 9c8d81cc..c1df352e 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -26,14 +26,7 @@ from utorrent.client import UTorrentClient import core from core import extractor, logger, main_db - -try: - from win32event import CreateMutex - from win32api import CloseHandle, GetLastError - from winerror import ERROR_ALREADY_EXISTS -except ImportError: - if os.name == 'nt': - raise +from core.utils.processes import RunningProcess try: import jaraco @@ -1321,86 +1314,3 @@ def get_download_info(input_name, status): [text_type(input_name), status]) return sql_results - - -class WindowsProcess(object): - def __init__(self): - self.mutex = None - self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' - self.CreateMutex = CreateMutex - self.CloseHandle = CloseHandle - self.GetLastError = GetLastError - self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS - - def alreadyrunning(self): - self.mutex = self.CreateMutex(None, 0, self.mutexname) - self.lasterror = self.GetLastError() - if self.lasterror == self.ERROR_ALREADY_EXISTS: - self.CloseHandle(self.mutex) - return True - else: - return False - - def __del__(self): - if self.mutex: - self.CloseHandle(self.mutex) - - -class PosixProcess(object): - def __init__(self): - self.pidpath = core.PID_FILE - self.lock_socket = None - - def alreadyrunning(self): - try: - self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) - self.lasterror = False - return self.lasterror - except socket.error as e: - if 'Address already in use' in e: - self.lasterror = True - return self.lasterror - except AttributeError: - pass - if os.path.exists(self.pidpath): - # Make sure it is not a 'stale' pidFile - try: - pid = int(open(self.pidpath, 'r').read().strip()) - except Exception: - pid = None - # Check list of running pids, if not running it is stale so overwrite - if isinstance(pid, int): - try: - os.kill(pid, 0) - self.lasterror = True - except OSError: - self.lasterror = False - else: - self.lasterror = False - else: - self.lasterror = False - - if not self.lasterror: - # Write my pid into pidFile to keep multiple copies of program from running - try: - fp = open(self.pidpath, 'w') - fp.write(str(os.getpid())) - fp.close() - except Exception: - pass - - return self.lasterror - - def __del__(self): - if not self.lasterror: - if self.lock_socket: - self.lock_socket.close() - if os.path.isfile(self.pidpath): - os.unlink(self.pidpath) - - -if os.name == 'nt': - RunningProcess = WindowsProcess -else: - RunningProcess = PosixProcess diff --git a/core/utils/processes.py b/core/utils/processes.py new file mode 100644 index 00000000..6513a12c --- /dev/null +++ b/core/utils/processes.py @@ -0,0 +1,92 @@ +import os +import socket + +import core + +if os.name == 'nt': + from win32event import CreateMutex + from win32api import CloseHandle, GetLastError + from winerror import ERROR_ALREADY_EXISTS + + +class WindowsProcess(object): + def __init__(self): + self.mutex = None + self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' + self.CreateMutex = CreateMutex + self.CloseHandle = CloseHandle + self.GetLastError = GetLastError + self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS + + def alreadyrunning(self): + self.mutex = self.CreateMutex(None, 0, self.mutexname) + self.lasterror = self.GetLastError() + if self.lasterror == self.ERROR_ALREADY_EXISTS: + self.CloseHandle(self.mutex) + return True + else: + return False + + def __del__(self): + if self.mutex: + self.CloseHandle(self.mutex) + + +class PosixProcess(object): + def __init__(self): + self.pidpath = core.PID_FILE + self.lock_socket = None + + def alreadyrunning(self): + try: + self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) + self.lasterror = False + return self.lasterror + except socket.error as e: + if 'Address already in use' in e: + self.lasterror = True + return self.lasterror + except AttributeError: + pass + if os.path.exists(self.pidpath): + # Make sure it is not a 'stale' pidFile + try: + pid = int(open(self.pidpath, 'r').read().strip()) + except Exception: + pid = None + # Check list of running pids, if not running it is stale so overwrite + if isinstance(pid, int): + try: + os.kill(pid, 0) + self.lasterror = True + except OSError: + self.lasterror = False + else: + self.lasterror = False + else: + self.lasterror = False + + if not self.lasterror: + # Write my pid into pidFile to keep multiple copies of program from running + try: + fp = open(self.pidpath, 'w') + fp.write(str(os.getpid())) + fp.close() + except Exception: + pass + + return self.lasterror + + def __del__(self): + if not self.lasterror: + if self.lock_socket: + self.lock_socket.close() + if os.path.isfile(self.pidpath): + os.unlink(self.pidpath) + + +if os.name == 'nt': + RunningProcess = WindowsProcess +else: + RunningProcess = PosixProcess From 4143aa77f8f0c6d849ad5ec88e8bbb58e33ef152 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:09:46 -0500 Subject: [PATCH 018/406] Refactor torrents from utils to utils.torrents --- core/utils/__init__.py | 100 +-------------------------------------- core/utils/torrents.py | 104 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 99 deletions(-) create mode 100644 core/utils/torrents.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index c1df352e..500f1498 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -16,17 +16,14 @@ from babelfish import Language import beets import guessit import linktastic -from qbittorrent import Client as qBittorrentClient import requests from six import text_type import subliminal -from synchronousdeluge.client import DelugeClient -from transmissionrpc.client import Client as TransmissionClient -from utorrent.client import UTorrentClient import core from core import extractor, logger, main_db from core.utils.processes import RunningProcess +from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent try: import jaraco @@ -812,101 +809,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def create_torrent_class(client_agent): - # Hardlink solution for Torrents - tc = None - - if client_agent == 'utorrent': - try: - logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) - tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) - except Exception: - logger.error('Failed to connect to uTorrent') - - if client_agent == 'transmission': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format( - client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) - tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, - core.TRANSMISSIONUSR, - core.TRANSMISSIONPWD) - except Exception: - logger.error('Failed to connect to Transmission') - - if client_agent == 'deluge': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) - tc = DelugeClient() - tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, - password=core.DELUGEPWD) - except Exception: - logger.error('Failed to connect to Deluge') - - if client_agent == 'qbittorrent': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) - except Exception: - logger.error('Failed to connect to qBittorrent') - - return tc - - -def pause_torrent(client_agent, input_hash, input_id, input_name): - logger.debug('Stopping torrent {0} in {1} while processing'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.stop(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.stop_torrent(input_id) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.pause_torrent([input_id]) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.pause(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to stop torrent {0} in {1}'.format(input_name, client_agent)) - - -def resume_torrent(client_agent, input_hash, input_id, input_name): - if not core.TORRENT_RESUME == 1: - return - logger.debug('Starting torrent {0} in {1}'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.start(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.start_torrent(input_id) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.resume_torrent([input_id]) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.resume(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to start torrent {0} in {1}'.format(input_name, client_agent)) - - -def remove_torrent(client_agent, input_hash, input_id, input_name): - if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': - logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.removedata(input_hash) - core.TORRENT_CLASS.remove(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.remove_torrent(input_id, True) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.remove_torrent(input_id, True) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.delete_permanently(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to delete torrent {0} in {1}'.format(input_name, client_agent)) - else: - resume_torrent(client_agent, input_hash, input_id, input_name) - - def find_download(client_agent, download_id): logger.debug('Searching for Download on {0} ...'.format(client_agent)) if client_agent == 'utorrent': diff --git a/core/utils/torrents.py b/core/utils/torrents.py new file mode 100644 index 00000000..e9786da9 --- /dev/null +++ b/core/utils/torrents.py @@ -0,0 +1,104 @@ +import time + +from qbittorrent import Client as qBittorrentClient +from synchronousdeluge.client import DelugeClient +from transmissionrpc.client import Client as TransmissionClient +from utorrent.client import UTorrentClient + +import core +from core import logger + + +def create_torrent_class(client_agent): + # Hardlink solution for Torrents + tc = None + + if client_agent == 'utorrent': + try: + logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) + tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) + except Exception: + logger.error('Failed to connect to uTorrent') + + if client_agent == 'transmission': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format( + client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) + tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, + core.TRANSMISSIONUSR, + core.TRANSMISSIONPWD) + except Exception: + logger.error('Failed to connect to Transmission') + + if client_agent == 'deluge': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) + tc = DelugeClient() + tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, + password=core.DELUGEPWD) + except Exception: + logger.error('Failed to connect to Deluge') + + if client_agent == 'qbittorrent': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) + except Exception: + logger.error('Failed to connect to qBittorrent') + + return tc + + +def pause_torrent(client_agent, input_hash, input_id, input_name): + logger.debug('Stopping torrent {0} in {1} while processing'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.pause_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.pause(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to stop torrent {0} in {1}'.format(input_name, client_agent)) + + +def resume_torrent(client_agent, input_hash, input_id, input_name): + if not core.TORRENT_RESUME == 1: + return + logger.debug('Starting torrent {0} in {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.resume_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.resume(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to start torrent {0} in {1}'.format(input_name, client_agent)) + + +def remove_torrent(client_agent, input_hash, input_id, input_name): + if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': + logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.removedata(input_hash) + core.TORRENT_CLASS.remove(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.remove_torrent(input_id, True) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.remove_torrent(input_id, True) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.delete_permanently(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to delete torrent {0} in {1}'.format(input_name, client_agent)) + else: + resume_torrent(client_agent, input_hash, input_id, input_name) From bd16f11485c71200ba6a017fe60e198625984e77 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:10:35 -0500 Subject: [PATCH 019/406] Refactor nzbs from utils to utils.nzbs --- core/utils/__init__.py | 64 -------------------------------------- core/utils/nzbs.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 64 deletions(-) create mode 100644 core/utils/nzbs.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 500f1498..8e224974 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -44,22 +44,6 @@ def copyfileobj_fast(fsrc, fdst, length=512 * 1024): shutil.copyfileobj = copyfileobj_fast -def report_nzb(failure_link, client_agent): - # Contact indexer site - logger.info('Sending failure notification to indexer site') - if client_agent == 'nzbget': - headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} - elif client_agent == 'sabnzbd': - headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} - else: - return - try: - requests.post(failure_link, headers=headers, timeout=(30, 300)) - except Exception as e: - logger.error('Unable to open URL {0} due to {1}'.format(failure_link, e)) - return - - def sanitize_name(name): """ >>> sanitize_name('a/b/c') @@ -853,54 +837,6 @@ def find_download(client_agent, download_id): return False -def get_nzoid(input_name): - nzoid = None - slots = [] - logger.debug('Searching for nzoid from SAbnzbd ...') - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - url = base_url - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': 'queue', - 'output': 'json', - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return nzoid # failure - try: - result = r.json() - clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) - except Exception: - logger.warning('Data from SABnzbd queue could not be parsed') - params['mode'] = 'history' - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return nzoid # failure - try: - result = r.json() - clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) - except Exception: - logger.warning('Data from SABnzbd history could not be parsed') - try: - for nzo_id, name in slots: - if name in [input_name, clean_name]: - nzoid = nzo_id - logger.debug('Found nzoid: {0}'.format(nzoid)) - break - except Exception: - logger.warning('Data from SABnzbd could not be parsed') - return nzoid - - def clean_file_name(filename): """Cleans up nzb name by removing any . and _ characters, along with any trailing hyphens. diff --git a/core/utils/nzbs.py b/core/utils/nzbs.py new file mode 100644 index 00000000..ad3b4e04 --- /dev/null +++ b/core/utils/nzbs.py @@ -0,0 +1,70 @@ +import os + +import requests + +import core +from core import logger + + +def get_nzoid(input_name): + nzoid = None + slots = [] + logger.debug('Searching for nzoid from SAbnzbd ...') + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'queue', + 'output': 'json', + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) + except Exception: + logger.warning('Data from SABnzbd queue could not be parsed') + params['mode'] = 'history' + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) + except Exception: + logger.warning('Data from SABnzbd history could not be parsed') + try: + for nzo_id, name in slots: + if name in [input_name, clean_name]: + nzoid = nzo_id + logger.debug('Found nzoid: {0}'.format(nzoid)) + break + except Exception: + logger.warning('Data from SABnzbd could not be parsed') + return nzoid + + +def report_nzb(failure_link, client_agent): + # Contact indexer site + logger.info('Sending failure notification to indexer site') + if client_agent == 'nzbget': + headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} + elif client_agent == 'sabnzbd': + headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} + else: + return + try: + requests.post(failure_link, headers=headers, timeout=(30, 300)) + except Exception as e: + logger.error('Unable to open URL {0} due to {1}'.format(failure_link, e)) + return From 2d6e8034e2a5dd9275e0a8348fc7c97def45aa32 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:19:50 -0500 Subject: [PATCH 020/406] Refactor parses from utils to utils.parsers --- core/utils/__init__.py | 157 ++--------------------------------------- core/utils/parsers.py | 156 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 153 deletions(-) create mode 100644 core/utils/parsers.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 8e224974..130baa95 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,10 @@ import subliminal import core from core import extractor, logger, main_db +from core.utils.parsers import ( + parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, + parse_utorrent, parse_vuze, +) from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -479,159 +483,6 @@ def convert_to_ascii(input_name, dir_name): return input_name, dir_name -def parse_other(args): - return os.path.normpath(args[1]), '', '', '', '' - - -def parse_rtorrent(args): - # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, - # 'execute={/path/to/nzbToMedia/TorrentToMedia.py,\'$d.get_base_path=\',\'$d.get_name=\',\'$d.get_custom1=\',\'$d.get_hash=\'}' - input_directory = os.path.normpath(args[1]) - try: - input_name = args[2] - except Exception: - input_name = '' - try: - input_category = args[3] - except Exception: - input_category = '' - try: - input_hash = args[4] - except Exception: - input_hash = '' - try: - input_id = args[4] - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_utorrent(args): - # uTorrent usage: call TorrentToMedia.py '%D' '%N' '%L' '%I' - input_directory = os.path.normpath(args[1]) - input_name = args[2] - try: - input_category = args[3] - except Exception: - input_category = '' - try: - input_hash = args[4] - except Exception: - input_hash = '' - try: - input_id = args[4] - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_deluge(args): - # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR - input_directory = os.path.normpath(args[3]) - input_name = args[2] - input_hash = args[1] - input_id = args[1] - try: - input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] - except Exception: - input_category = '' - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_transmission(args): - # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) - input_directory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) - input_name = os.getenv('TR_TORRENT_NAME') - input_category = '' # We dont have a category yet - input_hash = os.getenv('TR_TORRENT_HASH') - input_id = os.getenv('TR_TORRENT_ID') - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_vuze(args): - # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D%N%L%I%K%F' - try: - cur_input = args[1].split(',') - except Exception: - cur_input = [] - try: - input_directory = os.path.normpath(cur_input[0]) - except Exception: - input_directory = '' - try: - input_name = cur_input[1] - except Exception: - input_name = '' - try: - input_category = cur_input[2] - except Exception: - input_category = '' - try: - input_hash = cur_input[3] - except Exception: - input_hash = '' - try: - input_id = cur_input[3] - except Exception: - input_id = '' - try: - if cur_input[4] == 'single': - input_name = cur_input[5] - except Exception: - pass - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_qbittorrent(args): - # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D|%N|%L|%I' - try: - cur_input = args[1].split('|') - except Exception: - cur_input = [] - try: - input_directory = os.path.normpath(cur_input[0].replace('\'', '')) - except Exception: - input_directory = '' - try: - input_name = cur_input[1].replace('\'', '') - except Exception: - input_name = '' - try: - input_category = cur_input[2].replace('\'', '') - except Exception: - input_category = '' - try: - input_hash = cur_input[3].replace('\'', '') - except Exception: - input_hash = '' - try: - input_id = cur_input[3].replace('\'', '') - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_args(client_agent, args): - clients = { - 'other': parse_other, - 'rtorrent': parse_rtorrent, - 'utorrent': parse_utorrent, - 'deluge': parse_deluge, - 'transmission': parse_transmission, - 'qbittorrent': parse_qbittorrent, - 'vuze': parse_vuze, - } - - try: - return clients[client_agent](args) - except Exception: - return None, None, None, None, None - - def get_dirs(section, subsection, link='hard'): to_return = [] diff --git a/core/utils/parsers.py b/core/utils/parsers.py new file mode 100644 index 00000000..d5f6d933 --- /dev/null +++ b/core/utils/parsers.py @@ -0,0 +1,156 @@ +import os + +import core + + +def parse_other(args): + return os.path.normpath(args[1]), '', '', '', '' + + +def parse_rtorrent(args): + # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, + # 'execute={/path/to/nzbToMedia/TorrentToMedia.py,\'$d.get_base_path=\',\'$d.get_name=\',\'$d.get_custom1=\',\'$d.get_hash=\'}' + input_directory = os.path.normpath(args[1]) + try: + input_name = args[2] + except Exception: + input_name = '' + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_utorrent(args): + # uTorrent usage: call TorrentToMedia.py '%D' '%N' '%L' '%I' + input_directory = os.path.normpath(args[1]) + input_name = args[2] + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_deluge(args): + # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR + input_directory = os.path.normpath(args[3]) + input_name = args[2] + input_hash = args[1] + input_id = args[1] + try: + input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] + except Exception: + input_category = '' + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_transmission(args): + # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) + input_directory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) + input_name = os.getenv('TR_TORRENT_NAME') + input_category = '' # We dont have a category yet + input_hash = os.getenv('TR_TORRENT_HASH') + input_id = os.getenv('TR_TORRENT_ID') + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_vuze(args): + # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D%N%L%I%K%F' + try: + cur_input = args[1].split(',') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0]) + except Exception: + input_directory = '' + try: + input_name = cur_input[1] + except Exception: + input_name = '' + try: + input_category = cur_input[2] + except Exception: + input_category = '' + try: + input_hash = cur_input[3] + except Exception: + input_hash = '' + try: + input_id = cur_input[3] + except Exception: + input_id = '' + try: + if cur_input[4] == 'single': + input_name = cur_input[5] + except Exception: + pass + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_qbittorrent(args): + # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D|%N|%L|%I' + try: + cur_input = args[1].split('|') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0].replace('\'', '')) + except Exception: + input_directory = '' + try: + input_name = cur_input[1].replace('\'', '') + except Exception: + input_name = '' + try: + input_category = cur_input[2].replace('\'', '') + except Exception: + input_category = '' + try: + input_hash = cur_input[3].replace('\'', '') + except Exception: + input_hash = '' + try: + input_id = cur_input[3].replace('\'', '') + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_args(client_agent, args): + clients = { + 'other': parse_other, + 'rtorrent': parse_rtorrent, + 'utorrent': parse_utorrent, + 'deluge': parse_deluge, + 'transmission': parse_transmission, + 'qbittorrent': parse_qbittorrent, + 'vuze': parse_vuze, + } + + try: + return clients[client_agent](args) + except Exception: + return None, None, None, None, None From a50a5edbf70d2ce9941cd52e39c0ee2ceaf329b2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:31:48 -0500 Subject: [PATCH 021/406] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 39 ++------------------------------------ core/utils/paths.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 core/utils/paths.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 130baa95..23867ae1 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import datetime -from functools import partial import os import re import shutil @@ -26,6 +25,7 @@ from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, ) +from core.utils.paths import get_dir_size, make_dir, remote_dir from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -74,33 +74,6 @@ def sanitize_name(name): return name -def make_dir(path): - if not os.path.isdir(path): - try: - os.makedirs(path) - except Exception: - return False - return True - - -def remote_dir(path): - if not core.REMOTEPATHS: - return path - for local, remote in core.REMOTEPATHS: - if local in path: - base_dirs = path.replace(local, '').split(os.sep) - if '/' in remote: - remote_sep = '/' - else: - remote_sep = '\\' - new_path = remote_sep.join([remote] + base_dirs) - new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) - new_path = re.sub(r'(/+)', r'/', new_path) - new_path = re.sub(r'([/\\])$', r'', new_path) - return new_path - return path - - def category_search(input_directory, input_name, input_category, root, categories): tordir = False @@ -195,14 +168,6 @@ def category_search(input_directory, input_name, input_category, root, categorie return input_directory, input_name, input_category, root -def get_dir_size(input_path): - prepend = partial(os.path.join, input_path) - return sum([ - (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) - for f in map(prepend, os.listdir(text_type(input_path))) - ]) - - def is_min_size(input_name, min_size): file_name, file_ext = os.path.splitext(os.path.basename(input_name)) @@ -597,7 +562,7 @@ def onerror(func, path, exc_info): it attempts to add write permission and then retries. If the error is for another reason it re-raises the error. - + Usage : ``shutil.rmtree(path, onerror=onerror)`` """ if not os.access(path, os.W_OK): diff --git a/core/utils/paths.py b/core/utils/paths.py new file mode 100644 index 00000000..04392617 --- /dev/null +++ b/core/utils/paths.py @@ -0,0 +1,43 @@ + +from functools import partial +import os +import re + +from six import text_type + +import core + + +def make_dir(path): + if not os.path.isdir(path): + try: + os.makedirs(path) + except Exception: + return False + return True + + +def remote_dir(path): + if not core.REMOTEPATHS: + return path + for local, remote in core.REMOTEPATHS: + if local in path: + base_dirs = path.replace(local, '').split(os.sep) + if '/' in remote: + remote_sep = '/' + else: + remote_sep = '\\' + new_path = remote_sep.join([remote] + base_dirs) + new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) + new_path = re.sub(r'(/+)', r'/', new_path) + new_path = re.sub(r'([/\\])$', r'', new_path) + return new_path + return path + + +def get_dir_size(input_path): + prepend = partial(os.path.join, input_path) + return sum([ + (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) + for f in map(prepend, os.listdir(text_type(input_path))) + ]) From 04942bf6ad35a4c851f7df6aab5d538c71d17dfd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:51:20 -0500 Subject: [PATCH 022/406] Refactor download info to utils.download_info --- core/utils/__init__.py | 22 ++-------------------- core/utils/download_info.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 core/utils/download_info.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 23867ae1..f4a1f92c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -2,7 +2,6 @@ from __future__ import print_function, unicode_literals -import datetime import os import re import shutil @@ -20,7 +19,8 @@ from six import text_type import subliminal import core -from core import extractor, logger, main_db +from core import extractor, logger +from core.utils.download_info import get_download_info, update_download_info_status from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, @@ -950,21 +950,3 @@ def backup_versioned_file(old_file, version): return False return True - - -def update_download_info_status(input_name, status): - logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - - my_db = main_db.DBConnection() - my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) - - -def get_download_info(input_name, status): - logger.db('Getting download info for {0} from the DB'.format(input_name)) - - my_db = main_db.DBConnection() - sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) - - return sql_results diff --git a/core/utils/download_info.py b/core/utils/download_info.py new file mode 100644 index 00000000..2c2f7da4 --- /dev/null +++ b/core/utils/download_info.py @@ -0,0 +1,23 @@ +import datetime + +from six import text_type + +from core import logger, main_db + + +def update_download_info_status(input_name, status): + logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) + + my_db = main_db.DBConnection() + my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', + [status, datetime.date.today().toordinal(), text_type(input_name)]) + + +def get_download_info(input_name, status): + logger.db('Getting download info for {0} from the DB'.format(input_name)) + + my_db = main_db.DBConnection() + sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', + [text_type(input_name), status]) + + return sql_results From 542893b30bd1a41012e4cf6396c3417f31f69615 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:52:53 -0500 Subject: [PATCH 023/406] Refactor download_info db connection to module variable --- core/utils/download_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index 2c2f7da4..1ef100a2 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -4,11 +4,12 @@ from six import text_type from core import logger, main_db +my_db = main_db.DBConnection() + def update_download_info_status(input_name, status): logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - my_db = main_db.DBConnection() my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', [status, datetime.date.today().toordinal(), text_type(input_name)]) @@ -16,7 +17,6 @@ def update_download_info_status(input_name, status): def get_download_info(input_name, status): logger.db('Getting download info for {0} from the DB'.format(input_name)) - my_db = main_db.DBConnection() sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', [text_type(input_name), status]) From 7b8721b27709a86e596ba18b87e97f21df86f6a6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:53:13 -0500 Subject: [PATCH 024/406] Refactor my_db -> database --- core/utils/download_info.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index 1ef100a2..c4cc530e 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -4,20 +4,20 @@ from six import text_type from core import logger, main_db -my_db = main_db.DBConnection() +database = main_db.DBConnection() def update_download_info_status(input_name, status): logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) + database.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', + [status, datetime.date.today().toordinal(), text_type(input_name)]) def get_download_info(input_name, status): logger.db('Getting download info for {0} from the DB'.format(input_name)) - sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) + sql_results = database.select('SELECT * FROM downloads WHERE input_name=? AND status=?', + [text_type(input_name), status]) return sql_results From 84061fea2f05c5f6c24e5221c35e98da43bb7b4b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 15:30:53 -0500 Subject: [PATCH 025/406] Fix PEP8 line length --- core/utils/download_info.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index c4cc530e..ce6e6717 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -8,16 +8,16 @@ database = main_db.DBConnection() def update_download_info_status(input_name, status): - logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - - database.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) + msg = 'Updating DB download status of {0} to {1}' + action = 'UPDATE downloads SET status=?, last_update=? WHERE input_name=?' + args = [status, datetime.date.today().toordinal(), text_type(input_name)] + logger.db(msg.format(input_name, status)) + database.action(action, args) def get_download_info(input_name, status): - logger.db('Getting download info for {0} from the DB'.format(input_name)) - - sql_results = database.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) - - return sql_results + msg = 'Getting download info for {0} from the DB' + action = 'SELECT * FROM downloads WHERE input_name=? AND status=?' + args = [text_type(input_name), status] + logger.db(msg.format(input_name)) + return database.select(action, args) From 42553df5cbfe9c66d454239393fbd508167c6785 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 15:33:56 -0500 Subject: [PATCH 026/406] Refactor network utils to utils.network --- core/utils/__init__.py | 53 +-------------------------------------- core/utils/network.py | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 52 deletions(-) create mode 100644 core/utils/network.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index f4a1f92c..d97e9611 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -5,9 +5,7 @@ from __future__ import print_function, unicode_literals import os import re import shutil -import socket import stat -import struct import time from babelfish import Language @@ -21,6 +19,7 @@ import subliminal import core from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, @@ -319,56 +318,6 @@ def remove_read_only(filename): logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) -# Wake function -def wake_on_lan(ethernet_address): - addr_byte = ethernet_address.split(':') - hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), - int(addr_byte[1], 16), - int(addr_byte[2], 16), - int(addr_byte[3], 16), - int(addr_byte[4], 16), - int(addr_byte[5], 16)) - - # Build the Wake-On-LAN 'Magic Packet'... - - msg = b'\xff' * 6 + hw_addr * 16 - - # ...and send it to the broadcast address using UDP - - ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - ss.sendto(msg, ('', 9)) - ss.close() - - -# Test Connection function -def test_connection(host, port): - try: - socket.create_connection((host, port)) - return 'Up' - except Exception: - return 'Down' - - -def wake_up(): - host = core.CFG['WakeOnLan']['host'] - port = int(core.CFG['WakeOnLan']['port']) - mac = core.CFG['WakeOnLan']['mac'] - - i = 1 - while test_connection(host, port) == 'Down' and i < 4: - logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) - wake_on_lan(mac) - time.sleep(20) - i = i + 1 - - if test_connection(host, port) == 'Down': # final check. - logger.warning('System with mac: {0} has not woken after 3 attempts. ' - 'Continuing with the rest of the script.'.format(mac)) - else: - logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) - - def char_replace(name): # Special character hex range: # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) diff --git a/core/utils/network.py b/core/utils/network.py new file mode 100644 index 00000000..4409c486 --- /dev/null +++ b/core/utils/network.py @@ -0,0 +1,56 @@ +import socket +import struct +import time + +import core +from core import logger + + +# Wake function +def wake_on_lan(ethernet_address): + addr_byte = ethernet_address.split(':') + hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), + int(addr_byte[1], 16), + int(addr_byte[2], 16), + int(addr_byte[3], 16), + int(addr_byte[4], 16), + int(addr_byte[5], 16)) + + # Build the Wake-On-LAN 'Magic Packet'... + + msg = b'\xff' * 6 + hw_addr * 16 + + # ...and send it to the broadcast address using UDP + + ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + ss.sendto(msg, ('', 9)) + ss.close() + + +# Test Connection function +def test_connection(host, port): + try: + socket.create_connection((host, port)) + return 'Up' + except Exception: + return 'Down' + + +def wake_up(): + host = core.CFG['WakeOnLan']['host'] + port = int(core.CFG['WakeOnLan']['port']) + mac = core.CFG['WakeOnLan']['mac'] + + i = 1 + while test_connection(host, port) == 'Down' and i < 4: + logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) + wake_on_lan(mac) + time.sleep(20) + i = i + 1 + + if test_connection(host, port) == 'Down': # final check. + logger.warning('System with mac: {0} has not woken after 3 attempts. ' + 'Continuing with the rest of the script.'.format(mac)) + else: + logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) From 094fe555b8f60acf4c379e659e9075da07ae8ac4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 16:38:33 -0500 Subject: [PATCH 027/406] Clean up network utils --- core/utils/network.py | 74 ++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/core/utils/network.py b/core/utils/network.py index 4409c486..dae99837 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -6,51 +6,61 @@ import core from core import logger -# Wake function +def make_wake_on_lan_packet(mac_address): + """Build the Wake-On-LAN 'Magic Packet'.""" + address = ( + int(value, 16) + for value in mac_address.split(':') + ) + fmt = 'BBBBBB' + hardware_address = struct.pack(fmt, *address) + broadcast_address = b'\xFF' * 6 # FF:FF:FF:FF:FF:FF + return broadcast_address + hardware_address * 16 + + def wake_on_lan(ethernet_address): - addr_byte = ethernet_address.split(':') - hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), - int(addr_byte[1], 16), - int(addr_byte[2], 16), - int(addr_byte[3], 16), - int(addr_byte[4], 16), - int(addr_byte[5], 16)) - - # Build the Wake-On-LAN 'Magic Packet'... - - msg = b'\xff' * 6 + hw_addr * 16 + """Send a WakeOnLan request.""" + # Create the WoL magic packet + magic_packet = make_wake_on_lan_packet(ethernet_address) # ...and send it to the broadcast address using UDP + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + connection.sendto(magic_packet, ('', 9)) - ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - ss.sendto(msg, ('', 9)) - ss.close() + logger.info('WakeOnLan sent for mac: {0}'.format(ethernet_address)) -# Test Connection function def test_connection(host, port): + """Test network connection.""" + address = host, port try: - socket.create_connection((host, port)) - return 'Up' - except Exception: + socket.create_connection(address) + except socket.error: return 'Down' + else: + return 'Up' def wake_up(): - host = core.CFG['WakeOnLan']['host'] - port = int(core.CFG['WakeOnLan']['port']) - mac = core.CFG['WakeOnLan']['mac'] + wol = core.CFG['WakeOnLan'] + host = wol['host'] + port = int(wol['port']) + mac = wol['mac'] + max_attempts = 4 - i = 1 - while test_connection(host, port) == 'Down' and i < 4: - logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) + logger.info('Trying to wake On lan.') + + for attempt in range(0, max_attempts): + logger.info('Attempt {0} of {1}'.format(attempt + 1, max_attempts, mac)) + if test_connection(host, port) == 'Up': + logger.info('System with mac: {0} has been woken.'.format(mac)) + break wake_on_lan(mac) time.sleep(20) - i = i + 1 - - if test_connection(host, port) == 'Down': # final check. - logger.warning('System with mac: {0} has not woken after 3 attempts. ' - 'Continuing with the rest of the script.'.format(mac)) else: - logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) + if test_connection(host, port) == 'Down': # final check. + msg = 'System with mac: {0} has not woken after {1} attempts.' + logger.warning(msg.format(mac, max_attempts)) + + logger.info('Continuing with the rest of the script.') From a6d2c6e96fd871bc999b4f0ccc20e7bb43b46a25 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:12:43 -0500 Subject: [PATCH 028/406] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 52 ++++++++++++------------------------------ core/utils/paths.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d97e9611..376c4898 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -21,10 +21,21 @@ from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( - parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, - parse_utorrent, parse_vuze, + parse_args, + parse_deluge, + parse_other, + parse_qbittorrent, + parse_rtorrent, + parse_transmission, + parse_utorrent, + parse_vuze, +) +from core.utils.paths import ( + get_dir_size, make_dir, + remote_dir, + remove_empty_folders, + remove_read_only, ) -from core.utils.paths import get_dir_size, make_dir, remote_dir from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -283,41 +294,6 @@ def flatten(output_destination): remove_empty_folders(output_destination) # Cleanup empty directories -def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders""" - if not os.path.isdir(path): - return - - # remove empty subfolders - logger.debug('Checking for empty folders in:{0}'.format(path)) - files = os.listdir(text_type(path)) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - remove_empty_folders(fullpath) - - # if folder empty, delete it - files = os.listdir(text_type(path)) - if len(files) == 0 and remove_root: - logger.debug('Removing empty folder:{}'.format(path)) - os.rmdir(path) - - -def remove_read_only(filename): - if os.path.isfile(filename): - # check first the read-only attribute - file_attribute = os.stat(filename)[0] - if not file_attribute & stat.S_IWRITE: - # File is read-only, so make it writeable - logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format - (name=filename)) - try: - os.chmod(filename, stat.S_IWRITE) - except Exception: - logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) - - def char_replace(name): # Special character hex range: # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) diff --git a/core/utils/paths.py b/core/utils/paths.py index 04392617..31308650 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -2,10 +2,12 @@ from functools import partial import os import re +import stat from six import text_type import core +from core import logger def make_dir(path): @@ -41,3 +43,38 @@ def get_dir_size(input_path): (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) for f in map(prepend, os.listdir(text_type(input_path))) ]) + + +def remove_empty_folders(path, remove_root=True): + """Function to remove empty folders""" + if not os.path.isdir(path): + return + + # remove empty subfolders + logger.debug('Checking for empty folders in:{0}'.format(path)) + files = os.listdir(text_type(path)) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + remove_empty_folders(fullpath) + + # if folder empty, delete it + files = os.listdir(text_type(path)) + if len(files) == 0 and remove_root: + logger.debug('Removing empty folder:{}'.format(path)) + os.rmdir(path) + + +def remove_read_only(filename): + if os.path.isfile(filename): + # check first the read-only attribute + file_attribute = os.stat(filename)[0] + if not file_attribute & stat.S_IWRITE: + # File is read-only, so make it writeable + logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format + (name=filename)) + try: + os.chmod(filename, stat.S_IWRITE) + except Exception: + logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) From 5c644890e83662e7b749590c70e5ddf0dc7adca5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:02:03 -0500 Subject: [PATCH 029/406] Fix shutil.copyfileobj monkey patching --- core/utils/__init__.py | 11 ++--------- core/utils/shutil_custom.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 core/utils/shutil_custom.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 376c4898..7716011f 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -19,6 +19,7 @@ import subliminal import core from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status +from core.utils import shutil_custom from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -47,15 +48,7 @@ except ImportError: requests.packages.urllib3.disable_warnings() -# Monkey Patch shutil.copyfileobj() to adjust the buffer length to 512KB rather than 4KB -shutil.copyfileobjOrig = shutil.copyfileobj - - -def copyfileobj_fast(fsrc, fdst, length=512 * 1024): - shutil.copyfileobjOrig(fsrc, fdst, length=length) - - -shutil.copyfileobj = copyfileobj_fast +shutil_custom.monkey_patch() def sanitize_name(name): diff --git a/core/utils/shutil_custom.py b/core/utils/shutil_custom.py new file mode 100644 index 00000000..5525df1f --- /dev/null +++ b/core/utils/shutil_custom.py @@ -0,0 +1,11 @@ +from functools import partial +import shutil +from six import PY2 + + +def monkey_patch(length=512 * 1024): + if PY2: + # On Python 2 monkey patch shutil.copyfileobj() + # to adjust the buffer length to 512KB rather than 4KB + original_copyfileobj = shutil.copyfileobj + shutil.copyfileobj = partial(original_copyfileobj, length=length) From a14a286a8e66bb51aa8edf0d92d68acf90fac1dd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:07:52 -0500 Subject: [PATCH 030/406] Refactor links to utils.links --- core/utils/__init__.py | 78 +-------------------------------------- core/utils/links.py | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 76 deletions(-) create mode 100644 core/utils/links.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 7716011f..0ad08985 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -11,15 +11,15 @@ import time from babelfish import Language import beets import guessit -import linktastic import requests from six import text_type import subliminal import core from core import extractor, logger -from core.utils.download_info import get_download_info, update_download_info_status from core.utils import shutil_custom +from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.links import copy_link, replace_links from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -194,80 +194,6 @@ def is_sample(input_name): return True -def copy_link(src, target_link, use_link): - logger.info('MEDIAFILE: [{0}]'.format(os.path.basename(target_link)), 'COPYLINK') - logger.info('SOURCE FOLDER: [{0}]'.format(os.path.dirname(src)), 'COPYLINK') - logger.info('TARGET FOLDER: [{0}]'.format(os.path.dirname(target_link)), 'COPYLINK') - - if src != target_link and os.path.exists(target_link): - logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') - return True - elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): - logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') - return True - elif src == os.path.dirname(target_link): - logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') - return True - - make_dir(os.path.dirname(target_link)) - try: - if use_link == 'dir': - logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') - linktastic.dirlink(src, target_link) - return True - if use_link == 'junction': - logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') - linktastic.dirlink(src, target_link) - return True - elif use_link == 'hard': - logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - linktastic.link(src, target_link) - return True - elif use_link == 'sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - linktastic.symlink(src, target_link) - return True - elif use_link == 'move-sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.move(src, target_link) - linktastic.symlink(target_link, src) - return True - elif use_link == 'move': - logger.info('Moving SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.move(src, target_link) - return True - except Exception as e: - logger.warning('Error: {0}, copying instead ... '.format(e), 'COPYLINK') - - logger.info('Copying SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.copy(src, target_link) - - return True - - -def replace_links(link): - n = 0 - target = link - if os.name == 'nt': - if not jaraco.windows.filesystem.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while jaraco.windows.filesystem.islink(target): - target = jaraco.windows.filesystem.readlink(target) - n = n + 1 - else: - if not os.path.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while os.path.islink(target): - target = os.readlink(target) - n = n + 1 - if n > 1: - logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') - os.unlink(link) - linktastic.symlink(target, link) - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): diff --git a/core/utils/links.py b/core/utils/links.py new file mode 100644 index 00000000..ca5e99e1 --- /dev/null +++ b/core/utils/links.py @@ -0,0 +1,84 @@ +import os +import shutil + +import linktastic + +from core import logger +from core.utils.paths import make_dir + +if os.name == 'nt': + import jaraco + + +def copy_link(src, target_link, use_link): + logger.info('MEDIAFILE: [{0}]'.format(os.path.basename(target_link)), 'COPYLINK') + logger.info('SOURCE FOLDER: [{0}]'.format(os.path.dirname(src)), 'COPYLINK') + logger.info('TARGET FOLDER: [{0}]'.format(os.path.dirname(target_link)), 'COPYLINK') + + if src != target_link and os.path.exists(target_link): + logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') + return True + elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): + logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') + return True + elif src == os.path.dirname(target_link): + logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') + return True + + make_dir(os.path.dirname(target_link)) + try: + if use_link == 'dir': + logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + if use_link == 'junction': + logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + elif use_link == 'hard': + logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.link(src, target_link) + return True + elif use_link == 'sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.symlink(src, target_link) + return True + elif use_link == 'move-sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + linktastic.symlink(target_link, src) + return True + elif use_link == 'move': + logger.info('Moving SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + return True + except Exception as e: + logger.warning('Error: {0}, copying instead ... '.format(e), 'COPYLINK') + + logger.info('Copying SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.copy(src, target_link) + + return True + + +def replace_links(link): + n = 0 + target = link + if os.name == 'nt': + if not jaraco.windows.filesystem.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while jaraco.windows.filesystem.islink(target): + target = jaraco.windows.filesystem.readlink(target) + n = n + 1 + else: + if not os.path.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while os.path.islink(target): + target = os.readlink(target) + n = n + 1 + if n > 1: + logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') + os.unlink(link) + linktastic.symlink(target, link) From c4d9faeb23f2c8c15b32986676c6dea4e3ef6cca Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:19:21 -0500 Subject: [PATCH 031/406] Refactor network utils to utils.network --- core/utils/__init__.py | 46 +-------------------------------- core/utils/network.py | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 0ad08985..23dfb678 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -20,7 +20,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.links import copy_link, replace_links -from core.utils.network import test_connection, wake_on_lan, wake_up +from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, parse_deluge, @@ -453,50 +453,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def find_download(client_agent, download_id): - logger.debug('Searching for Download on {0} ...'.format(client_agent)) - if client_agent == 'utorrent': - torrents = core.TORRENT_CLASS.list()[1]['torrents'] - for torrent in torrents: - if download_id in torrent: - return True - if client_agent == 'transmission': - torrents = core.TORRENT_CLASS.get_torrents() - for torrent in torrents: - torrent_hash = torrent.hashString - if torrent_hash == download_id: - return True - if client_agent == 'deluge': - return False - if client_agent == 'qbittorrent': - torrents = core.TORRENT_CLASS.torrents() - for torrent in torrents: - if torrent['hash'] == download_id: - return True - if client_agent == 'sabnzbd': - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - url = base_url - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': 'get_files', - 'output': 'json', - 'value': download_id, - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return False # failure - - result = r.json() - if result['files']: - return True - return False - - def clean_file_name(filename): """Cleans up nzb name by removing any . and _ characters, along with any trailing hyphens. diff --git a/core/utils/network.py b/core/utils/network.py index dae99837..99a89e1e 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -2,6 +2,8 @@ import socket import struct import time +import requests + import core from core import logger @@ -64,3 +66,59 @@ def wake_up(): logger.warning(msg.format(mac, max_attempts)) logger.info('Continuing with the rest of the script.') + + +def server_responding(base_url): + logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') + try: + requests.get(base_url, timeout=(60, 120), verify=False) + except (requests.ConnectionError, requests.exceptions.Timeout): + logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') + return False + else: + logger.debug('Server responded at {0}'.format(base_url), 'SERVER') + return True + + +def find_download(client_agent, download_id): + logger.debug('Searching for Download on {0} ...'.format(client_agent)) + if client_agent == 'utorrent': + torrents = core.TORRENT_CLASS.list()[1]['torrents'] + for torrent in torrents: + if download_id in torrent: + return True + if client_agent == 'transmission': + torrents = core.TORRENT_CLASS.get_torrents() + for torrent in torrents: + torrent_hash = torrent.hashString + if torrent_hash == download_id: + return True + if client_agent == 'deluge': + return False + if client_agent == 'qbittorrent': + torrents = core.TORRENT_CLASS.torrents() + for torrent in torrents: + if torrent['hash'] == download_id: + return True + if client_agent == 'sabnzbd': + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'get_files', + 'output': 'json', + 'value': download_id, + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return False # failure + + result = r.json() + if result['files']: + return True + return False From f042e014b1f9615ad849abb5f8e5a7d9022c8269 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:22:40 -0500 Subject: [PATCH 032/406] Refactor naming utils to utils.naming --- core/utils/__init__.py | 33 +-------------------------- core/utils/naming.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 core/utils/naming.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 23dfb678..4987314b 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -20,6 +20,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.links import copy_link, replace_links +from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -51,32 +52,6 @@ requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() -def sanitize_name(name): - """ - >>> sanitize_name('a/b/c') - 'a-b-c' - >>> sanitize_name('abc') - 'abc' - >>> sanitize_name('a"b') - 'ab' - >>> sanitize_name('.a.b..') - 'a.b' - """ - - # remove bad chars from the filename - name = re.sub(r'[\\/*]', '-', name) - name = re.sub(r'[:\'<>|?]', '', name) - - # remove leading/trailing periods and spaces - name = name.strip(' .') - try: - name = name.encode(core.SYS_ENCODING) - except Exception: - pass - - return name - - def category_search(input_directory, input_name, input_category, root, categories): tordir = False @@ -188,12 +163,6 @@ def is_min_size(input_name, min_size): return True -def is_sample(input_name): - # Ignore 'sample' in files - if re.search('(^|[\\W_])sample\\d*[\\W_]', input_name.lower()): - return True - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): diff --git a/core/utils/naming.py b/core/utils/naming.py new file mode 100644 index 00000000..352d51ba --- /dev/null +++ b/core/utils/naming.py @@ -0,0 +1,52 @@ +import re +import core + + +def sanitize_name(name): + """ + >>> sanitize_name('a/b/c') + 'a-b-c' + >>> sanitize_name('abc') + 'abc' + >>> sanitize_name('a"b') + 'ab' + >>> sanitize_name('.a.b..') + 'a.b' + """ + + # remove bad chars from the filename + name = re.sub(r'[\\/*]', '-', name) + name = re.sub(r'[:\'<>|?]', '', name) + + # remove leading/trailing periods and spaces + name = name.strip(' .') + try: + name = name.encode(core.SYS_ENCODING) + except Exception: + pass + + return name + + +def clean_file_name(filename): + """Cleans up nzb name by removing any . and _ + characters, along with any trailing hyphens. + + Is basically equivalent to replacing all _ and . with a + space, but handles decimal numbers in string, for example: + """ + + filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) + filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot + filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) + filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) + filename = filename.replace('_', ' ') + filename = re.sub('-$', '', filename) + filename = re.sub(r'^\[.*]', '', filename) + return filename.strip() + + +def is_sample(input_name): + # Ignore 'sample' in files + if re.search('(^|[\\W_])sample\\d*[\\W_]', input_name.lower()): + return True From 9d43e0d60b9d721b723608705ee1640df3174fe0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 16:45:02 -0500 Subject: [PATCH 033/406] Refactor notification utils to utils.notifications --- core/utils/__init__.py | 25 +------------------------ core/utils/notifications.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 core/utils/notifications.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 4987314b..46a30431 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,7 @@ from core.utils.download_info import get_download_info, update_download_info_sta from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up +from core.utils.notifications import plex_update from core.utils.parsers import ( parse_args, parse_deluge, @@ -668,30 +669,6 @@ def server_responding(base_url): return False -def plex_update(category): - if core.FAILED: - return - url = '{scheme}://{host}:{port}/library/sections/'.format( - scheme='https' if core.PLEXSSL else 'http', - host=core.PLEXHOST, - port=core.PLEXPORT, - ) - section = None - if not core.PLEXSEC: - return - logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') - for item in core.PLEXSEC: - if item[0] == category: - section = item[1] - - if section: - url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) - requests.get(url, timeout=(60, 120), verify=False) - logger.debug('Plex Library has been refreshed.', 'PLEX') - else: - logger.debug('Could not identify section for plex update', 'PLEX') - - def backup_versioned_file(old_file, version): num_tries = 0 diff --git a/core/utils/notifications.py b/core/utils/notifications.py new file mode 100644 index 00000000..ed89f65b --- /dev/null +++ b/core/utils/notifications.py @@ -0,0 +1,30 @@ +import requests + +import core +from core import logger + + +def plex_update(category): + if core.FAILED: + return + url = '{scheme}://{host}:{port}/library/sections/'.format( + scheme='https' if core.PLEXSSL else 'http', + host=core.PLEXHOST, + port=core.PLEXPORT, + ) + section = None + if not core.PLEXSEC: + return + logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') + for item in core.PLEXSEC: + if item[0] == category: + section = item[1] + + if section: + url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) + requests.get(url, timeout=(60, 120), verify=False) + logger.debug('Plex Library has been refreshed.', 'PLEX') + else: + logger.debug('Could not identify section for plex update', 'PLEX') + + From 6a9ff96e8c9e4d28d4deff98fb13aeb2016d1b78 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:25:36 -0500 Subject: [PATCH 034/406] Refactor encoding utils to utils.encoding --- core/utils/__init__.py | 80 +-------------------------------------- core/utils/encoding.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 79 deletions(-) create mode 100644 core/utils/encoding.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 46a30431..d0bc778a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -19,6 +19,7 @@ import core from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.encoding import char_replace, convert_to_ascii from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -183,85 +184,6 @@ def flatten(output_destination): remove_empty_folders(output_destination) # Cleanup empty directories -def char_replace(name): - # Special character hex range: - # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) - # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF - # ISO-8859-15: 0xA6-0xFF - # The function will detect if Name contains a special character - # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding - encoded = False - encoding = None - if isinstance(name, text_type): - return encoded, name.encode(core.SYS_ENCODING) - for Idx in range(len(name)): - # /!\ detection is done 2char by 2char for UTF-8 special character - if (len(name) != 1) & (Idx < (len(name) - 1)): - # Detect UTF-8 - if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & ( - (name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')): - encoding = 'utf-8' - break - # Detect CP850 - elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - else: - # Detect CP850 - if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - if encoding and not encoding == core.SYS_ENCODING: - encoded = True - name = name.decode(encoding).encode(core.SYS_ENCODING) - return encoded, name - - -def convert_to_ascii(input_name, dir_name): - - ascii_convert = int(core.CFG['ASCII']['convert']) - if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. - return input_name, dir_name - - encoded, input_name = char_replace(input_name) - - directory, base = os.path.split(dir_name) - if not base: # ended with '/' - directory, base = os.path.split(directory) - - encoded, base2 = char_replace(base) - if encoded: - dir_name = os.path.join(directory, base2) - logger.info('Renaming directory to: {0}.'.format(base2), 'ENCODER') - os.rename(os.path.join(directory, base), dir_name) - if 'NZBOP_SCRIPTDIR' in os.environ: - print('[NZB] DIRECTORY={0}'.format(dir_name)) - - for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): - for subdirname in dirnames: - encoded, subdirname2 = char_replace(subdirname) - if encoded: - logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') - os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) - - for dirname, dirnames, filenames in os.walk(dir_name): - for filename in filenames: - encoded, filename2 = char_replace(filename) - if encoded: - logger.info('Renaming file to: {0}.'.format(filename2), 'ENCODER') - os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) - - return input_name, dir_name - - def get_dirs(section, subsection, link='hard'): to_return = [] diff --git a/core/utils/encoding.py b/core/utils/encoding.py new file mode 100644 index 00000000..ca19e054 --- /dev/null +++ b/core/utils/encoding.py @@ -0,0 +1,85 @@ +import os + +from six import text_type + +import core +from core import logger + + +def char_replace(name): + # Special character hex range: + # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) + # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF + # ISO-8859-15: 0xA6-0xFF + # The function will detect if Name contains a special character + # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding + encoded = False + encoding = None + if isinstance(name, text_type): + return encoded, name.encode(core.SYS_ENCODING) + for Idx in range(len(name)): + # /!\ detection is done 2char by 2char for UTF-8 special character + if (len(name) != 1) & (Idx < (len(name) - 1)): + # Detect UTF-8 + if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & ( + (name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')): + encoding = 'utf-8' + break + # Detect CP850 + elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + else: + # Detect CP850 + if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + if encoding and not encoding == core.SYS_ENCODING: + encoded = True + name = name.decode(encoding).encode(core.SYS_ENCODING) + return encoded, name + + +def convert_to_ascii(input_name, dir_name): + + ascii_convert = int(core.CFG['ASCII']['convert']) + if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. + return input_name, dir_name + + encoded, input_name = char_replace(input_name) + + directory, base = os.path.split(dir_name) + if not base: # ended with '/' + directory, base = os.path.split(directory) + + encoded, base2 = char_replace(base) + if encoded: + dir_name = os.path.join(directory, base2) + logger.info('Renaming directory to: {0}.'.format(base2), 'ENCODER') + os.rename(os.path.join(directory, base), dir_name) + if 'NZBOP_SCRIPTDIR' in os.environ: + print('[NZB] DIRECTORY={0}'.format(dir_name)) + + for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): + for subdirname in dirnames: + encoded, subdirname2 = char_replace(subdirname) + if encoded: + logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') + os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) + + for dirname, dirnames, filenames in os.walk(dir_name): + for filename in filenames: + encoded, filename2 = char_replace(filename) + if encoded: + logger.info('Renaming file to: {0}.'.format(filename2), 'ENCODER') + os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) + + return input_name, dir_name From 2d4951267bca16ca154ad21f108159be823e9bed Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:27:14 -0500 Subject: [PATCH 035/406] Refactor subtitle utils to utils.subtitles --- core/utils/__init__.py | 29 +---------------------------- core/utils/subtitles.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 core/utils/subtitles.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d0bc778a..d04c4f1c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -8,12 +8,10 @@ import shutil import stat import time -from babelfish import Language import beets import guessit import requests from six import text_type -import subliminal import core from core import extractor, logger @@ -41,6 +39,7 @@ from core.utils.paths import ( remove_read_only, ) from core.utils.processes import RunningProcess +from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent try: @@ -554,32 +553,6 @@ def extract_files(src, dst=None, keep_archive=None): logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) -def import_subs(filename): - if not core.GETSUBS: - return - try: - subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) - except Exception: - pass - - languages = set() - for item in core.SLANGUAGES: - try: - languages.add(Language(item)) - except Exception: - pass - if not languages: - return - - logger.info('Attempting to download subtitles for {0}'.format(filename), 'SUBTITLES') - try: - video = subliminal.scan_video(filename) - subtitles = subliminal.download_best_subtitles({video}, languages) - subliminal.save_subtitles(video, subtitles[video]) - except Exception as e: - logger.error('Failed to download subtitles for {0} due to: {1}'.format(filename, e), 'SUBTITLES') - - def server_responding(base_url): logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') try: diff --git a/core/utils/subtitles.py b/core/utils/subtitles.py new file mode 100644 index 00000000..f62f90de --- /dev/null +++ b/core/utils/subtitles.py @@ -0,0 +1,31 @@ +from babelfish import Language +import subliminal + +import core +from core import logger + + +def import_subs(filename): + if not core.GETSUBS: + return + try: + subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) + except Exception: + pass + + languages = set() + for item in core.SLANGUAGES: + try: + languages.add(Language(item)) + except Exception: + pass + if not languages: + return + + logger.info('Attempting to download subtitles for {0}'.format(filename), 'SUBTITLES') + try: + video = subliminal.scan_video(filename) + subtitles = subliminal.download_best_subtitles({video}, languages) + subliminal.save_subtitles(video, subtitles[video]) + except Exception as e: + logger.error('Failed to download subtitles for {0} due to: {1}'.format(filename, e), 'SUBTITLES') From d1f5211e78d9baae5691d6a21186f53eb1bb398c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 02:33:15 -0500 Subject: [PATCH 036/406] Refactor nzbs from utils to utils.nzbs --- core/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d04c4f1c..fd646d7e 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,7 @@ from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.notifications import plex_update +from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( parse_args, parse_deluge, From 9b0d53942373ad79383ec5acd30a15235cdfdbef Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:30:58 -0500 Subject: [PATCH 037/406] Make replace_links more DRY and add max_depth for following links --- core/utils/links.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/core/utils/links.py b/core/utils/links.py index ca5e99e1..ab558e9b 100644 --- a/core/utils/links.py +++ b/core/utils/links.py @@ -6,8 +6,14 @@ import linktastic from core import logger from core.utils.paths import make_dir -if os.name == 'nt': - import jaraco +try: + from jaraco.windows.filesystem import islink, readlink +except ImportError: + if os.name == 'nt': + raise + else: + from os.path import islink + from os import readlink def copy_link(src, target_link, use_link): @@ -61,24 +67,21 @@ def copy_link(src, target_link, use_link): return True -def replace_links(link): - n = 0 +def replace_links(link, max_depth=10): + link_depth = 0 target = link - if os.name == 'nt': - if not jaraco.windows.filesystem.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while jaraco.windows.filesystem.islink(target): - target = jaraco.windows.filesystem.readlink(target) - n = n + 1 + + for attempt in range(0, max_depth): + if not islink(target): + break + target = readlink(target) + link_depth = attempt + + if not link_depth: + logger.debug('{0} is not a link'.format(link)) + elif link_depth > max_depth or (link_depth == max_depth and islink(target)): + logger.warning('Exceeded maximum depth {0} while following link {1}'.format(max_depth, link)) else: - if not os.path.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while os.path.islink(target): - target = os.readlink(target) - n = n + 1 - if n > 1: logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') os.unlink(link) linktastic.symlink(target, link) From a074e566297252b0acff7c088c109a4003ecf9b4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:45:26 -0500 Subject: [PATCH 038/406] Refactor naming utils to utils.naming --- core/utils/__init__.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fd646d7e..a8dbe792 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -345,24 +345,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def clean_file_name(filename): - """Cleans up nzb name by removing any . and _ - characters, along with any trailing hyphens. - - Is basically equivalent to replacing all _ and . with a - space, but handles decimal numbers in string, for example: - """ - - filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) - filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot - filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) - filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) - filename = filename.replace('_', ' ') - filename = re.sub('-$', '', filename) - filename = re.sub(r'^\[.*]', '', filename) - return filename.strip() - - def is_archive_file(filename): """Check if the filename is allowed for the Archive""" for regext in core.COMPRESSEDCONTAINER: From 0cccecb43565eddccdcc21cb3afdd8f1dd889682 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:53:01 -0500 Subject: [PATCH 039/406] Refactor file type detection to utils.files --- core/utils/__init__.py | 95 +------------------------------------- core/utils/files.py | 102 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 94 deletions(-) create mode 100644 core/utils/files.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index a8dbe792..cdfb9418 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -18,6 +18,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii +from core.utils.files import is_archive_file, is_media_file, is_min_size, list_media_files from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -148,23 +149,6 @@ def category_search(input_directory, input_name, input_category, root, categorie return input_directory, input_name, input_category, root -def is_min_size(input_name, min_size): - file_name, file_ext = os.path.splitext(os.path.basename(input_name)) - - # audio files we need to check directory size not file size - input_size = os.path.getsize(input_name) - if file_ext in core.AUDIOCONTAINER: - try: - input_size = get_dir_size(os.path.dirname(input_name)) - except Exception: - logger.error('Failed to get file size for {0}'.format(input_name), 'MINSIZE') - return True - - # Ignore files under a certain size - if input_size > min_size * 1048576: - return True - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): @@ -345,83 +329,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def is_archive_file(filename): - """Check if the filename is allowed for the Archive""" - for regext in core.COMPRESSEDCONTAINER: - if regext.search(filename): - return regext.split(filename)[0] - return False - - -def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): - if otherext is None: - otherext = [] - - file_name, file_ext = os.path.splitext(mediafile) - - try: - # ignore MAC OS's 'resource fork' files - if file_name.startswith('._'): - return False - except Exception: - pass - if (media and file_ext.lower() in core.MEDIACONTAINER) \ - or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ - or (meta and file_ext.lower() in core.METACONTAINER) \ - or (archives and is_archive_file(mediafile)) \ - or (other and (file_ext.lower() in otherext or 'all' in otherext)): - return True - else: - return False - - -def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): - if otherext is None: - otherext = [] - - files = [] - if not os.path.isdir(path): - if os.path.isfile(path): # Single file downloads. - cur_file = os.path.split(path)[1] - if is_media_file(cur_file, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(path) or not is_min_size(path, min_size): - if delete_ignored == 1: - try: - os.unlink(path) - logger.debug('Ignored file {0} has been removed ...'.format - (cur_file)) - except Exception: - pass - else: - files.append(path) - - return files - - for cur_file in os.listdir(text_type(path)): - full_cur_file = os.path.join(path, cur_file) - - # if it's a folder do it recursively - if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): - files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) - - elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): - if delete_ignored == 1: - try: - os.unlink(full_cur_file) - logger.debug('Ignored file {0} has been removed ...'.format - (cur_file)) - except Exception: - pass - continue - - files.append(full_cur_file) - - return sorted(files, key=len) - - def find_imdbid(dir_name, input_name, omdb_api_key): imdbid = None diff --git a/core/utils/files.py b/core/utils/files.py new file mode 100644 index 00000000..1df3114b --- /dev/null +++ b/core/utils/files.py @@ -0,0 +1,102 @@ +import os + +from six import text_type + +import core +from core import logger +from core.utils.naming import is_sample +from core.utils.paths import get_dir_size + + +def is_min_size(input_name, min_size): + file_name, file_ext = os.path.splitext(os.path.basename(input_name)) + + # audio files we need to check directory size not file size + input_size = os.path.getsize(input_name) + if file_ext in core.AUDIOCONTAINER: + try: + input_size = get_dir_size(os.path.dirname(input_name)) + except Exception: + logger.error('Failed to get file size for {0}'.format(input_name), 'MINSIZE') + return True + + # Ignore files under a certain size + if input_size > min_size * 1048576: + return True + + +def is_archive_file(filename): + """Check if the filename is allowed for the Archive""" + for regext in core.COMPRESSEDCONTAINER: + if regext.search(filename): + return regext.split(filename)[0] + return False + + +def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + file_name, file_ext = os.path.splitext(mediafile) + + try: + # ignore MAC OS's 'resource fork' files + if file_name.startswith('._'): + return False + except Exception: + pass + if (media and file_ext.lower() in core.MEDIACONTAINER) \ + or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ + or (meta and file_ext.lower() in core.METACONTAINER) \ + or (archives and is_archive_file(mediafile)) \ + or (other and (file_ext.lower() in otherext or 'all' in otherext)): + return True + else: + return False + + +def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + files = [] + if not os.path.isdir(path): + if os.path.isfile(path): # Single file downloads. + cur_file = os.path.split(path)[1] + if is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(path) or not is_min_size(path, min_size): + if delete_ignored == 1: + try: + os.unlink(path) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + else: + files.append(path) + + return files + + for cur_file in os.listdir(text_type(path)): + full_cur_file = os.path.join(path, cur_file) + + # if it's a folder do it recursively + if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): + files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) + + elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): + if delete_ignored == 1: + try: + os.unlink(full_cur_file) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + continue + + files.append(full_cur_file) + + return sorted(files, key=len) From 4424e217863eb5a419197f43833146949b5d6909 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:03:27 -0500 Subject: [PATCH 040/406] Streamline is_media_file --- core/utils/files.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/utils/files.py b/core/utils/files.py index 1df3114b..ee9c246c 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -45,14 +45,14 @@ def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, o return False except Exception: pass - if (media and file_ext.lower() in core.MEDIACONTAINER) \ - or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ - or (meta and file_ext.lower() in core.METACONTAINER) \ - or (archives and is_archive_file(mediafile)) \ - or (other and (file_ext.lower() in otherext or 'all' in otherext)): - return True - else: - return False + + return any([ + (media and file_ext.lower() in core.MEDIACONTAINER), + (audio and file_ext.lower() in core.AUDIOCONTAINER), + (meta and file_ext.lower() in core.METACONTAINER), + (archives and is_archive_file(mediafile)), + (other and (file_ext.lower() in otherext or 'all' in otherext)), + ]) def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): From 0f7c74dd78f8035fce80f082e27114b354e29a33 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:22:42 -0500 Subject: [PATCH 041/406] Refactor file type detection to utils.files --- core/utils/__init__.py | 78 +++++------------------------------------- core/utils/files.py | 72 +++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index cdfb9418..c3e95150 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -6,7 +6,6 @@ import os import re import shutil import stat -import time import beets import guessit @@ -14,11 +13,18 @@ import requests from six import text_type import core -from core import extractor, logger +from core import logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii -from core.utils.files import is_archive_file, is_media_file, is_min_size, list_media_files +from core.utils.files import ( + backup_versioned_file, + extract_files, + is_archive_file, + is_media_file, + is_min_size, + list_media_files, +) from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -406,43 +412,6 @@ def find_imdbid(dir_name, input_name, omdb_api_key): return imdbid -def extract_files(src, dst=None, keep_archive=None): - extracted_folder = [] - extracted_archive = [] - - for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): - dir_path = os.path.dirname(inputFile) - full_file_name = os.path.basename(inputFile) - archive_name = os.path.splitext(full_file_name)[0] - archive_name = re.sub(r'part[0-9]+', '', archive_name) - - if dir_path in extracted_folder and archive_name in extracted_archive: - continue # no need to extract this, but keep going to look for other archives and sub directories. - - try: - if extractor.extract(inputFile, dst or dir_path): - extracted_folder.append(dir_path) - extracted_archive.append(archive_name) - except Exception: - logger.error('Extraction failed for: {0}'.format(full_file_name)) - - for folder in extracted_folder: - for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): - full_file_name = os.path.basename(inputFile) - archive_name = os.path.splitext(full_file_name)[0] - archive_name = re.sub(r'part[0-9]+', '', archive_name) - if archive_name not in extracted_archive or keep_archive: - continue # don't remove if we haven't extracted this archive, or if we want to preserve them. - logger.info('Removing extracted archive {0} from folder {1} ...'.format(full_file_name, folder)) - try: - if not os.access(inputFile, os.W_OK): - os.chmod(inputFile, stat.S_IWUSR) - os.remove(inputFile) - time.sleep(1) - except Exception as e: - logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) - - def server_responding(base_url): logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') try: @@ -452,32 +421,3 @@ def server_responding(base_url): except (requests.ConnectionError, requests.exceptions.Timeout): logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') return False - - -def backup_versioned_file(old_file, version): - num_tries = 0 - - new_file = '{old}.v{version}'.format(old=old_file, version=version) - - while not os.path.isfile(new_file): - if not os.path.isfile(old_file): - logger.log(u'Not creating backup, {file} doesn\'t exist'.format(file=old_file), logger.DEBUG) - break - - try: - logger.log(u'Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) - shutil.copy(old_file, new_file) - logger.log(u'Backup done', logger.DEBUG) - break - except Exception as error: - logger.log(u'Error while trying to back up {old} to {new} : {msg}'.format - (old=old_file, new=new_file, msg=error), logger.WARNING) - num_tries += 1 - time.sleep(1) - logger.log(u'Trying again.', logger.DEBUG) - - if num_tries >= 10: - logger.log(u'Unable to back up {old} to {new} please do it manually.'.format(old=old_file, new=new_file), logger.ERROR) - return False - - return True diff --git a/core/utils/files.py b/core/utils/files.py index ee9c246c..8800d239 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,9 +1,13 @@ +import shutil import os +import re +import stat +import time from six import text_type import core -from core import logger +from core import extractor, logger from core.utils.naming import is_sample from core.utils.paths import get_dir_size @@ -100,3 +104,69 @@ def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, files.append(full_cur_file) return sorted(files, key=len) + + +def extract_files(src, dst=None, keep_archive=None): + extracted_folder = [] + extracted_archive = [] + + for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): + dir_path = os.path.dirname(inputFile) + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + + if dir_path in extracted_folder and archive_name in extracted_archive: + continue # no need to extract this, but keep going to look for other archives and sub directories. + + try: + if extractor.extract(inputFile, dst or dir_path): + extracted_folder.append(dir_path) + extracted_archive.append(archive_name) + except Exception: + logger.error('Extraction failed for: {0}'.format(full_file_name)) + + for folder in extracted_folder: + for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + if archive_name not in extracted_archive or keep_archive: + continue # don't remove if we haven't extracted this archive, or if we want to preserve them. + logger.info('Removing extracted archive {0} from folder {1} ...'.format(full_file_name, folder)) + try: + if not os.access(inputFile, os.W_OK): + os.chmod(inputFile, stat.S_IWUSR) + os.remove(inputFile) + time.sleep(1) + except Exception as e: + logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) + + +def backup_versioned_file(old_file, version): + num_tries = 0 + + new_file = '{old}.v{version}'.format(old=old_file, version=version) + + while not os.path.isfile(new_file): + if not os.path.isfile(old_file): + logger.log(u'Not creating backup, {file} doesn\'t exist'.format(file=old_file), logger.DEBUG) + break + + try: + logger.log(u'Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) + shutil.copy(old_file, new_file) + logger.log(u'Backup done', logger.DEBUG) + break + except Exception as error: + logger.log(u'Error while trying to back up {old} to {new} : {msg}'.format + (old=old_file, new=new_file, msg=error), logger.WARNING) + num_tries += 1 + time.sleep(1) + logger.log(u'Trying again.', logger.DEBUG) + + if num_tries >= 10: + logger.log(u'Unable to back up {old} to {new} please do it manually.'.format(old=old_file, new=new_file), logger.ERROR) + return False + + return True From dade3f669804d3899a6e3c9fb0973c1ad352009b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:26:33 -0500 Subject: [PATCH 042/406] Refactor network utils to utils.network --- core/utils/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index c3e95150..2ef8d8e0 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -410,14 +410,3 @@ def find_imdbid(dir_name, input_name, omdb_api_key): logger.warning('Unable to find a imdbID for {0}'.format(input_name)) return imdbid - - -def server_responding(base_url): - logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') - try: - requests.get(base_url, timeout=(60, 120), verify=False) - logger.debug('Server responded at {0}'.format(base_url), 'SERVER') - return True - except (requests.ConnectionError, requests.exceptions.Timeout): - logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') - return False From 6cc3df73b32f62cdfd0bf0e83a4eaab09797948e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:32:52 -0500 Subject: [PATCH 043/406] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 50 +++++------------------------------------- core/utils/paths.py | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2ef8d8e0..bea35b3a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -41,7 +41,11 @@ from core.utils.parsers import ( parse_vuze, ) from core.utils.paths import ( - get_dir_size, make_dir, + flatten_dir, + get_dir_size, + make_dir, + onerror, + remove_dir, remote_dir, remove_empty_folders, remove_read_only, @@ -156,22 +160,7 @@ def category_search(input_directory, input_name, input_category, root, categorie def flatten(output_destination): - logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) - for outputFile in list_media_files(output_destination): - dir_path = os.path.dirname(outputFile) - file_name = os.path.basename(outputFile) - - if dir_path == output_destination: - continue - - target = os.path.join(output_destination, file_name) - - try: - shutil.move(outputFile, target) - except Exception: - logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') - - remove_empty_folders(output_destination) # Cleanup empty directories + return flatten_dir(output_destination, list_media_files(output_destination)) def get_dirs(section, subsection, link='hard'): @@ -280,33 +269,6 @@ def get_dirs(section, subsection, link='hard'): return list(set(to_return)) -def onerror(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise Exception - - -def remove_dir(dir_name): - logger.info('Deleting {0}'.format(dir_name)) - try: - shutil.rmtree(text_type(dir_name), onerror=onerror) - except Exception: - logger.error('Unable to delete folder {0}'.format(dir_name)) - - def clean_dir(path, section, subsection): cfg = dict(core.CFG[section][subsection]) if not os.path.exists(path): diff --git a/core/utils/paths.py b/core/utils/paths.py index 31308650..948287b7 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -2,6 +2,7 @@ from functools import partial import os import re +import shutil import stat from six import text_type @@ -10,6 +11,33 @@ import core from core import logger +def onerror(func, path, exc_info): + """ + Error handler for ``shutil.rmtree``. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Usage : ``shutil.rmtree(path, onerror=onerror)`` + """ + if not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise Exception + + +def remove_dir(dir_name): + logger.info('Deleting {0}'.format(dir_name)) + try: + shutil.rmtree(text_type(dir_name), onerror=onerror) + except Exception: + logger.error('Unable to delete folder {0}'.format(dir_name)) + + def make_dir(path): if not os.path.isdir(path): try: @@ -78,3 +106,22 @@ def remove_read_only(filename): os.chmod(filename, stat.S_IWRITE) except Exception: logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) + + +def flatten_dir(destination, files): + logger.info('FLATTEN: Flattening directory: {0}'.format(destination)) + for outputFile in files: + dir_path = os.path.dirname(outputFile) + file_name = os.path.basename(outputFile) + + if dir_path == destination: + continue + + target = os.path.join(destination, file_name) + + try: + shutil.move(outputFile, target) + except Exception: + logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') + + remove_empty_folders(destination) # Cleanup empty directories From 36932e25c6954a84afbcd053da6bd2775aa06cda Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:36:52 -0500 Subject: [PATCH 044/406] Fix clean_dir for Python 3 TypeError when testing str > int --- core/utils/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index bea35b3a..8ab397b7 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -283,7 +283,8 @@ def clean_dir(path, section, subsection): try: num_files = len(list_media_files(path, min_size=min_size, delete_ignored=delete_ignored)) except Exception: - num_files = 'unknown' + num_files = 0 + if num_files > 0: logger.info( 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, num_files), From e44c0bb56ab0564c328d4ac074d308272afcd49d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:40:05 -0500 Subject: [PATCH 045/406] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 45 ++++++++++++------------------------------ core/utils/paths.py | 24 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 8ab397b7..2dcaaa9a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,8 +4,6 @@ from __future__ import print_function, unicode_literals import os import re -import shutil -import stat import beets import guessit @@ -41,12 +39,13 @@ from core.utils.parsers import ( parse_vuze, ) from core.utils.paths import ( + clean_directory, flatten_dir, get_dir_size, make_dir, onerror, - remove_dir, remote_dir, + remove_dir, remove_empty_folders, remove_read_only, ) @@ -163,6 +162,17 @@ def flatten(output_destination): return flatten_dir(output_destination, list_media_files(output_destination)) +def clean_dir(path, section, subsection): + cfg = dict(core.CFG[section][subsection]) + min_size = int(cfg.get('minSize', 0)) + delete_ignored = int(cfg.get('delete_ignored', 0)) + try: + files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) + except Exception: + files = [] + return clean_directory(path, files) + + def get_dirs(section, subsection, link='hard'): to_return = [] @@ -269,35 +279,6 @@ def get_dirs(section, subsection, link='hard'): return list(set(to_return)) -def clean_dir(path, section, subsection): - cfg = dict(core.CFG[section][subsection]) - if not os.path.exists(path): - logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') - return - if core.FORCE_CLEAN and not core.FAILED: - logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') - remove_dir(path) - return - min_size = int(cfg.get('minSize', 0)) - delete_ignored = int(cfg.get('delete_ignored', 0)) - try: - num_files = len(list_media_files(path, min_size=min_size, delete_ignored=delete_ignored)) - except Exception: - num_files = 0 - - if num_files > 0: - logger.info( - 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, num_files), - 'CLEANDIRS') - return - - logger.info('Directory {0} has been processed, removing ...'.format(path), 'CLEANDIRS') - try: - shutil.rmtree(path, onerror=onerror) - except Exception: - logger.error('Unable to delete directory {0}'.format(path)) - - def find_imdbid(dir_name, input_name, omdb_api_key): imdbid = None diff --git a/core/utils/paths.py b/core/utils/paths.py index 948287b7..5576206f 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -125,3 +125,27 @@ def flatten_dir(destination, files): logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') remove_empty_folders(destination) # Cleanup empty directories + + +def clean_directory(path, files): + if not os.path.exists(path): + logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') + return + + if core.FORCE_CLEAN and not core.FAILED: + logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') + remove_dir(path) + return + + if files: + logger.info( + 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, len(files)), + 'CLEANDIRS', + ) + return + + logger.info('Directory {0} has been processed, removing ...'.format(path), 'CLEANDIRS') + try: + shutil.rmtree(path, onerror=onerror) + except Exception: + logger.error('Unable to delete directory {0}'.format(path)) From 03cb11dae3107ac3a16cf518707c9c6cdd54a223 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:52:22 -0500 Subject: [PATCH 046/406] Refactor identification utils from utils to utils.identification --- core/utils/__init__.py | 173 +-------------------------------- core/utils/identification.py | 181 +++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 172 deletions(-) create mode 100644 core/utils/identification.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2dcaaa9a..2d5954e6 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import os -import re import beets import guessit @@ -23,6 +22,7 @@ from core.utils.files import ( is_min_size, list_media_files, ) +from core.utils.identification import find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -64,100 +64,6 @@ requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() -def category_search(input_directory, input_name, input_category, root, categories): - tordir = False - - try: - input_name = input_name.encode(core.SYS_ENCODING) - except Exception: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except Exception: - pass - - if input_directory is None: # =Nothing to process here. - return input_directory, input_name, input_category, root - - pathlist = os.path.normpath(input_directory).split(os.sep) - - if input_category and input_category in pathlist: - logger.debug('SEARCH: Found the Category: {0} in directory structure'.format(input_category)) - elif input_category: - logger.debug('SEARCH: Could not find the category: {0} in the directory structure'.format(input_category)) - else: - try: - input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. - logger.debug('SEARCH: Found Category: {0} in directory structure'.format(input_category)) - except IndexError: - input_category = '' - logger.debug('SEARCH: Could not find a category in the directory structure') - if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file - if not input_name: - input_name = os.path.split(os.path.normpath(input_directory))[1] - return input_directory, input_name, input_category, root - - if input_category and os.path.isdir(os.path.join(input_directory, input_category)): - logger.info( - 'SEARCH: Found category directory {0} in input directory directory {1}'.format(input_category, input_directory)) - input_directory = os.path.join(input_directory, input_category) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - if input_name and os.path.isdir(os.path.join(input_directory, input_name)): - logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format(input_name, input_directory)) - input_directory = os.path.join(input_directory, input_name) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): - logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format( - sanitize_name(input_name), input_directory)) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): - logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(input_name, input_directory)) - input_directory = os.path.join(input_directory, input_name) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): - logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format( - sanitize_name(input_name), input_directory)) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - - imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. - if imdbid and '.cp(tt' not in input_name: - input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP - tordir = True - - if input_category and not tordir: - try: - index = pathlist.index(input_category) - if index + 1 < len(pathlist): - tordir = True - logger.info('SEARCH: Found a unique directory {0} in the category directory'.format - (pathlist[index + 1])) - if not input_name: - input_name = pathlist[index + 1] - except ValueError: - pass - - if input_name and not tordir: - if input_name in pathlist or sanitize_name(input_name) in pathlist: - logger.info('SEARCH: Found torrent directory {0} in the directory structure'.format(input_name)) - tordir = True - else: - root = 1 - if not tordir: - root = 2 - - if root > 0: - logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') - logger.info('SEARCH: We will try and determine which files to process, individually') - - return input_directory, input_name, input_category, root - - def flatten(output_destination): return flatten_dir(output_destination, list_media_files(output_destination)) @@ -277,80 +183,3 @@ def get_dirs(section, subsection, link='hard'): logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) return list(set(to_return)) - - -def find_imdbid(dir_name, input_name, omdb_api_key): - imdbid = None - - logger.info('Attemping imdbID lookup for {0}'.format(input_name)) - - # find imdbid in dirName - logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7})', dir_name + input_name) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}]'.format(imdbid)) - return imdbid - if os.path.isdir(dir_name): - for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7})', file) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}] via file name'.format(imdbid)) - return imdbid - if 'NZBPR__DNZB_MOREINFO' in os.environ: - dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') - if dnzb_more_info != '': - regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) - m = regex.match(dnzb_more_info) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}] from DNZB-MoreInfo'.format(imdbid)) - return imdbid - logger.info('Searching IMDB for imdbID ...') - try: - guess = guessit.guessit(input_name) - except Exception: - guess = None - if guess: - # Movie Title - title = None - if 'title' in guess: - title = guess['title'] - - # Movie Year - year = None - if 'year' in guess: - year = guess['year'] - - url = 'http://www.omdbapi.com' - - if not omdb_api_key: - logger.info('Unable to determine imdbID: No api key provided for ombdapi.com.') - return - - logger.debug('Opening URL: {0}'.format(url)) - - try: - r = requests.get(url, params={'apikey': omdb_api_key, 'y': year, 't': title}, - verify=False, timeout=(60, 300)) - except requests.ConnectionError: - logger.error('Unable to open URL {0}'.format(url)) - return - - try: - results = r.json() - except Exception: - logger.error('No json data returned from omdbapi.com') - - try: - imdbid = results['imdbID'] - except Exception: - logger.error('No imdbID returned from omdbapi.com') - - if imdbid: - logger.info('Found imdbID [{0}]'.format(imdbid)) - return imdbid - - logger.warning('Unable to find a imdbID for {0}'.format(input_name)) - return imdbid diff --git a/core/utils/identification.py b/core/utils/identification.py new file mode 100644 index 00000000..c2dc0352 --- /dev/null +++ b/core/utils/identification.py @@ -0,0 +1,181 @@ +import os +import re + +import guessit +import requests +from six import text_type + +import core +from core import logger +from core.utils.naming import sanitize_name + + +def find_imdbid(dir_name, input_name, omdb_api_key): + imdbid = None + + logger.info('Attemping imdbID lookup for {0}'.format(input_name)) + + # find imdbid in dirName + logger.info('Searching folder and file names for imdbID ...') + m = re.search(r'(tt\d{7})', dir_name + input_name) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + if os.path.isdir(dir_name): + for file in os.listdir(text_type(dir_name)): + m = re.search(r'(tt\d{7})', file) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] via file name'.format(imdbid)) + return imdbid + if 'NZBPR__DNZB_MOREINFO' in os.environ: + dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') + if dnzb_more_info != '': + regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) + m = regex.match(dnzb_more_info) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] from DNZB-MoreInfo'.format(imdbid)) + return imdbid + logger.info('Searching IMDB for imdbID ...') + try: + guess = guessit.guessit(input_name) + except Exception: + guess = None + if guess: + # Movie Title + title = None + if 'title' in guess: + title = guess['title'] + + # Movie Year + year = None + if 'year' in guess: + year = guess['year'] + + url = 'http://www.omdbapi.com' + + if not omdb_api_key: + logger.info('Unable to determine imdbID: No api key provided for ombdapi.com.') + return + + logger.debug('Opening URL: {0}'.format(url)) + + try: + r = requests.get(url, params={'apikey': omdb_api_key, 'y': year, 't': title}, + verify=False, timeout=(60, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url)) + return + + try: + results = r.json() + except Exception: + logger.error('No json data returned from omdbapi.com') + + try: + imdbid = results['imdbID'] + except Exception: + logger.error('No imdbID returned from omdbapi.com') + + if imdbid: + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + + logger.warning('Unable to find a imdbID for {0}'.format(input_name)) + return imdbid + + +def category_search(input_directory, input_name, input_category, root, categories): + tordir = False + + try: + input_name = input_name.encode(core.SYS_ENCODING) + except Exception: + pass + try: + input_directory = input_directory.encode(core.SYS_ENCODING) + except Exception: + pass + + if input_directory is None: # =Nothing to process here. + return input_directory, input_name, input_category, root + + pathlist = os.path.normpath(input_directory).split(os.sep) + + if input_category and input_category in pathlist: + logger.debug('SEARCH: Found the Category: {0} in directory structure'.format(input_category)) + elif input_category: + logger.debug('SEARCH: Could not find the category: {0} in the directory structure'.format(input_category)) + else: + try: + input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. + logger.debug('SEARCH: Found Category: {0} in directory structure'.format(input_category)) + except IndexError: + input_category = '' + logger.debug('SEARCH: Could not find a category in the directory structure') + if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file + if not input_name: + input_name = os.path.split(os.path.normpath(input_directory))[1] + return input_directory, input_name, input_category, root + + if input_category and os.path.isdir(os.path.join(input_directory, input_category)): + logger.info( + 'SEARCH: Found category directory {0} in input directory directory {1}'.format(input_category, input_directory)) + input_directory = os.path.join(input_directory, input_category) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + if input_name and os.path.isdir(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + + imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. + if imdbid and '.cp(tt' not in input_name: + input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP + tordir = True + + if input_category and not tordir: + try: + index = pathlist.index(input_category) + if index + 1 < len(pathlist): + tordir = True + logger.info('SEARCH: Found a unique directory {0} in the category directory'.format + (pathlist[index + 1])) + if not input_name: + input_name = pathlist[index + 1] + except ValueError: + pass + + if input_name and not tordir: + if input_name in pathlist or sanitize_name(input_name) in pathlist: + logger.info('SEARCH: Found torrent directory {0} in the directory structure'.format(input_name)) + tordir = True + else: + root = 1 + if not tordir: + root = 2 + + if root > 0: + logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') + logger.info('SEARCH: We will try and determine which files to process, individually') + + return input_directory, input_name, input_category, root From e67f29cb7bc4d3237ec407edfa35671583de21ef Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:58:02 -0500 Subject: [PATCH 047/406] Flatten get_dirs function --- core/utils/__init__.py | 165 +++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2d5954e6..724c85fe 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -79,94 +79,95 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) +def process_dir(path, link): + folders = [] + + logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) + sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + # search for single files and move them into their own folder for post-processing + for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isfile(os.path.join(path, o))]: + if len(sync) > 0: + break + if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: + continue + try: + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # removeEmptyFolders(path, removeRoot=False) + + if os.listdir(text_type(path)): + for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isdir(os.path.join(path, o))]: + sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: + continue + folders.extend([directory]) + return folders + + def get_dirs(section, subsection, link='hard'): to_return = [] - def process_dir(path): - folders = [] - - logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - # search for single files and move them into their own folder for post-processing - for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isfile(os.path.join(path, o))]: - if len(sync) > 0: - break - if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: - continue - try: - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) - - # removeEmptyFolders(path, removeRoot=False) - - if os.listdir(text_type(path)): - for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isdir(os.path.join(path, o))]: - sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: - continue - folders.extend([directory]) - return folders - try: watch_dir = os.path.join(core.CFG[section][subsection]['watch_dir'], subsection) if os.path.exists(watch_dir): - to_return.extend(process_dir(watch_dir)) + to_return.extend(process_dir(watch_dir, link)) elif os.path.exists(core.CFG[section][subsection]['watch_dir']): - to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'])) + to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'], link)) except Exception as e: logger.error('Failed to add directories from {0} for post-processing: {1}'.format (core.CFG[section][subsection]['watch_dir'], e)) @@ -175,7 +176,7 @@ def get_dirs(section, subsection, link='hard'): try: output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) if os.path.exists(output_directory): - to_return.extend(process_dir(output_directory)) + to_return.extend(process_dir(output_directory, link)) except Exception as e: logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) From 8d458f10ac370c334a2473632af8ff5617d79b48 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:02:54 -0500 Subject: [PATCH 048/406] Refactor get_dirs --- core/utils/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 724c85fe..f0c822cc 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -162,15 +162,16 @@ def process_dir(path, link): def get_dirs(section, subsection, link='hard'): to_return = [] + watch_directory = core.CFG[section][subsection]['watch_dir'] + directory = os.path.join(watch_directory, subsection) + + if not os.path.exists(directory): + directory = watch_directory + try: - watch_dir = os.path.join(core.CFG[section][subsection]['watch_dir'], subsection) - if os.path.exists(watch_dir): - to_return.extend(process_dir(watch_dir, link)) - elif os.path.exists(core.CFG[section][subsection]['watch_dir']): - to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'], link)) + to_return.extend(process_dir(directory, link)) except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format - (core.CFG[section][subsection]['watch_dir'], e)) + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) if core.USELINK == 'move': try: From cb422a0cea75a90fbcbf6089510cb9c546cb8d2f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:09:11 -0500 Subject: [PATCH 049/406] Flatten process_dir --- core/utils/__init__.py | 110 +++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index f0c822cc..665453fe 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,7 +4,7 @@ from __future__ import print_function, unicode_literals import os -import beets +import beets.mediafile import guessit import requests from six import text_type @@ -79,6 +79,61 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) +def move_file(mediafile, path, link): + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) + + def process_dir(path, link): folders = [] @@ -92,58 +147,7 @@ def process_dir(path, link): if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: continue try: - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) + move_file(mediafile, path, link) except Exception as e: logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) From a888d741d3f11ef66cedcf4d4eb9eecd65acb4e0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:12:17 -0500 Subject: [PATCH 050/406] Refactor file type detection to utils.files --- core/utils/__init__.py | 58 +------------------------------------- core/utils/files.py | 64 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 60 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 665453fe..fba6002d 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,8 +4,6 @@ from __future__ import print_function, unicode_literals import os -import beets.mediafile -import guessit import requests from six import text_type @@ -21,6 +19,7 @@ from core.utils.files import ( is_media_file, is_min_size, list_media_files, + move_file, ) from core.utils.identification import find_imdbid from core.utils.links import copy_link, replace_links @@ -79,61 +78,6 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) -def move_file(mediafile, path, link): - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) - - def process_dir(path, link): folders = [] diff --git a/core/utils/files.py b/core/utils/files.py index 8800d239..b753af0c 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,15 +1,73 @@ -import shutil import os import re +import shutil import stat import time +import beets.mediafile +import guessit from six import text_type import core from core import extractor, logger -from core.utils.naming import is_sample -from core.utils.paths import get_dir_size +from core.utils.links import copy_link +from core.utils.naming import is_sample, sanitize_name +from core.utils.paths import get_dir_size, make_dir + + +def move_file(mediafile, path, link): + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) def is_min_size(input_name, min_size): From 648ecd40487c6ebcc24f68cde374e34b807183d5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:36:32 -0500 Subject: [PATCH 051/406] Refactor process_dir to use generators --- core/utils/__init__.py | 69 +++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fba6002d..fb1ca12c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -82,28 +82,61 @@ def process_dir(path, link): folders = [] logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + dir_contents = os.listdir(text_type(path)) + # search for single files and move them into their own folder for post-processing - for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isfile(os.path.join(path, o))]: - if len(sync) > 0: - break - if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: - continue - try: - move_file(mediafile, path, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # Generate list of sync files + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + + # Generate a list of file paths + filepaths = ( + os.path.join(path, item) for item in dir_contents + if item not in ['Thumbs.db', 'thumbs.db'] + ) + + # Generate a list of media files + mediafiles = ( + item for item in filepaths + if os.path.isfile(item) + ) + + if any(sync_files): + logger.info('') + else: + for mediafile in mediafiles: + try: + move_file(mediafile, path, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) # removeEmptyFolders(path, removeRoot=False) - if os.listdir(text_type(path)): - for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isdir(os.path.join(path, o))]: - sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: - continue - folders.extend([directory]) + # Generate all path contents + path_contents = ( + os.path.join(path, item) + for item in os.listdir(text_type(path)) + ) + + # Generate all directories from path contents + directories = ( + path for path in path_contents + if os.path.isdir(path) + ) + + for directory in directories: + dir_contents = os.listdir(directory) + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + if not any(dir_contents) or any(sync_files): + continue + folders.append(directory) + return folders From 383eb5eaf2142d54d50393e9d4b8b2358fd1c893 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 12:04:13 -0500 Subject: [PATCH 052/406] Refactor identification utils from utils to utils.identification --- core/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fb1ca12c..a9c4e6cf 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -21,7 +21,7 @@ from core.utils.files import ( list_media_files, move_file, ) -from core.utils.identification import find_imdbid +from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up From bd5b970bc7350fd7d5d62b8ace95357c88bc21b0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 12:05:26 -0500 Subject: [PATCH 053/406] Refactor network utils to utils.network --- core/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index a9c4e6cf..176c92db 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -24,7 +24,7 @@ from core.utils.files import ( from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name -from core.utils.network import find_download, test_connection, wake_on_lan, wake_up +from core.utils.network import find_download, test_connection, wake_on_lan, wake_up, server_responding from core.utils.notifications import plex_update from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( From e89bbcf9bec4b1d4fe1659d63c1b6ef14e2065d5 Mon Sep 17 00:00:00 2001 From: TheHolyRoger <39387497+TheHolyRoger@users.noreply.github.com> Date: Fri, 11 Jan 2019 14:38:20 +0000 Subject: [PATCH 054/406] hotfix/processresult bug --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 0e14e60d..4971c13d 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -358,7 +358,7 @@ def main(args): results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) - if results[0] != 0: + if results.status_code != 0: logger.error('A problem was reported when trying to perform a manual run for {0}:{1}.'.format (section, subsection)) result = results From 3a2ed4bc57f2ce7a82f004ef555f791319047a86 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 13 Jan 2019 19:41:04 +1300 Subject: [PATCH 055/406] fixed manual Torrent run result parsing. Fixes #1520 --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 0e14e60d..4971c13d 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -358,7 +358,7 @@ def main(args): results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) - if results[0] != 0: + if results.status_code != 0: logger.error('A problem was reported when trying to perform a manual run for {0}:{1}.'.format (section, subsection)) result = results From b5b48082939648ff09446f6773f02ea9f7c6882f Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 13 Jan 2019 20:10:45 +1300 Subject: [PATCH 056/406] update version details for next release. --- .bumpversion.cfg | 2 +- README.md | 2 +- changelog.txt | 28 ++++++++++++++++++++++++++++ core/__init__.py | 5 ++--- setup.py | 2 +- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index feb03ca3..f9e4c7da 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.5 +current_version = 12.0.6 commit = True tag = False diff --git a/README.md b/README.md index 882cf785..bf381998 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.5 +nzbToMedia v12.0.6 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/changelog.txt b/changelog.txt index 9ad64594..380dca71 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,33 @@ Change_LOG / History +V12.0.6 + +Hotfix for Manual Torrent run results. + +V12.0.5 + +Proper fix for source cleaner + +V12.0.4 + +Hotfix missed commit for source cleaner + +V12.0.3 + +Hotfix cleaning for source installs + +V12.0.2 + +Fix missed ProcessResult + +V12.0.1 + +Added Python 3 support +Updated all dependencies +Major code refactoring +Various bug fixes +Hotfix NZBGet not working without comment + V12.0.0 NOTE: diff --git a/core/__init__.py b/core/__init__.py index 68d4a95c..9958308b 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -52,7 +52,7 @@ from core.utils import ( resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -__version__ = '12.0.5' +__version__ = '12.0.6' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] @@ -101,7 +101,7 @@ SYS_ENCODING = None FAILED = False AUTO_UPDATE = None -NZBTOMEDIA_VERSION = None +NZBTOMEDIA_VERSION = __version__ NEWEST_VERSION = None NEWEST_VERSION_STRING = None VERSION_NOTIFY = None @@ -328,7 +328,6 @@ def initialize(section=None): main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) # Set Version and GIT variables - NZBTOMEDIA_VERSION = '11.06' VERSION_NOTIFY = int(CFG['General']['version_notify']) AUTO_UPDATE = int(CFG['General']['auto_update']) GIT_REPO = 'nzbToMedia' diff --git a/setup.py b/setup.py index e8cb7477..665c5705 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.5', + version='12.0.6', license='GPLv3', description='Efficient on demand post processing', long_description=""" From c4be677a6209a1530942538ee21f462d1553bd19 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:22:26 -0500 Subject: [PATCH 057/406] Fix git subprocess --- core/version_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version_check.py b/core/version_check.py index 8fac2d37..53945e1f 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -116,7 +116,7 @@ class GitUpdateManager(UpdateManager): test_cmd = 'version' if core.GIT_PATH: - main_git = '\'{git}\''.format(git=core.GIT_PATH) + main_git = '"{git}"'.format(git=core.GIT_PATH) else: main_git = 'git' From 7185e0b31ba84cb1bae3174d8aa64ab7291a97e2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:35:41 -0500 Subject: [PATCH 058/406] Add docstring --- cleanup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cleanup.py b/cleanup.py index 02bdf648..3e1559cf 100644 --- a/cleanup.py +++ b/cleanup.py @@ -37,6 +37,13 @@ class WorkingDirectory(object): def module_path(module=__file__, parent=False): + """ + Detect path for a module. + + :param module: The module who's path is being detected. Defaults to current module. + :param parent: True to return the parent folder of the current module. + :return: The absolute normalized path to the module or its parent. + """ try: path = module.__file__ except AttributeError: @@ -122,6 +129,12 @@ def clean_folders(*paths): def force_clean_folder(path, required): + """ + Force clean a folder and exclude any required subfolders. + + :param path: Target folder to remove subfolders + :param required: Keep only the required subfolders + """ root, dirs, files = next(os.walk(path)) required = sorted(required) if required: From 844c1d15e98f43df705563933e2455d919e17550 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:47:22 -0500 Subject: [PATCH 059/406] Fix cleanup script output --- cleanup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cleanup.py b/cleanup.py index 3e1559cf..b78df372 100644 --- a/cleanup.py +++ b/cleanup.py @@ -151,6 +151,11 @@ def force_clean_folder(path, required): def clean(paths): """Clean up bytecode and obsolete folders.""" + def _report_error(msg): + print('WARNING: Automatic cleanup could not be executed.') + print(' If errors occur, manual cleanup may be required.') + print('REASON : {}'.format(msg)) + with WorkingDirectory(module_path()) as cwd: if cwd.working_directory != cwd.original_directory: print('Changing to directory:', cwd.working_directory) @@ -159,7 +164,7 @@ def clean(paths): try: result = clean_bytecode() except SystemExit as error: - print(error) + _report_error(error) else: print(result or 'No bytecode to clean') @@ -168,7 +173,7 @@ def clean(paths): try: result = clean_folders(*paths) except SystemExit as error: - print(error) + _report_error(error) else: print(result or 'No folders to clean\n') else: @@ -176,7 +181,7 @@ def clean(paths): try: items = paths.items() except AttributeError: - print('Failed to clean, no subfolder structure given') + _report_error('Failed to clean, no subfolder structure given') else: for folder, subfolders in items: print('\nForce cleaning folder:', folder) From aa057e65d5809b8b261a32b47f413cc9b655dbb6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:55:43 -0500 Subject: [PATCH 060/406] Refactor common utils to utils.common --- core/utils/__init__.py | 122 +---------------------------------------- core/utils/common.py | 114 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 120 deletions(-) create mode 100644 core/utils/common.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 176c92db..ee2eba05 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -1,15 +1,9 @@ # coding=utf-8 -from __future__ import print_function, unicode_literals - -import os - import requests -from six import text_type -import core -from core import logger from core.utils import shutil_custom +from core.utils.common import clean_dir, flatten, get_dirs, process_dir from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii from core.utils.files import ( @@ -24,7 +18,7 @@ from core.utils.files import ( from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name -from core.utils.network import find_download, test_connection, wake_on_lan, wake_up, server_responding +from core.utils.network import find_download, server_responding, test_connection, wake_on_lan, wake_up from core.utils.notifications import plex_update from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( @@ -52,117 +46,5 @@ from core.utils.processes import RunningProcess from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent -try: - import jaraco -except ImportError: - if os.name == 'nt': - raise - requests.packages.urllib3.disable_warnings() - shutil_custom.monkey_patch() - - -def flatten(output_destination): - return flatten_dir(output_destination, list_media_files(output_destination)) - - -def clean_dir(path, section, subsection): - cfg = dict(core.CFG[section][subsection]) - min_size = int(cfg.get('minSize', 0)) - delete_ignored = int(cfg.get('delete_ignored', 0)) - try: - files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) - except Exception: - files = [] - return clean_directory(path, files) - - -def process_dir(path, link): - folders = [] - - logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - dir_contents = os.listdir(text_type(path)) - - # search for single files and move them into their own folder for post-processing - - # Generate list of sync files - sync_files = ( - item for item in dir_contents - if os.path.splitext(item)[1] in ['.!sync', '.bts'] - ) - - # Generate a list of file paths - filepaths = ( - os.path.join(path, item) for item in dir_contents - if item not in ['Thumbs.db', 'thumbs.db'] - ) - - # Generate a list of media files - mediafiles = ( - item for item in filepaths - if os.path.isfile(item) - ) - - if any(sync_files): - logger.info('') - else: - for mediafile in mediafiles: - try: - move_file(mediafile, path, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) - - # removeEmptyFolders(path, removeRoot=False) - - # Generate all path contents - path_contents = ( - os.path.join(path, item) - for item in os.listdir(text_type(path)) - ) - - # Generate all directories from path contents - directories = ( - path for path in path_contents - if os.path.isdir(path) - ) - - for directory in directories: - dir_contents = os.listdir(directory) - sync_files = ( - item for item in dir_contents - if os.path.splitext(item)[1] in ['.!sync', '.bts'] - ) - if not any(dir_contents) or any(sync_files): - continue - folders.append(directory) - - return folders - - -def get_dirs(section, subsection, link='hard'): - to_return = [] - - watch_directory = core.CFG[section][subsection]['watch_dir'] - directory = os.path.join(watch_directory, subsection) - - if not os.path.exists(directory): - directory = watch_directory - - try: - to_return.extend(process_dir(directory, link)) - except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) - - if core.USELINK == 'move': - try: - output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) - if os.path.exists(output_directory): - to_return.extend(process_dir(output_directory, link)) - except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) - - if not to_return: - logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) - - return list(set(to_return)) diff --git a/core/utils/common.py b/core/utils/common.py new file mode 100644 index 00000000..f352fb87 --- /dev/null +++ b/core/utils/common.py @@ -0,0 +1,114 @@ + +import os.path + +from six import text_type + +import core +from core import logger +from core.utils.files import list_media_files, move_file +from core.utils.paths import clean_directory, flatten_dir + + +def flatten(output_destination): + return flatten_dir(output_destination, list_media_files(output_destination)) + + +def clean_dir(path, section, subsection): + cfg = dict(core.CFG[section][subsection]) + min_size = int(cfg.get('minSize', 0)) + delete_ignored = int(cfg.get('delete_ignored', 0)) + try: + files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) + except Exception: + files = [] + return clean_directory(path, files) + + +def process_dir(path, link): + folders = [] + + logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) + dir_contents = os.listdir(text_type(path)) + + # search for single files and move them into their own folder for post-processing + + # Generate list of sync files + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + + # Generate a list of file paths + filepaths = ( + os.path.join(path, item) for item in dir_contents + if item not in ['Thumbs.db', 'thumbs.db'] + ) + + # Generate a list of media files + mediafiles = ( + item for item in filepaths + if os.path.isfile(item) + ) + + if any(sync_files): + logger.info('') + else: + for mediafile in mediafiles: + try: + move_file(mediafile, path, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # removeEmptyFolders(path, removeRoot=False) + + # Generate all path contents + path_contents = ( + os.path.join(path, item) + for item in os.listdir(text_type(path)) + ) + + # Generate all directories from path contents + directories = ( + path for path in path_contents + if os.path.isdir(path) + ) + + for directory in directories: + dir_contents = os.listdir(directory) + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + if not any(dir_contents) or any(sync_files): + continue + folders.append(directory) + + return folders + + +def get_dirs(section, subsection, link='hard'): + to_return = [] + + watch_directory = core.CFG[section][subsection]['watch_dir'] + directory = os.path.join(watch_directory, subsection) + + if not os.path.exists(directory): + directory = watch_directory + + try: + to_return.extend(process_dir(directory, link)) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) + + if core.USELINK == 'move': + try: + output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) + if os.path.exists(output_directory): + to_return.extend(process_dir(output_directory, link)) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) + + if not to_return: + logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) + + return list(set(to_return)) From d960c432eb982f229433805475d1de54714b98a0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:02:36 -0500 Subject: [PATCH 061/406] Refactor rchmod to utils.paths.rchmod --- core/__init__.py | 38 +++++++++++++++++++++----------------- core/utils/__init__.py | 1 + core/utils/paths.py | 13 +++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 9958308b..7dcd4fd9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -46,10 +46,27 @@ from six.moves import reload_module from core import logger, main_db, version_check, databases, transcoder from core.configuration import config from core.utils import ( - RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, - create_torrent_class, extract_files, flatten, get_dirs, get_download_info, - list_media_files, make_dir, parse_args, pause_torrent, remove_torrent, - resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, + RunningProcess, + category_search, + clean_dir, + copy_link, + create_torrent_class, + extract_files, + flatten, + get_dirs, + get_download_info, + list_media_files, + make_dir, + parse_args, + pause_torrent, + rchmod, + remove_dir, + remove_read_only, + remove_torrent, + resume_torrent, + sanitize_name, + update_download_info_status, + wake_up, ) __version__ = '12.0.6' @@ -866,16 +883,3 @@ def restart(): status = p.returncode os._exit(status) - - -def rchmod(path, mod): - logger.log('Changing file mode of {0} to {1}'.format(path, oct(mod))) - os.chmod(path, mod) - if not os.path.isdir(path): - return # Skip files - - for root, dirs, files in os.walk(path): - for d in dirs: - os.chmod(os.path.join(root, d), mod) - for f in files: - os.chmod(os.path.join(root, f), mod) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index ee2eba05..99bb0376 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -37,6 +37,7 @@ from core.utils.paths import ( get_dir_size, make_dir, onerror, + rchmod, remote_dir, remove_dir, remove_empty_folders, diff --git a/core/utils/paths.py b/core/utils/paths.py index 5576206f..ec99a246 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -149,3 +149,16 @@ def clean_directory(path, files): shutil.rmtree(path, onerror=onerror) except Exception: logger.error('Unable to delete directory {0}'.format(path)) + + +def rchmod(path, mod): + logger.log('Changing file mode of {0} to {1}'.format(path, oct(mod))) + os.chmod(path, mod) + if not os.path.isdir(path): + return # Skip files + + for root, dirs, files in os.walk(path): + for d in dirs: + os.chmod(os.path.join(root, d), mod) + for f in files: + os.chmod(os.path.join(root, f), mod) From b6672ccf09825fc7cdf2d589ac3ce022fd2bc548 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:06:05 -0500 Subject: [PATCH 062/406] Refactor restart to utils.processes.restart --- core/__init__.py | 21 +-------------------- core/utils/__init__.py | 2 +- core/utils/processes.py | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 7dcd4fd9..593c4a5a 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -63,6 +63,7 @@ from core.utils import ( remove_dir, remove_read_only, remove_torrent, + restart, resume_torrent, sanitize_name, update_download_info_status, @@ -863,23 +864,3 @@ def initialize(section=None): # finished initalizing return True - - -def restart(): - install_type = version_check.CheckVersion().install_type - - status = 0 - popen_list = [] - - if install_type in ('git', 'source'): - popen_list = [sys.executable, APP_FILENAME] - - if popen_list: - popen_list += SYS_ARGV - logger.log(u'Restarting nzbToMedia with {args}'.format(args=popen_list)) - logger.close() - p = subprocess.Popen(popen_list, cwd=os.getcwd()) - p.wait() - status = p.returncode - - os._exit(status) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 99bb0376..0083fbbd 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -43,7 +43,7 @@ from core.utils.paths import ( remove_empty_folders, remove_read_only, ) -from core.utils.processes import RunningProcess +from core.utils.processes import RunningProcess, restart from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent diff --git a/core/utils/processes.py b/core/utils/processes.py index 6513a12c..6edcfa44 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -1,7 +1,10 @@ import os import socket +import subprocess +import sys import core +from core import logger, version_check, APP_FILENAME, SYS_ARGV if os.name == 'nt': from win32event import CreateMutex @@ -90,3 +93,23 @@ if os.name == 'nt': RunningProcess = WindowsProcess else: RunningProcess = PosixProcess + + +def restart(): + install_type = version_check.CheckVersion().install_type + + status = 0 + popen_list = [] + + if install_type in ('git', 'source'): + popen_list = [sys.executable, APP_FILENAME] + + if popen_list: + popen_list += SYS_ARGV + logger.log(u'Restarting nzbToMedia with {args}'.format(args=popen_list)) + logger.close() + p = subprocess.Popen(popen_list, cwd=os.getcwd()) + p.wait() + status = p.returncode + + os._exit(status) From d9436603abb7ada9895f0b717e23a8b4be7e2b62 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:11:31 -0500 Subject: [PATCH 063/406] =?UTF-8?q?Bump=20version:=2012.0.6=20=E2=86=92=20?= =?UTF-8?q?12.0.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f9e4c7da..a1d61c63 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.6 +current_version = 12.0.7 commit = True tag = False diff --git a/README.md b/README.md index bf381998..c78ffe83 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.6 +nzbToMedia v12.0.7 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 593c4a5a..e7280ded 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -70,7 +70,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.6' +__version__ = '12.0.7' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 665c5705..941b4477 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.6', + version='12.0.7', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 30872db797e813f9a0906ee41e59a0af331c6f88 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:37:45 -0500 Subject: [PATCH 064/406] Update changelog --- changelog.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/changelog.txt b/changelog.txt index 380dca71..b197e7fa 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,13 @@ Change_LOG / History +V12.0.7 + +Refactor utils +Fix git subprocess +Fix cleanup script output +Add extra logging for fork detection +Additional code clean up + V12.0.6 Hotfix for Manual Torrent run results. From 649febdedd994de7faa8fd97871dbc5dba17e16b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:19:09 -0500 Subject: [PATCH 065/406] Refactor NZBGET_POSTPROCESS_PARCHECK -> NZBGET_POSTPROCESS_PAR_CHECK --- core/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e7280ded..0423ea62 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -105,7 +105,7 @@ FORKS = { ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} # NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_PAR_CHECK = 92 NZBGET_POSTPROCESS_SUCCESS = 93 NZBGET_POSTPROCESS_ERROR = 94 NZBGET_POSTPROCESS_NONE = 95 @@ -252,7 +252,7 @@ __INITIALIZED__ = False def initialize(section=None): - global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PARCHECK, NZBGET_POSTPROCESS_SUCCESS, \ + global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ From 20bd765a4bbd540859240c5de2440fe03648b441 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:20:04 -0500 Subject: [PATCH 066/406] Refactor UTORRENTWEBUI -> UTORRENT_WEB_UI --- core/__init__.py | 6 +++--- core/utils/torrents.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 0423ea62..435da837 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -150,7 +150,7 @@ TORRENT_RESUME_ON_FAILURE = None REMOTEPATHS = [] -UTORRENTWEBUI = None +UTORRENT_WEB_UI = None UTORRENTUSR = None UTORRENTPWD = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENTPWD, UTORRENTUSR, UTORRENTWEBUI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENTPWD, UTORRENTUSR, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ TRANSMISSIONHOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -406,7 +406,7 @@ def initialize(section=None): TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) TORRENT_RESUME = int(CFG['Torrent']['resume']) - UTORRENTWEBUI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ + UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ UTORRENTUSR = CFG['Torrent']['uTorrentUSR'] # mysecretusr UTORRENTPWD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr diff --git a/core/utils/torrents.py b/core/utils/torrents.py index e9786da9..1ecbca95 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -15,8 +15,8 @@ def create_torrent_class(client_agent): if client_agent == 'utorrent': try: - logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) - tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) + logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENT_WEB_UI)) + tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENTUSR, core.UTORRENTPWD) except Exception: logger.error('Failed to connect to uTorrent') From 22d2c1b108002a56c3eb7c0eef8b6adc2b47e6d7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:20:30 -0500 Subject: [PATCH 067/406] Refactor UTORRENTUSR -> UTORRENT_USER --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 435da837..321c744b 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -151,7 +151,7 @@ TORRENT_RESUME_ON_FAILURE = None REMOTEPATHS = [] UTORRENT_WEB_UI = None -UTORRENTUSR = None +UTORRENT_USER = None UTORRENTPWD = None TRANSMISSIONHOST = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENTPWD, UTORRENTUSR, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENTPWD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ TRANSMISSIONHOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -407,7 +407,7 @@ def initialize(section=None): TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) TORRENT_RESUME = int(CFG['Torrent']['resume']) UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ - UTORRENTUSR = CFG['Torrent']['uTorrentUSR'] # mysecretusr + UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr UTORRENTPWD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr TRANSMISSIONHOST = CFG['Torrent']['TransmissionHost'] # localhost diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 1ecbca95..37943258 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -16,7 +16,7 @@ def create_torrent_class(client_agent): if client_agent == 'utorrent': try: logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENT_WEB_UI)) - tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENTUSR, core.UTORRENTPWD) + tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENT_USER, core.UTORRENTPWD) except Exception: logger.error('Failed to connect to uTorrent') From 39974e62cc97c1de6fe909e4637bcacf084603eb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:20:59 -0500 Subject: [PATCH 068/406] Refactor UTORRENTPWD -> UTORRENT_PASSWORD --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 321c744b..6cb95416 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -152,7 +152,7 @@ REMOTEPATHS = [] UTORRENT_WEB_UI = None UTORRENT_USER = None -UTORRENTPWD = None +UTORRENT_PASSWORD = None TRANSMISSIONHOST = None TRANSMISSIONPORT = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENTPWD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ TRANSMISSIONHOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -408,7 +408,7 @@ def initialize(section=None): TORRENT_RESUME = int(CFG['Torrent']['resume']) UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr - UTORRENTPWD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr + UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr TRANSMISSIONHOST = CFG['Torrent']['TransmissionHost'] # localhost TRANSMISSIONPORT = int(CFG['Torrent']['TransmissionPort']) diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 37943258..b471a297 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -16,7 +16,7 @@ def create_torrent_class(client_agent): if client_agent == 'utorrent': try: logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENT_WEB_UI)) - tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENT_USER, core.UTORRENTPWD) + tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENT_USER, core.UTORRENT_PASSWORD) except Exception: logger.error('Failed to connect to uTorrent') From 5d5eb798c97660daad901e4dc1b00934af79d42d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:21:34 -0500 Subject: [PATCH 069/406] Refactor TRANSMISSIONHOST -> TRANSMISSION_HOST --- core/__init__.py | 6 +++--- core/utils/torrents.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 6cb95416..7794fa14 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -154,7 +154,7 @@ UTORRENT_WEB_UI = None UTORRENT_USER = None UTORRENT_PASSWORD = None -TRANSMISSIONHOST = None +TRANSMISSION_HOST = None TRANSMISSIONPORT = None TRANSMISSIONUSR = None TRANSMISSIONPWD = None @@ -257,7 +257,7 @@ def initialize(section=None): NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ - TRANSMISSIONHOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ + TRANSMISSION_HOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ @@ -410,7 +410,7 @@ def initialize(section=None): UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr - TRANSMISSIONHOST = CFG['Torrent']['TransmissionHost'] # localhost + TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost TRANSMISSIONPORT = int(CFG['Torrent']['TransmissionPort']) TRANSMISSIONUSR = CFG['Torrent']['TransmissionUSR'] # mysecretusr TRANSMISSIONPWD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr diff --git a/core/utils/torrents.py b/core/utils/torrents.py index b471a297..21e954b5 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -23,8 +23,8 @@ def create_torrent_class(client_agent): if client_agent == 'transmission': try: logger.debug('Connecting to {0}: http://{1}:{2}'.format( - client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) - tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, + client_agent, core.TRANSMISSION_HOST, core.TRANSMISSIONPORT)) + tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSIONPORT, core.TRANSMISSIONUSR, core.TRANSMISSIONPWD) except Exception: From e66ad2b66d623bc4540d7cfce8930b375a31759f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:21:58 -0500 Subject: [PATCH 070/406] Refactor TRANSMISSIONPORT -> TRANSMISSION_PORT --- core/__init__.py | 6 +++--- core/utils/torrents.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 7794fa14..21daaffc 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -155,7 +155,7 @@ UTORRENT_USER = None UTORRENT_PASSWORD = None TRANSMISSION_HOST = None -TRANSMISSIONPORT = None +TRANSMISSION_PORT = None TRANSMISSIONUSR = None TRANSMISSIONPWD = None @@ -257,7 +257,7 @@ def initialize(section=None): NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSIONPORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ + TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ @@ -411,7 +411,7 @@ def initialize(section=None): UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost - TRANSMISSIONPORT = int(CFG['Torrent']['TransmissionPort']) + TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) TRANSMISSIONUSR = CFG['Torrent']['TransmissionUSR'] # mysecretusr TRANSMISSIONPWD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 21e954b5..5d885495 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -23,8 +23,8 @@ def create_torrent_class(client_agent): if client_agent == 'transmission': try: logger.debug('Connecting to {0}: http://{1}:{2}'.format( - client_agent, core.TRANSMISSION_HOST, core.TRANSMISSIONPORT)) - tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSIONPORT, + client_agent, core.TRANSMISSION_HOST, core.TRANSMISSION_PORT)) + tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSION_PORT, core.TRANSMISSIONUSR, core.TRANSMISSIONPWD) except Exception: From 42dfdf73abcac92349268c9f1d82d7e8a347c78a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:22:27 -0500 Subject: [PATCH 071/406] Refactor TRANSMISSIONUSR -> TRANSMISSION_USER --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 21daaffc..cd9d6260 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -156,7 +156,7 @@ UTORRENT_PASSWORD = None TRANSMISSION_HOST = None TRANSMISSION_PORT = None -TRANSMISSIONUSR = None +TRANSMISSION_USER = None TRANSMISSIONPWD = None DELUGEHOST = None @@ -257,7 +257,7 @@ def initialize(section=None): NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, \ + TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSIONPWD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ @@ -412,7 +412,7 @@ def initialize(section=None): TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) - TRANSMISSIONUSR = CFG['Torrent']['TransmissionUSR'] # mysecretusr + TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr TRANSMISSIONPWD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr DELUGEHOST = CFG['Torrent']['DelugeHost'] # localhost diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 5d885495..0179d93b 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -25,7 +25,7 @@ def create_torrent_class(client_agent): logger.debug('Connecting to {0}: http://{1}:{2}'.format( client_agent, core.TRANSMISSION_HOST, core.TRANSMISSION_PORT)) tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSION_PORT, - core.TRANSMISSIONUSR, + core.TRANSMISSION_USER, core.TRANSMISSIONPWD) except Exception: logger.error('Failed to connect to Transmission') From a62415d71120d65b9a3f62ec84de7b1dcc0f6625 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:22:49 -0500 Subject: [PATCH 072/406] Refactor TRANSMISSIONPWD -> TRANSMISSION_PASSWORD --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index cd9d6260..2d0f44db 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -157,7 +157,7 @@ UTORRENT_PASSWORD = None TRANSMISSION_HOST = None TRANSMISSION_PORT = None TRANSMISSION_USER = None -TRANSMISSIONPWD = None +TRANSMISSION_PASSWORD = None DELUGEHOST = None DELUGEPORT = None @@ -257,7 +257,7 @@ def initialize(section=None): NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSIONPWD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ + TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ @@ -413,7 +413,7 @@ def initialize(section=None): TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr - TRANSMISSIONPWD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr + TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr DELUGEHOST = CFG['Torrent']['DelugeHost'] # localhost DELUGEPORT = int(CFG['Torrent']['DelugePort']) # 8084 diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 0179d93b..60a2ec5d 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -26,7 +26,7 @@ def create_torrent_class(client_agent): client_agent, core.TRANSMISSION_HOST, core.TRANSMISSION_PORT)) tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSION_PORT, core.TRANSMISSION_USER, - core.TRANSMISSIONPWD) + core.TRANSMISSION_PASSWORD) except Exception: logger.error('Failed to connect to Transmission') From 9262ba9cd0fc76f44fcad2e76ab3f109759323bd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:23:14 -0500 Subject: [PATCH 073/406] Refactor DELUGEHOST -> DELUGE_HOST --- core/__init__.py | 6 +++--- core/utils/torrents.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 2d0f44db..b761fd9e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -159,7 +159,7 @@ TRANSMISSION_PORT = None TRANSMISSION_USER = None TRANSMISSION_PASSWORD = None -DELUGEHOST = None +DELUGE_HOST = None DELUGEPORT = None DELUGEUSR = None DELUGEPWD = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -415,7 +415,7 @@ def initialize(section=None): TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr - DELUGEHOST = CFG['Torrent']['DelugeHost'] # localhost + DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost DELUGEPORT = int(CFG['Torrent']['DelugePort']) # 8084 DELUGEUSR = CFG['Torrent']['DelugeUSR'] # mysecretusr DELUGEPWD = CFG['Torrent']['DelugePWD'] # mysecretpwr diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 60a2ec5d..b38da66b 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -32,9 +32,9 @@ def create_torrent_class(client_agent): if client_agent == 'deluge': try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGEPORT)) tc = DelugeClient() - tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, + tc.connect(host=core.DELUGE_HOST, port=core.DELUGEPORT, username=core.DELUGEUSR, password=core.DELUGEPWD) except Exception: logger.error('Failed to connect to Deluge') From df5291fd4fddeaf2e11a975f5809f9342ed60367 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:46:54 -0500 Subject: [PATCH 074/406] Refactor DELUGEPORT -> DELUGE_PORT --- core/__init__.py | 6 +++--- core/utils/torrents.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index b761fd9e..1024b16b 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -160,7 +160,7 @@ TRANSMISSION_USER = None TRANSMISSION_PASSWORD = None DELUGE_HOST = None -DELUGEPORT = None +DELUGE_PORT = None DELUGEUSR = None DELUGEPWD = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -416,7 +416,7 @@ def initialize(section=None): TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost - DELUGEPORT = int(CFG['Torrent']['DelugePort']) # 8084 + DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 DELUGEUSR = CFG['Torrent']['DelugeUSR'] # mysecretusr DELUGEPWD = CFG['Torrent']['DelugePWD'] # mysecretpwr diff --git a/core/utils/torrents.py b/core/utils/torrents.py index b38da66b..d72437a9 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -32,9 +32,9 @@ def create_torrent_class(client_agent): if client_agent == 'deluge': try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGEPORT)) + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGE_PORT)) tc = DelugeClient() - tc.connect(host=core.DELUGE_HOST, port=core.DELUGEPORT, username=core.DELUGEUSR, + tc.connect(host=core.DELUGE_HOST, port=core.DELUGE_PORT, username=core.DELUGEUSR, password=core.DELUGEPWD) except Exception: logger.error('Failed to connect to Deluge') From 74bc6fb5b4b23ffa6c976cafe345f7d69865cede Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:49:00 -0500 Subject: [PATCH 075/406] Refactor DELUGEUSR -> DELUGE_USER --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 1024b16b..203b507a 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -161,7 +161,7 @@ TRANSMISSION_PASSWORD = None DELUGE_HOST = None DELUGE_PORT = None -DELUGEUSR = None +DELUGE_USER = None DELUGEPWD = None QBITTORRENTHOST = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGEPWD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -417,7 +417,7 @@ def initialize(section=None): DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 - DELUGEUSR = CFG['Torrent']['DelugeUSR'] # mysecretusr + DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr DELUGEPWD = CFG['Torrent']['DelugePWD'] # mysecretpwr QBITTORRENTHOST = CFG['Torrent']['qBittorrenHost'] # localhost diff --git a/core/utils/torrents.py b/core/utils/torrents.py index d72437a9..1138b952 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -34,7 +34,7 @@ def create_torrent_class(client_agent): try: logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGE_PORT)) tc = DelugeClient() - tc.connect(host=core.DELUGE_HOST, port=core.DELUGE_PORT, username=core.DELUGEUSR, + tc.connect(host=core.DELUGE_HOST, port=core.DELUGE_PORT, username=core.DELUGE_USER, password=core.DELUGEPWD) except Exception: logger.error('Failed to connect to Deluge') From 1aa0ea6e75011455f41264a4424a8ea5d2900121 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 22:50:23 -0500 Subject: [PATCH 076/406] Refactor DELUGEPWD -> DELUGE_PASSWORD --- core/__init__.py | 6 +++--- core/utils/torrents.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 203b507a..874632b8 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -162,7 +162,7 @@ TRANSMISSION_PASSWORD = None DELUGE_HOST = None DELUGE_PORT = None DELUGE_USER = None -DELUGEPWD = None +DELUGE_PASSWORD = None QBITTORRENTHOST = None QBITTORRENTPORT = None @@ -256,7 +256,7 @@ def initialize(section=None): NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGEPWD, VLEVEL, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ @@ -418,7 +418,7 @@ def initialize(section=None): DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr - DELUGEPWD = CFG['Torrent']['DelugePWD'] # mysecretpwr + DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr QBITTORRENTHOST = CFG['Torrent']['qBittorrenHost'] # localhost QBITTORRENTPORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 1138b952..867df7ee 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -35,7 +35,7 @@ def create_torrent_class(client_agent): logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGE_PORT)) tc = DelugeClient() tc.connect(host=core.DELUGE_HOST, port=core.DELUGE_PORT, username=core.DELUGE_USER, - password=core.DELUGEPWD) + password=core.DELUGE_PASSWORD) except Exception: logger.error('Failed to connect to Deluge') From 182a542bdaea4048b98faddf29b59be1b9b273ff Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 18 Jan 2019 23:48:41 -0500 Subject: [PATCH 077/406] Refactor QBITTORENT* --- core/__init__.py | 18 +++++++++--------- core/utils/torrents.py | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 874632b8..4acda833 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -164,10 +164,10 @@ DELUGE_PORT = None DELUGE_USER = None DELUGE_PASSWORD = None -QBITTORRENTHOST = None -QBITTORRENTPORT = None -QBITTORRENTUSR = None -QBITTORRENTPWD = None +QBITTORRENT_HOST = None +QBITTORRENT_PORT = None +QBITTORRENT_USER = None +QBITTORRENT_PASSWORD = None PLEXSSL = None PLEXHOST = None @@ -269,7 +269,7 @@ def initialize(section=None): DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ TORRENT_DEFAULTDIR, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULTDIR, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PLEXSSL, PLEXHOST, PLEXPORT, PLEXTOKEN, PLEXSEC, TORRENT_RESUME, PAR2CMD, QBITTORRENTHOST, QBITTORRENTPORT, QBITTORRENTUSR, QBITTORRENTPWD + PLEXSSL, PLEXHOST, PLEXPORT, PLEXTOKEN, PLEXSEC, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: return False @@ -420,10 +420,10 @@ def initialize(section=None): DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr - QBITTORRENTHOST = CFG['Torrent']['qBittorrenHost'] # localhost - QBITTORRENTPORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 - QBITTORRENTUSR = CFG['Torrent']['qBittorrentUSR'] # mysecretusr - QBITTORRENTPWD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr + QBITTORRENT_HOST = CFG['Torrent']['qBittorrenHost'] # localhost + QBITTORRENT_PORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 + QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr + QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr REMOTEPATHS = CFG['Network']['mount_points'] or [] if REMOTEPATHS: diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 867df7ee..c20ba97f 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -41,9 +41,9 @@ def create_torrent_class(client_agent): if client_agent == 'qbittorrent': try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENT_HOST, core.QBITTORRENT_PORT)) + tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENT_HOST, core.QBITTORRENT_PORT)) + tc.login(core.QBITTORRENT_USER, core.QBITTORRENT_PASSWORD) except Exception: logger.error('Failed to connect to qBittorrent') From d2346b0ea6407a871742e700b9740a5baec9adb0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:13:13 -0500 Subject: [PATCH 078/406] Refactor PLEX* --- core/__init__.py | 30 +++++++++++++++--------------- core/utils/notifications.py | 12 ++++++------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 4acda833..c234fef3 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -169,11 +169,11 @@ QBITTORRENT_PORT = None QBITTORRENT_USER = None QBITTORRENT_PASSWORD = None -PLEXSSL = None -PLEXHOST = None -PLEXPORT = None -PLEXTOKEN = None -PLEXSEC = [] +PLEX_SSL = None +PLEX_HOST = None +PLEX_PORT = None +PLEX_TOKEN = None +PLEX_SECTION = [] EXTCONTAINER = [] COMPRESSEDCONTAINER = [] @@ -269,7 +269,7 @@ def initialize(section=None): DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ TORRENT_DEFAULTDIR, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULTDIR, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PLEXSSL, PLEXHOST, PLEXPORT, PLEXTOKEN, PLEXSEC, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD + PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: return False @@ -434,15 +434,15 @@ def initialize(section=None): REMOTEPATHS = [(local.strip(), remote.strip()) for local, remote in REMOTEPATHS] # strip trailing and leading whitespaces - PLEXSSL = int(CFG['Plex']['plex_ssl']) - PLEXHOST = CFG['Plex']['plex_host'] - PLEXPORT = CFG['Plex']['plex_port'] - PLEXTOKEN = CFG['Plex']['plex_token'] - PLEXSEC = CFG['Plex']['plex_sections'] or [] - if PLEXSEC: - if isinstance(PLEXSEC, list): - PLEXSEC = ','.join(PLEXSEC) # fix in case this imported as list. - PLEXSEC = [tuple(item.split(',')) for item in PLEXSEC.split('|')] + PLEX_SSL = int(CFG['Plex']['plex_ssl']) + PLEX_HOST = CFG['Plex']['plex_host'] + PLEX_PORT = CFG['Plex']['plex_port'] + PLEX_TOKEN = CFG['Plex']['plex_token'] + PLEX_SECTION = CFG['Plex']['plex_sections'] or [] + if PLEX_SECTION: + if isinstance(PLEX_SECTION, list): + PLEX_SECTION = ','.join(PLEX_SECTION) # fix in case this imported as list. + PLEX_SECTION = [tuple(item.split(',')) for item in PLEX_SECTION.split('|')] devnull = open(os.devnull, 'w') try: diff --git a/core/utils/notifications.py b/core/utils/notifications.py index ed89f65b..ddad0c1b 100644 --- a/core/utils/notifications.py +++ b/core/utils/notifications.py @@ -8,20 +8,20 @@ def plex_update(category): if core.FAILED: return url = '{scheme}://{host}:{port}/library/sections/'.format( - scheme='https' if core.PLEXSSL else 'http', - host=core.PLEXHOST, - port=core.PLEXPORT, + scheme='https' if core.PLEX_SSL else 'http', + host=core.PLEX_HOST, + port=core.PLEX_PORT, ) section = None - if not core.PLEXSEC: + if not core.PLEX_SECTION: return logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') - for item in core.PLEXSEC: + for item in core.PLEX_SECTION: if item[0] == category: section = item[1] if section: - url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) + url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEX_TOKEN) requests.get(url, timeout=(60, 120), verify=False) logger.debug('Plex Library has been refreshed.', 'PLEX') else: From 28f1bc35c592fe798ef14570825c0d864fa427db Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:13:51 -0500 Subject: [PATCH 079/406] Refactor USELINK -> USE_LINK --- TorrentToMedia.py | 4 ++-- core/__init__.py | 6 +++--- core/auto_process/movies.py | 2 +- core/auto_process/tv.py | 2 +- core/utils/common.py | 2 +- core/utils/torrents.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 4971c13d..9042fea1 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -196,7 +196,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if torrent_no_link == 0: try: - core.copy_link(inputFile, target_file, core.USELINK) + core.copy_link(inputFile, target_file, core.USE_LINK) core.remove_read_only(target_file) except Exception: logger.error('Failed to link: {0} to {1}'.format(inputFile, target_file)) @@ -270,7 +270,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.update_download_info_status(input_name, 1) # remove torrent - if core.USELINK == 'move-sym' and not core.DELETE_ORIGINAL == 1: + if core.USE_LINK == 'move-sym' and not core.DELETE_ORIGINAL == 1: logger.debug('Checking for sym-links to re-direct in: {0}'.format(input_directory)) for dirpath, dirs, files in os.walk(input_directory): for file in files: diff --git a/core/__init__.py b/core/__init__.py index c234fef3..99d3b974 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -139,7 +139,7 @@ NZB_DEFAULTDIR = None TORRENT_CLIENTAGENT = None TORRENT_CLASS = None -USELINK = None +USE_LINK = None OUTPUTDIRECTORY = None NOFLATTEN = [] DELETE_ORIGINAL = None @@ -255,7 +255,7 @@ def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USE_LINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ @@ -393,7 +393,7 @@ def initialize(section=None): GROUPS = None TORRENT_CLIENTAGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other - USELINK = CFG['Torrent']['useLink'] # no | hard | sym + USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUTDIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULTDIR = CFG['Torrent']['default_downloadDirectory'] CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 0c2f5880..ab84627e 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -152,7 +152,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', if not release and '.cp(tt' not in video and imdbid: video_name, video_ext = os.path.splitext(video) video2 = '{0}.cp({1}){2}'.format(video_name, imdbid, video_ext) - if not (client_agent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USELINK == 'move-sym'): + if not (client_agent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USE_LINK == 'move-sym'): logger.debug('Renaming: {0} to: {1}'.format(video, video2)) os.rename(video, video2) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 8abfd0df..56323894 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -47,7 +47,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu delete_failed = int(cfg.get('delete_failed', 0)) nzb_extraction_by = cfg.get('nzbExtractionBy', 'Downloader') process_method = cfg.get('process_method') - if client_agent == core.TORRENT_CLIENTAGENT and core.USELINK == 'move-sym': + if client_agent == core.TORRENT_CLIENTAGENT and core.USE_LINK == 'move-sym': process_method = 'symlink' remote_path = int(cfg.get('remote_path', 0)) wait_for = int(cfg.get('wait_for', 2)) diff --git a/core/utils/common.py b/core/utils/common.py index f352fb87..c2f9ef79 100644 --- a/core/utils/common.py +++ b/core/utils/common.py @@ -100,7 +100,7 @@ def get_dirs(section, subsection, link='hard'): except Exception as e: logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) - if core.USELINK == 'move': + if core.USE_LINK == 'move': try: output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) if os.path.exists(output_directory): diff --git a/core/utils/torrents.py b/core/utils/torrents.py index c20ba97f..810bfc42 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -85,7 +85,7 @@ def resume_torrent(client_agent, input_hash, input_id, input_name): def remove_torrent(client_agent, input_hash, input_id, input_name): - if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': + if core.DELETE_ORIGINAL == 1 or core.USE_LINK == 'move': logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) try: if client_agent == 'utorrent' and core.TORRENT_CLASS != '': From a24367113b95b2ddf9b1d5bef393462a20b29cac Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:14:48 -0500 Subject: [PATCH 080/406] Refactor OUTPUTDIRECTORY -> OUTPUT_DIRECTORY --- TorrentToMedia.py | 6 +++--- core/__init__.py | 6 +++--- core/utils/common.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 9042fea1..2e316d12 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -114,13 +114,13 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp basename = os.path.basename(input_directory) basename = core.sanitize_name(input_name) \ if input_name == basename else os.path.splitext(core.sanitize_name(input_name))[0] - output_destination = os.path.join(core.OUTPUTDIRECTORY, input_category, basename) + output_destination = os.path.join(core.OUTPUT_DIRECTORY, input_category, basename) elif unique_path: output_destination = os.path.normpath( - core.os.path.join(core.OUTPUTDIRECTORY, input_category, core.sanitize_name(input_name).replace(' ', '.'))) + core.os.path.join(core.OUTPUT_DIRECTORY, input_category, core.sanitize_name(input_name).replace(' ', '.'))) else: output_destination = os.path.normpath( - core.os.path.join(core.OUTPUTDIRECTORY, input_category)) + core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) try: output_destination = output_destination.encode(core.SYS_ENCODING) except UnicodeError: diff --git a/core/__init__.py b/core/__init__.py index 99d3b974..5c04fc4e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -140,7 +140,7 @@ NZB_DEFAULTDIR = None TORRENT_CLIENTAGENT = None TORRENT_CLASS = None USE_LINK = None -OUTPUTDIRECTORY = None +OUTPUT_DIRECTORY = None NOFLATTEN = [] DELETE_ORIGINAL = None TORRENT_CHMOD_DIRECTORY = None @@ -255,7 +255,7 @@ def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USE_LINK, OUTPUTDIRECTORY, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ @@ -394,7 +394,7 @@ def initialize(section=None): TORRENT_CLIENTAGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym - OUTPUTDIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ + OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULTDIR = CFG['Torrent']['default_downloadDirectory'] CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software NOFLATTEN = (CFG['Torrent']['noFlatten']) diff --git a/core/utils/common.py b/core/utils/common.py index c2f9ef79..5ecdf830 100644 --- a/core/utils/common.py +++ b/core/utils/common.py @@ -102,11 +102,11 @@ def get_dirs(section, subsection, link='hard'): if core.USE_LINK == 'move': try: - output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) + output_directory = os.path.join(core.OUTPUT_DIRECTORY, subsection) if os.path.exists(output_directory): to_return.extend(process_dir(output_directory, link)) except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUT_DIRECTORY, e)) if not to_return: logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) From 4bf842b4f4a3aedae16919f15289a5b66c199c0b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:24:40 -0500 Subject: [PATCH 081/406] Refactor TORRENT_DEFAULTDIR -> TORRENT_DEFAULT_DIRECTORY --- TorrentToMedia.py | 2 +- core/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 2e316d12..7de26883 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -131,7 +131,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp logger.info('Output directory set to: {0}'.format(output_destination)) - if core.SAFE_MODE and output_destination == core.TORRENT_DEFAULTDIR: + if core.SAFE_MODE and output_destination == core.TORRENT_DEFAULT_DIRECTORY: logger.error('The output directory:[{0}] is the Download Directory. ' 'Edit outputDirectory in autoProcessMedia.cfg. Exiting'.format (input_directory)) diff --git a/core/__init__.py b/core/__init__.py index 5c04fc4e..51d7fdb3 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -144,7 +144,7 @@ OUTPUT_DIRECTORY = None NOFLATTEN = [] DELETE_ORIGINAL = None TORRENT_CHMOD_DIRECTORY = None -TORRENT_DEFAULTDIR = None +TORRENT_DEFAULT_DIRECTORY = None TORRENT_RESUME = None TORRENT_RESUME_ON_FAILURE = None @@ -268,7 +268,7 @@ def initialize(section=None): NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIOCONTAINER, EXTCONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ - TORRENT_DEFAULTDIR, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULTDIR, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULTDIR, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: @@ -395,7 +395,7 @@ def initialize(section=None): TORRENT_CLIENTAGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ - TORRENT_DEFAULTDIR = CFG['Torrent']['default_downloadDirectory'] + TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software NOFLATTEN = (CFG['Torrent']['noFlatten']) if isinstance(NOFLATTEN, str): From 5bea8f121ecb50d9d3e0d676b671f7502a74541e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:26:08 -0500 Subject: [PATCH 082/406] Refactor SABNZBD* --- core/__init__.py | 14 +++++++------- core/utils/network.py | 8 ++++---- core/utils/nzbs.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 51d7fdb3..2e735c10 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -132,9 +132,9 @@ SAFE_MODE = None NOEXTRACTFAILED = None NZB_CLIENTAGENT = None -SABNZBDHOST = None -SABNZBDPORT = None -SABNZBDAPIKEY = None +SABNZBD_HOST = None +SABNZBD_PORT = None +SABNZBD_APIKEY = None NZB_DEFAULTDIR = None TORRENT_CLIENTAGENT = None @@ -260,7 +260,7 @@ def initialize(section=None): TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ - GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ + GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ @@ -382,9 +382,9 @@ def initialize(section=None): wake_up() NZB_CLIENTAGENT = CFG['Nzb']['clientAgent'] # sabnzbd - SABNZBDHOST = CFG['Nzb']['sabnzbd_host'] - SABNZBDPORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet - SABNZBDAPIKEY = CFG['Nzb']['sabnzbd_apikey'] + SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] + SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet + SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] NZB_DEFAULTDIR = CFG['Nzb']['default_downloadDirectory'] GROUPS = CFG['Custom']['remove_group'] if isinstance(GROUPS, str): diff --git a/core/utils/network.py b/core/utils/network.py index 99a89e1e..5a7a5758 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -101,13 +101,13 @@ def find_download(client_agent, download_id): if torrent['hash'] == download_id: return True if client_agent == 'sabnzbd': - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + if 'http' in core.SABNZBD_HOST: + base_url = '{0}:{1}/api'.format(core.SABNZBD_HOST, core.SABNZBD_PORT) else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + base_url = 'http://{0}:{1}/api'.format(core.SABNZBD_HOST, core.SABNZBD_PORT) url = base_url params = { - 'apikey': core.SABNZBDAPIKEY, + 'apikey': core.SABNZBD_APIKEY, 'mode': 'get_files', 'output': 'json', 'value': download_id, diff --git a/core/utils/nzbs.py b/core/utils/nzbs.py index ad3b4e04..23061c96 100644 --- a/core/utils/nzbs.py +++ b/core/utils/nzbs.py @@ -10,13 +10,13 @@ def get_nzoid(input_name): nzoid = None slots = [] logger.debug('Searching for nzoid from SAbnzbd ...') - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + if 'http' in core.SABNZBD_HOST: + base_url = '{0}:{1}/api'.format(core.SABNZBD_HOST, core.SABNZBD_PORT) else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + base_url = 'http://{0}:{1}/api'.format(core.SABNZBD_HOST, core.SABNZBD_PORT) url = base_url params = { - 'apikey': core.SABNZBDAPIKEY, + 'apikey': core.SABNZBD_APIKEY, 'mode': 'queue', 'output': 'json', } From a0d8940f70d83083933e152fd017567e233142d0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:26:34 -0500 Subject: [PATCH 083/406] Refactor NZB_CLIENTAGENT -> NZB_CLIENT_AGENT --- core/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 2e735c10..7adfc38e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -131,7 +131,7 @@ FORCE_CLEAN = None SAFE_MODE = None NOEXTRACTFAILED = None -NZB_CLIENTAGENT = None +NZB_CLIENT_AGENT = None SABNZBD_HOST = None SABNZBD_PORT = None SABNZBD_APIKEY = None @@ -260,7 +260,7 @@ def initialize(section=None): TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ - GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ + GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ @@ -381,7 +381,7 @@ def initialize(section=None): if int(CFG['WakeOnLan']['wake']) == 1: wake_up() - NZB_CLIENTAGENT = CFG['Nzb']['clientAgent'] # sabnzbd + NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] From fafcdb4ed57e4305a5e8c676c7725deebc98030d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:35:35 -0500 Subject: [PATCH 084/406] Refactor NZB_DEFAULTDIR -> NZB_DEFAULT_DIRECTORY --- core/__init__.py | 6 +++--- nzbToMedia.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 7adfc38e..cdeedda9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -135,7 +135,7 @@ NZB_CLIENT_AGENT = None SABNZBD_HOST = None SABNZBD_PORT = None SABNZBD_APIKEY = None -NZB_DEFAULTDIR = None +NZB_DEFAULT_DIRECTORY = None TORRENT_CLIENTAGENT = None TORRENT_CLASS = None @@ -268,7 +268,7 @@ def initialize(section=None): NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIOCONTAINER, EXTCONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULTDIR, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: @@ -385,7 +385,7 @@ def initialize(section=None): SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] - NZB_DEFAULTDIR = CFG['Nzb']['default_downloadDirectory'] + NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] GROUPS = CFG['Custom']['remove_group'] if isinstance(GROUPS, str): GROUPS = GROUPS.split(',') diff --git a/nzbToMedia.py b/nzbToMedia.py index fceed9cd..526540d1 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -645,7 +645,7 @@ except NameError: # post-processing def process(input_directory, input_name=None, status=0, client_agent='manual', download_id=None, input_category=None, failure_link=None): - if core.SAFE_MODE and input_directory == core.NZB_DEFAULTDIR: + if core.SAFE_MODE and input_directory == core.NZB_DEFAULT_DIRECTORY: logger.error( 'The input directory:[{0}] is the Default Download Directory. Please configure category directories to prevent processing of other media.'.format( input_directory)) From d973f4955fa242ee9c32b46fef7585a57316fe10 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:36:12 -0500 Subject: [PATCH 085/406] Refactor TORRENT_CLIENTAGENT -> TORRENT_CLIENT_AGENT --- TorrentToMedia.py | 2 +- core/__init__.py | 8 ++++---- core/auto_process/movies.py | 2 +- core/auto_process/tv.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 7de26883..fe84e5dc 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -291,7 +291,7 @@ def main(args): core.initialize() # clientAgent for Torrents - client_agent = core.TORRENT_CLIENTAGENT + client_agent = core.TORRENT_CLIENT_AGENT logger.info('#########################################################') logger.info('## ..::[{0}]::.. ##'.format(os.path.basename(__file__))) diff --git a/core/__init__.py b/core/__init__.py index cdeedda9..9ea59001 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -137,7 +137,7 @@ SABNZBD_PORT = None SABNZBD_APIKEY = None NZB_DEFAULT_DIRECTORY = None -TORRENT_CLIENTAGENT = None +TORRENT_CLIENT_AGENT = None TORRENT_CLASS = None USE_LINK = None OUTPUT_DIRECTORY = None @@ -255,7 +255,7 @@ def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USE_LINK, OUTPUT_DIRECTORY, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ @@ -392,7 +392,7 @@ def initialize(section=None): if GROUPS == ['']: GROUPS = None - TORRENT_CLIENTAGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other + TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] @@ -860,7 +860,7 @@ def initialize(section=None): CATEGORIES = list(set(CATEGORIES)) # create torrent class - TORRENT_CLASS = create_torrent_class(TORRENT_CLIENTAGENT) + TORRENT_CLASS = create_torrent_class(TORRENT_CLIENT_AGENT) # finished initalizing return True diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index ab84627e..7b1fb8e5 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -152,7 +152,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', if not release and '.cp(tt' not in video and imdbid: video_name, video_ext = os.path.splitext(video) video2 = '{0}.cp({1}){2}'.format(video_name, imdbid, video_ext) - if not (client_agent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USE_LINK == 'move-sym'): + if not (client_agent in [core.TORRENT_CLIENT_AGENT, 'manual'] and core.USE_LINK == 'move-sym'): logger.debug('Renaming: {0} to: {1}'.format(video, video2)) os.rename(video, video2) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 56323894..cc31f94e 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -47,7 +47,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu delete_failed = int(cfg.get('delete_failed', 0)) nzb_extraction_by = cfg.get('nzbExtractionBy', 'Downloader') process_method = cfg.get('process_method') - if client_agent == core.TORRENT_CLIENTAGENT and core.USE_LINK == 'move-sym': + if client_agent == core.TORRENT_CLIENT_AGENT and core.USE_LINK == 'move-sym': process_method = 'symlink' remote_path = int(cfg.get('remote_path', 0)) wait_for = int(cfg.get('wait_for', 2)) From 2ebe96e04930088b5b99f822cf95b25f4a46d134 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 00:50:00 -0500 Subject: [PATCH 086/406] Refactor REMOTEPATHS -> REMOTE_PATHS --- core/__init__.py | 20 ++++++++++---------- core/utils/paths.py | 4 ++-- nzbToMedia.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 9ea59001..9c040377 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -148,7 +148,7 @@ TORRENT_DEFAULT_DIRECTORY = None TORRENT_RESUME = None TORRENT_RESUME_ON_FAILURE = None -REMOTEPATHS = [] +REMOTE_PATHS = [] UTORRENT_WEB_UI = None UTORRENT_USER = None @@ -268,7 +268,7 @@ def initialize(section=None): NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIOCONTAINER, EXTCONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTEPATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: @@ -425,14 +425,14 @@ def initialize(section=None): QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr - REMOTEPATHS = CFG['Network']['mount_points'] or [] - if REMOTEPATHS: - if isinstance(REMOTEPATHS, list): - REMOTEPATHS = ','.join(REMOTEPATHS) # fix in case this imported as list. - REMOTEPATHS = [tuple(item.split(',')) for item in - REMOTEPATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\ - REMOTEPATHS = [(local.strip(), remote.strip()) for local, remote in - REMOTEPATHS] # strip trailing and leading whitespaces + REMOTE_PATHS = CFG['Network']['mount_points'] or [] + if REMOTE_PATHS: + if isinstance(REMOTE_PATHS, list): + REMOTE_PATHS = ','.join(REMOTE_PATHS) # fix in case this imported as list. + REMOTE_PATHS = [tuple(item.split(',')) for item in + REMOTE_PATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\ + REMOTE_PATHS = [(local.strip(), remote.strip()) for local, remote in + REMOTE_PATHS] # strip trailing and leading whitespaces PLEX_SSL = int(CFG['Plex']['plex_ssl']) PLEX_HOST = CFG['Plex']['plex_host'] diff --git a/core/utils/paths.py b/core/utils/paths.py index ec99a246..dba119f5 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -48,9 +48,9 @@ def make_dir(path): def remote_dir(path): - if not core.REMOTEPATHS: + if not core.REMOTE_PATHS: return path - for local, remote in core.REMOTEPATHS: + for local, remote in core.REMOTE_PATHS: if local in path: base_dirs = path.replace(local, '').split(os.sep) if '/' in remote: diff --git a/nzbToMedia.py b/nzbToMedia.py index 526540d1..e8daf4da 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -725,7 +725,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d extract = int(cfg.get('extract', 0)) try: - if int(cfg.get('remote_path')) and not core.REMOTEPATHS: + if int(cfg.get('remote_path')) and not core.REMOTE_PATHS: logger.error('Remote Path is enabled for {0}:{1} but no Network mount points are defined. Please check your autoProcessMedia.cfg, exiting!'.format( section_name, input_category)) return ProcessResult( From 7e52aec4afdd7766193d0ab88a5c870de1134f0d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 01:17:08 -0500 Subject: [PATCH 087/406] Refactor *CONTAINER --- core/__init__.py | 46 ++++++++++++++++++++++----------------------- core/transcoder.py | 2 +- core/utils/files.py | 14 +++++++------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 9c040377..4cce519d 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -175,11 +175,11 @@ PLEX_PORT = None PLEX_TOKEN = None PLEX_SECTION = [] -EXTCONTAINER = [] -COMPRESSEDCONTAINER = [] -MEDIACONTAINER = [] -AUDIOCONTAINER = [] -METACONTAINER = [] +EXT_CONTAINER = [] +COMPRESSED_CONTAINER = [] +MEDIA_CONTAINER = [] +AUDIO_CONTAINER = [] +META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] @@ -257,15 +257,15 @@ def initialize(section=None): NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSEDCONTAINER, MEDIACONTAINER, \ - METACONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ + TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ + META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, LOG_DIR, LOG_FILE, \ - NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIOCONTAINER, EXTCONTAINER, TORRENT_CLASS, \ + NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ @@ -467,21 +467,21 @@ def initialize(section=None): pass devnull.close() - COMPRESSEDCONTAINER = [re.compile(r'.r\d{2}$', re.I), - re.compile(r'.part\d+.rar$', re.I), - re.compile('.rar$', re.I)] - COMPRESSEDCONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG['Extensions']['compressedExtensions']] - MEDIACONTAINER = CFG['Extensions']['mediaExtensions'] - AUDIOCONTAINER = CFG['Extensions']['audioExtensions'] - METACONTAINER = CFG['Extensions']['metaExtensions'] # .nfo,.sub,.srt - if isinstance(COMPRESSEDCONTAINER, str): - COMPRESSEDCONTAINER = COMPRESSEDCONTAINER.split(',') - if isinstance(MEDIACONTAINER, str): - MEDIACONTAINER = MEDIACONTAINER.split(',') - if isinstance(AUDIOCONTAINER, str): - AUDIOCONTAINER = AUDIOCONTAINER.split(',') - if isinstance(METACONTAINER, str): - METACONTAINER = METACONTAINER.split(',') + COMPRESSED_CONTAINER = [re.compile(r'.r\d{2}$', re.I), + re.compile(r'.part\d+.rar$', re.I), + re.compile('.rar$', re.I)] + COMPRESSED_CONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG['Extensions']['compressedExtensions']] + MEDIA_CONTAINER = CFG['Extensions']['mediaExtensions'] + AUDIO_CONTAINER = CFG['Extensions']['audioExtensions'] + META_CONTAINER = CFG['Extensions']['metaExtensions'] # .nfo,.sub,.srt + if isinstance(COMPRESSED_CONTAINER, str): + COMPRESSED_CONTAINER = COMPRESSED_CONTAINER.split(',') + if isinstance(MEDIA_CONTAINER, str): + MEDIA_CONTAINER = MEDIA_CONTAINER.split(',') + if isinstance(AUDIO_CONTAINER, str): + AUDIO_CONTAINER = AUDIO_CONTAINER.split(',') + if isinstance(META_CONTAINER, str): + META_CONTAINER = META_CONTAINER.split(',') GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) diff --git a/core/transcoder.py b/core/transcoder.py index e2bfb33d..52a35292 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -22,7 +22,7 @@ def is_video_good(videofile, status): file_name_ext = os.path.basename(videofile) file_name, file_ext = os.path.splitext(file_name_ext) disable = False - if file_ext not in core.MEDIACONTAINER or not core.FFPROBE or not core.CHECK_MEDIA or file_ext in ['.iso'] or (status > 0 and core.NOEXTRACTFAILED): + if file_ext not in core.MEDIA_CONTAINER or not core.FFPROBE or not core.CHECK_MEDIA or file_ext in ['.iso'] or (status > 0 and core.NOEXTRACTFAILED): disable = True else: test_details, res = get_video_details(core.TEST_FILE) diff --git a/core/utils/files.py b/core/utils/files.py index b753af0c..895125e1 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -20,7 +20,7 @@ def move_file(mediafile, path, link): new_path = None file_ext = os.path.splitext(mediafile)[1] try: - if file_ext in core.AUDIOCONTAINER: + if file_ext in core.AUDIO_CONTAINER: f = beets.mediafile.MediaFile(mediafile) # get artist and album info @@ -29,7 +29,7 @@ def move_file(mediafile, path, link): # create new path new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: + elif file_ext in core.MEDIA_CONTAINER: f = guessit.guessit(mediafile) # get title @@ -75,7 +75,7 @@ def is_min_size(input_name, min_size): # audio files we need to check directory size not file size input_size = os.path.getsize(input_name) - if file_ext in core.AUDIOCONTAINER: + if file_ext in core.AUDIO_CONTAINER: try: input_size = get_dir_size(os.path.dirname(input_name)) except Exception: @@ -89,7 +89,7 @@ def is_min_size(input_name, min_size): def is_archive_file(filename): """Check if the filename is allowed for the Archive""" - for regext in core.COMPRESSEDCONTAINER: + for regext in core.COMPRESSED_CONTAINER: if regext.search(filename): return regext.split(filename)[0] return False @@ -109,9 +109,9 @@ def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, o pass return any([ - (media and file_ext.lower() in core.MEDIACONTAINER), - (audio and file_ext.lower() in core.AUDIOCONTAINER), - (meta and file_ext.lower() in core.METACONTAINER), + (media and file_ext.lower() in core.MEDIA_CONTAINER), + (audio and file_ext.lower() in core.AUDIO_CONTAINER), + (meta and file_ext.lower() in core.META_CONTAINER), (archives and is_archive_file(mediafile)), (other and (file_ext.lower() in otherext or 'all' in otherext)), ]) From 0c98912b7607f2f1f491a317086ec0ecb533a86b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 01:20:41 -0500 Subject: [PATCH 088/406] Refactor PASSWORDSFILE -> PASSWORDS_FILE Refactor DOWNLOADINFO -> DOWNLOAD_INFO --- TorrentToMedia.py | 12 ++++++------ core/__init__.py | 10 +++++----- core/extractor/__init__.py | 4 ++-- nzbToMedia.py | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index fe84e5dc..31fa152c 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -22,7 +22,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp root = 0 found_file = 0 - if client_agent != 'manual' and not core.DOWNLOADINFO: + if client_agent != 'manual' and not core.DOWNLOAD_INFO: logger.debug('Adding TORRENT download info for directory {0} to database'.format(input_directory)) my_db = main_db.DBConnection() @@ -328,11 +328,11 @@ def main(args): logger.info('Checking database for download info for {0} ...'.format (os.path.basename(dir_name))) - core.DOWNLOADINFO = core.get_download_info(os.path.basename(dir_name), 0) - if core.DOWNLOADINFO: - client_agent = text_type(core.DOWNLOADINFO[0].get('client_agent', 'manual')) - input_hash = text_type(core.DOWNLOADINFO[0].get('input_hash', '')) - input_id = text_type(core.DOWNLOADINFO[0].get('input_id', '')) + core.DOWNLOAD_INFO = core.get_download_info(os.path.basename(dir_name), 0) + if core.DOWNLOAD_INFO: + client_agent = text_type(core.DOWNLOAD_INFO[0].get('client_agent', 'manual')) + input_hash = text_type(core.DOWNLOAD_INFO[0].get('input_hash', '')) + input_id = text_type(core.DOWNLOAD_INFO[0].get('input_id', '')) logger.info('Found download info for {0}, ' 'setting variables now ...'.format(os.path.basename(dir_name))) else: diff --git a/core/__init__.py b/core/__init__.py index 4cce519d..8929a435 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -236,8 +236,8 @@ CHECK_MEDIA = None NICENESS = [] HWACCEL = False -PASSWORDSFILE = None -DOWNLOADINFO = None +PASSWORDS_FILE = None +DOWNLOAD_INFO = None GROUPS = None USER_SCRIPT_MEDIAEXTENSIONS = None @@ -266,8 +266,8 @@ def initialize(section=None): ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, LOG_DIR, LOG_FILE, \ NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ - DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDSFILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOADINFO, CHECK_MEDIA, SAFE_MODE, \ + DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ + USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD @@ -763,7 +763,7 @@ def initialize(section=None): extra = [item for item in codec_alias[codec] if item not in ACODEC3_ALLOW] ACODEC3_ALLOW.extend(extra) - PASSWORDSFILE = CFG['passwords']['PassWordFile'] + PASSWORDS_FILE = CFG['passwords']['PassWordFile'] # Setup FFMPEG, FFPROBE and SEVENZIP locations if platform.system() == 'Windows': diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index b1090ea6..5d1c51a0 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -90,8 +90,8 @@ def extract(file_path, output_destination): # Create outputDestination folder core.make_dir(output_destination) - if core.PASSWORDSFILE and os.path.isfile(os.path.normpath(core.PASSWORDSFILE)): - passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDSFILE))] + if core.PASSWORDS_FILE and os.path.isfile(os.path.normpath(core.PASSWORDS_FILE)): + passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDS_FILE))] else: passwords = [] diff --git a/nzbToMedia.py b/nzbToMedia.py index e8daf4da..f6c7e5db 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -657,7 +657,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d if not download_id and client_agent == 'sabnzbd': download_id = get_nzoid(input_name) - if client_agent != 'manual' and not core.DOWNLOADINFO: + if client_agent != 'manual' and not core.DOWNLOAD_INFO: logger.debug('Adding NZB download info for directory {0} to database'.format(input_directory)) my_db = main_db.DBConnection() @@ -899,13 +899,13 @@ def main(args, section=None): logger.info('Starting manual run for {0}:{1} - Folder: {2}'.format(section, subsection, dir_name)) logger.info('Checking database for download info for {0} ...'.format(os.path.basename(dir_name))) - core.DOWNLOADINFO = get_download_info(os.path.basename(dir_name), 0) - if core.DOWNLOADINFO: + core.DOWNLOAD_INFO = get_download_info(os.path.basename(dir_name), 0) + if core.DOWNLOAD_INFO: logger.info('Found download info for {0}, ' 'setting variables now ...'.format (os.path.basename(dir_name))) - client_agent = text_type(core.DOWNLOADINFO[0].get('client_agent', 'manual')) - download_id = text_type(core.DOWNLOADINFO[0].get('input_id', '')) + client_agent = text_type(core.DOWNLOAD_INFO[0].get('client_agent', 'manual')) + download_id = text_type(core.DOWNLOAD_INFO[0].get('input_id', '')) else: logger.info('Unable to locate download info for {0}, ' 'continuing to try and process this release ...'.format From 1404464ef9752b2cb7c6a5d289b549477be62b87 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 02:25:04 -0500 Subject: [PATCH 089/406] Refactor locale configuration --- core/__init__.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 8929a435..e5635589 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -251,6 +251,18 @@ USER_SCRIPT_RUNONCE = None __INITIALIZED__ = False +def configure_logging(): + global LOG_FILE + global LOG_DIR + + if 'NTM_LOGFILE' in os.environ: + LOG_FILE = os.environ['NTM_LOGFILE'] + LOG_DIR = os.path.split(LOG_FILE)[0] + + if not make_dir(LOG_DIR): + print('No log folder, logging to screen only') + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ @@ -264,7 +276,7 @@ def initialize(section=None): DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ - SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, LOG_DIR, LOG_FILE, \ + SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ @@ -274,12 +286,7 @@ def initialize(section=None): if __INITIALIZED__: return False - if 'NTM_LOGFILE' in os.environ: - LOG_FILE = os.environ['NTM_LOGFILE'] - LOG_DIR = os.path.split(LOG_FILE)[0] - - if not make_dir(LOG_DIR): - print('No log folder, logging to screen only') + configure_logging() MYAPP = RunningProcess() while MYAPP.alreadyrunning(): From e0de964fdafa87acf1e26f10794464cecfda1842 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 02:27:22 -0500 Subject: [PATCH 090/406] Refactor process configuration --- core/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e5635589..d442bdbd 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -263,6 +263,15 @@ def configure_logging(): print('No log folder, logging to screen only') +def configure_process(): + global MYAPP + + MYAPP = RunningProcess() + while MYAPP.alreadyrunning(): + print('Waiting for existing session to end') + time.sleep(30) + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ @@ -287,11 +296,7 @@ def initialize(section=None): return False configure_logging() - - MYAPP = RunningProcess() - while MYAPP.alreadyrunning(): - print('Waiting for existing session to end') - time.sleep(30) + configure_process() try: locale.setlocale(locale.LC_ALL, '') From 13846db0b6f22d988fcbf5187360956fc7bfed90 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 02:28:41 -0500 Subject: [PATCH 091/406] Refactor locale configuration --- core/__init__.py | 55 ++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index d442bdbd..7633646c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -272,31 +272,8 @@ def configure_process(): time.sleep(30) -def initialize(section=None): - global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ - META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ - __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ - GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ - DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ - VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ - ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ - SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ - DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD - - if __INITIALIZED__: - return False - - configure_logging() - configure_process() +def configure_locale(): + global SYS_ENCODING try: locale.setlocale(locale.LC_ALL, '') @@ -325,6 +302,34 @@ def initialize(section=None): else: sys.exit(1) + +def initialize(section=None): + global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ + NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ + NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ + NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ + TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ + META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ + __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ + GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ + DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ + VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ + ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ + SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ + NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ + USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD + + if __INITIALIZED__: + return False + + configure_logging() + configure_process() + configure_locale() + # init logging logger.ntm_log_instance.init_logging() From a31683f7e58deaffb60d79662cfc37a44b0d635d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 02:33:28 -0500 Subject: [PATCH 092/406] Refactor migration configuration --- core/__init__.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 7633646c..9afadb23 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -303,10 +303,31 @@ def configure_locale(): sys.exit(1) +def configure_migration(): + global CONFIG_FILE + global CFG + + # run migrate to convert old cfg to new style cfg plus fix any cfg missing values/options. + if not config.migrate(): + logger.error('Unable to migrate config file {0}, exiting ...'.format(CONFIG_FILE)) + if 'NZBOP_SCRIPTDIR' in os.environ: + pass # We will try and read config from Environment. + else: + sys.exit(-1) + + # run migrate to convert NzbGet data from old cfg style to new cfg style + if 'NZBOP_SCRIPTDIR' in os.environ: + CFG = config.addnzbget() + + else: # load newly migrated config + logger.info('Loading config from [{0}]'.format(CONFIG_FILE)) + CFG = config() + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ + NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ @@ -333,21 +354,7 @@ def initialize(section=None): # init logging logger.ntm_log_instance.init_logging() - # run migrate to convert old cfg to new style cfg plus fix any cfg missing values/options. - if not config.migrate(): - logger.error('Unable to migrate config file {0}, exiting ...'.format(CONFIG_FILE)) - if 'NZBOP_SCRIPTDIR' in os.environ: - pass # We will try and read config from Environment. - else: - sys.exit(-1) - - # run migrate to convert NzbGet data from old cfg style to new cfg style - if 'NZBOP_SCRIPTDIR' in os.environ: - CFG = config.addnzbget() - - else: # load newly migrated config - logger.info('Loading config from [{0}]'.format(CONFIG_FILE)) - CFG = config() + configure_migration() # Enable/Disable DEBUG Logging LOG_DEBUG = int(CFG['General']['log_debug']) From ca17c7a562c5da38166d61b6e11d1efa66e9a242 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 02:38:00 -0500 Subject: [PATCH 093/406] Refactor logging configuration --- core/__init__.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 9afadb23..1e4454b0 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -324,6 +324,23 @@ def configure_migration(): CFG = config() +def configure_logging_part_2(): + global LOG_DB + global LOG_DEBUG + global LOG_ENV + global LOG_GIT + + # Enable/Disable DEBUG Logging + LOG_DB = int(CFG['General']['log_db']) + LOG_DEBUG = int(CFG['General']['log_debug']) + LOG_ENV = int(CFG['General']['log_env']) + LOG_GIT = int(CFG['General']['log_git']) + + if LOG_ENV: + for item in os.environ: + logger.info('{0}: {1}'.format(item, os.environ[item]), 'ENVIRONMENT') + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ @@ -331,17 +348,17 @@ def initialize(section=None): SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ - META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, LOG_GIT, GROUPS, SEVENZIP, CONCAT, VCRF, \ + META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ - VFRAMERATE, LOG_DB, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ + VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - NICENESS, LOG_DEBUG, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + NICENESS, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, LOG_ENV, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: @@ -355,16 +372,7 @@ def initialize(section=None): logger.ntm_log_instance.init_logging() configure_migration() - - # Enable/Disable DEBUG Logging - LOG_DEBUG = int(CFG['General']['log_debug']) - LOG_DB = int(CFG['General']['log_db']) - LOG_ENV = int(CFG['General']['log_env']) - LOG_GIT = int(CFG['General']['log_git']) - - if LOG_ENV: - for item in os.environ: - logger.info('{0}: {1}'.format(item, os.environ[item]), 'ENVIRONMENT') + configure_logging_part_2() # initialize the main SB database main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) From 2512218d4a4a66d1cc5f2d168d9e1314098adeaa Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:25:45 -0500 Subject: [PATCH 094/406] Refactor general configuration --- core/__init__.py | 52 +++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 1e4454b0..e9a22e9f 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -341,23 +341,50 @@ def configure_logging_part_2(): logger.info('{0}: {1}'.format(item, os.environ[item]), 'ENVIRONMENT') +def configure_general(): + global VERSION_NOTIFY + global AUTO_UPDATE + global GIT_REPO + global GIT_PATH + global GIT_USER + global GIT_BRANCH + global FORCE_CLEAN + global FFMPEG_PATH + global CHECK_MEDIA + global SAFE_MODE + global NOEXTRACTFAILED + + # Set Version and GIT variables + VERSION_NOTIFY = int(CFG['General']['version_notify']) + AUTO_UPDATE = int(CFG['General']['auto_update']) + GIT_REPO = 'nzbToMedia' + GIT_PATH = CFG['General']['git_path'] + GIT_USER = CFG['General']['git_user'] or 'clinton-hall' + GIT_BRANCH = CFG['General']['git_branch'] or 'master' + FORCE_CLEAN = int(CFG['General']['force_clean']) + FFMPEG_PATH = CFG['General']['ffmpeg_path'] + CHECK_MEDIA = int(CFG['General']['check_media']) + SAFE_MODE = int(CFG['General']['safe_mode']) + NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, \ + NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ + NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, GROUPS, SEVENZIP, CONCAT, VCRF, \ - __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, GIT_PATH, GIT_USER, \ - GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ + __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ + SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - NICENESS, FORCE_CLEAN, FFMPEG_PATH, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, CHECK_MEDIA, SAFE_MODE, \ + USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD @@ -377,18 +404,7 @@ def initialize(section=None): # initialize the main SB database main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) - # Set Version and GIT variables - VERSION_NOTIFY = int(CFG['General']['version_notify']) - AUTO_UPDATE = int(CFG['General']['auto_update']) - GIT_REPO = 'nzbToMedia' - GIT_PATH = CFG['General']['git_path'] - GIT_USER = CFG['General']['git_user'] or 'clinton-hall' - GIT_BRANCH = CFG['General']['git_branch'] or 'master' - FORCE_CLEAN = int(CFG['General']['force_clean']) - FFMPEG_PATH = CFG['General']['ffmpeg_path'] - CHECK_MEDIA = int(CFG['General']['check_media']) - SAFE_MODE = int(CFG['General']['safe_mode']) - NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) + configure_general() # Check for updates via GitHUB if version_check.CheckVersion().check_for_new_version(): From c9e9d9748b3770d86fc124991e258a2960dbfee0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:29:58 -0500 Subject: [PATCH 095/406] Refactor updates configuration --- core/__init__.py | 49 ++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e9a22e9f..b6cf596e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -343,7 +343,6 @@ def configure_logging_part_2(): def configure_general(): global VERSION_NOTIFY - global AUTO_UPDATE global GIT_REPO global GIT_PATH global GIT_USER @@ -356,7 +355,6 @@ def configure_general(): # Set Version and GIT variables VERSION_NOTIFY = int(CFG['General']['version_notify']) - AUTO_UPDATE = int(CFG['General']['auto_update']) GIT_REPO = 'nzbToMedia' GIT_PATH = CFG['General']['git_path'] GIT_USER = CFG['General']['git_user'] or 'clinton-hall' @@ -368,6 +366,32 @@ def configure_general(): NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) +def configure_updates(): + global AUTO_UPDATE + + AUTO_UPDATE = int(CFG['General']['auto_update']) + + # Check for updates via GitHUB + if version_check.CheckVersion().check_for_new_version(): + if AUTO_UPDATE == 1: + logger.info('Auto-Updating nzbToMedia, Please wait ...') + updated = version_check.CheckVersion().update() + if updated: + # restart nzbToMedia + try: + del MYAPP + except Exception: + pass + restart() + else: + logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') + + # Set Current Version + logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format + (version=NZBTOMEDIA_VERSION, branch=GIT_BRANCH, + system=platform.system(), release=platform.release())) + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -405,26 +429,7 @@ def initialize(section=None): main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) configure_general() - - # Check for updates via GitHUB - if version_check.CheckVersion().check_for_new_version(): - if AUTO_UPDATE == 1: - logger.info('Auto-Updating nzbToMedia, Please wait ...') - updated = version_check.CheckVersion().update() - if updated: - # restart nzbToMedia - try: - del MYAPP - except Exception: - pass - restart() - else: - logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') - - # Set Current Version - logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format - (version=NZBTOMEDIA_VERSION, branch=GIT_BRANCH, - system=platform.system(), release=platform.release())) + configure_updates() if int(CFG['WakeOnLan']['wake']) == 1: wake_up() From 750c203216b635a99842ed312ede0cfdcef07258 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:32:18 -0500 Subject: [PATCH 096/406] Fix CheckVersion instance creation --- core/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index b6cf596e..f36767dc 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -370,12 +370,13 @@ def configure_updates(): global AUTO_UPDATE AUTO_UPDATE = int(CFG['General']['auto_update']) + version_checker = version_check.CheckVersion() # Check for updates via GitHUB - if version_check.CheckVersion().check_for_new_version(): + if version_checker.check_for_new_version(): if AUTO_UPDATE == 1: logger.info('Auto-Updating nzbToMedia, Please wait ...') - updated = version_check.CheckVersion().update() + updated = version_checker.update() if updated: # restart nzbToMedia try: From bd4c8303136a295a3a33d6478381b55c6defca8b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:32:47 -0500 Subject: [PATCH 097/406] Fix version check conditional --- core/__init__.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index f36767dc..3875a5b2 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -373,19 +373,17 @@ def configure_updates(): version_checker = version_check.CheckVersion() # Check for updates via GitHUB - if version_checker.check_for_new_version(): - if AUTO_UPDATE == 1: - logger.info('Auto-Updating nzbToMedia, Please wait ...') - updated = version_checker.update() - if updated: - # restart nzbToMedia - try: - del MYAPP - except Exception: - pass - restart() - else: - logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') + if version_checker.check_for_new_version() and AUTO_UPDATE: + logger.info('Auto-Updating nzbToMedia, Please wait ...') + if version_checker.update(): + # restart nzbToMedia + try: + del MYAPP + except Exception: + pass + restart() + else: + logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') # Set Current Version logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format From 2c963f1ffe9f59818c5f346a4d558ff386e395d4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:35:11 -0500 Subject: [PATCH 098/406] Fix error log --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 3875a5b2..d6757e73 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -383,7 +383,7 @@ def configure_updates(): pass restart() else: - logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') + logger.error('Update failed, not restarting. Check your log for more information.') # Set Current Version logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format From feffa0da41005adb78f0f8407748edc036cfb29e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:36:50 -0500 Subject: [PATCH 099/406] Refactor wake on lan configuration --- core/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index d6757e73..fdbed9e8 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -391,6 +391,11 @@ def configure_updates(): system=platform.system(), release=platform.release())) +def configure_wake_on_lan(): + if int(CFG['WakeOnLan']['wake']): + wake_up() + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -429,9 +434,7 @@ def initialize(section=None): configure_general() configure_updates() - - if int(CFG['WakeOnLan']['wake']) == 1: - wake_up() + configure_wake_on_lan() NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] From b3870e0d07a19c41605181b7a59c26367ce6c949 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:42:51 -0500 Subject: [PATCH 100/406] Refactor nzbs configuration --- core/__init__.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index fdbed9e8..e633826f 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -396,6 +396,20 @@ def configure_wake_on_lan(): wake_up() +def configure_nzbs(): + global NZB_CLIENT_AGENT + global SABNZBD_HOST + global SABNZBD_PORT + global SABNZBD_APIKEY + global NZB_DEFAULT_DIRECTORY + + NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd + SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] + SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accommodate NzbGet + SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] + NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -405,7 +419,7 @@ def initialize(section=None): TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, GROUPS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ - SYS_ENCODING, NZB_CLIENT_AGENT, SABNZBD_HOST, SABNZBD_PORT, SABNZBD_APIKEY, \ + SYS_ENCODING, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ @@ -413,7 +427,7 @@ def initialize(section=None): NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, NZB_DEFAULT_DIRECTORY, REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD if __INITIALIZED__: @@ -435,12 +449,8 @@ def initialize(section=None): configure_general() configure_updates() configure_wake_on_lan() + configure_nzbs() - NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd - SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] - SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet - SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] - NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] GROUPS = CFG['Custom']['remove_group'] if isinstance(GROUPS, str): GROUPS = GROUPS.split(',') From 62aca7ed3c572b208c85c0efd3d4b780cbbae48e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 03:45:13 -0500 Subject: [PATCH 101/406] Refactor groups configuration --- core/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e633826f..43e3a29c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -410,6 +410,18 @@ def configure_nzbs(): NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] +def configure_groups(): + global GROUPS + + GROUPS = CFG['Custom']['remove_group'] + + if isinstance(GROUPS, str): + GROUPS = GROUPS.split(',') + + if GROUPS == ['']: + GROUPS = None + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -417,7 +429,7 @@ def initialize(section=None): SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ - META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, GROUPS, SEVENZIP, CONCAT, VCRF, \ + META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ SYS_ENCODING, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ @@ -451,12 +463,6 @@ def initialize(section=None): configure_wake_on_lan() configure_nzbs() - GROUPS = CFG['Custom']['remove_group'] - if isinstance(GROUPS, str): - GROUPS = GROUPS.split(',') - if GROUPS == ['']: - GROUPS = None - TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ From 003d181bb0a79aba14f66809292456957f99093c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:10:27 -0500 Subject: [PATCH 102/406] Refactor torrents configuration --- core/__init__.py | 109 ++++++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 40 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 43e3a29c..308f4059 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -422,46 +422,32 @@ def configure_groups(): GROUPS = None -def initialize(section=None): - global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENT_AGENT, USE_LINK, OUTPUT_DIRECTORY, \ - NOFLATTEN, UTORRENT_PASSWORD, UTORRENT_USER, UTORRENT_WEB_UI, DELUGE_HOST, DELUGE_PORT, DELUGE_USER, DELUGE_PASSWORD, VLEVEL, \ - TRANSMISSION_HOST, TRANSMISSION_PORT, TRANSMISSION_PASSWORD, TRANSMISSION_USER, COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ - META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ - __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ - SYS_ENCODING, \ - DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ - VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ - ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ - SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ - DELETE_ORIGINAL, TORRENT_CHMOD_DIRECTORY, PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ - TORRENT_DEFAULT_DIRECTORY, TORRENT_RESUME_ON_FAILURE, REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, TORRENT_RESUME, PAR2CMD, QBITTORRENT_HOST, QBITTORRENT_PORT, QBITTORRENT_USER, QBITTORRENT_PASSWORD - - if __INITIALIZED__: - return False - - configure_logging() - configure_process() - configure_locale() - - # init logging - logger.ntm_log_instance.init_logging() - - configure_migration() - configure_logging_part_2() - - # initialize the main SB database - main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) - - configure_general() - configure_updates() - configure_wake_on_lan() - configure_nzbs() +def configure_torrents(): + global TORRENT_CLIENT_AGENT + global USE_LINK + global OUTPUT_DIRECTORY + global TORRENT_DEFAULT_DIRECTORY + global CATEGORIES + global NOFLATTEN + global DELETE_ORIGINAL + global TORRENT_CHMOD_DIRECTORY + global TORRENT_RESUME_ON_FAILURE + global TORRENT_RESUME + global UTORRENT_WEB_UI + global UTORRENT_USER + global UTORRENT_PASSWORD + global TRANSMISSION_HOST + global TRANSMISSION_PORT + global TRANSMISSION_USER + global TRANSMISSION_PASSWORD + global DELUGE_HOST + global DELUGE_PORT + global DELUGE_USER + global DELUGE_PASSWORD + global QBITTORRENT_HOST + global QBITTORRENT_PORT + global QBITTORRENT_USER + global QBITTORRENT_PASSWORD TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym @@ -496,6 +482,49 @@ def initialize(section=None): QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr + +def initialize(section=None): + global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ + NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ + NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, \ + VLEVEL, \ + COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ + META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ + __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ + SYS_ENCODING, \ + DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ + VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ + ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ + SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ + NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ + USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ + REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, PAR2CMD + + if __INITIALIZED__: + return False + + configure_logging() + configure_process() + configure_locale() + + # init logging + logger.ntm_log_instance.init_logging() + + configure_migration() + configure_logging_part_2() + + # initialize the main SB database + main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) + + configure_general() + configure_updates() + configure_wake_on_lan() + configure_nzbs() + configure_torrents() + REMOTE_PATHS = CFG['Network']['mount_points'] or [] if REMOTE_PATHS: if isinstance(REMOTE_PATHS, list): From c587a137a570a23f47f552ba19b852d5dbce8479 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:41:38 -0500 Subject: [PATCH 103/406] Refactor remote paths configuration --- core/__init__.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 308f4059..a1e47591 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -483,6 +483,20 @@ def configure_torrents(): QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr +def configure_remote_paths(): + global REMOTE_PATHS + + REMOTE_PATHS = CFG['Network']['mount_points'] or [] + + if REMOTE_PATHS: + if isinstance(REMOTE_PATHS, list): + REMOTE_PATHS = ','.join(REMOTE_PATHS) # fix in case this imported as list. + REMOTE_PATHS = [tuple(item.split(',')) for item in + REMOTE_PATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\ + REMOTE_PATHS = [(local.strip(), remote.strip()) for local, remote in + REMOTE_PATHS] # strip trailing and leading whitespaces + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -500,7 +514,7 @@ def initialize(section=None): NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ - REMOTE_PATHS, PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ + PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, PAR2CMD if __INITIALIZED__: @@ -524,15 +538,7 @@ def initialize(section=None): configure_wake_on_lan() configure_nzbs() configure_torrents() - - REMOTE_PATHS = CFG['Network']['mount_points'] or [] - if REMOTE_PATHS: - if isinstance(REMOTE_PATHS, list): - REMOTE_PATHS = ','.join(REMOTE_PATHS) # fix in case this imported as list. - REMOTE_PATHS = [tuple(item.split(',')) for item in - REMOTE_PATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\ - REMOTE_PATHS = [(local.strip(), remote.strip()) for local, remote in - REMOTE_PATHS] # strip trailing and leading whitespaces + configure_remote_paths() PLEX_SSL = int(CFG['Plex']['plex_ssl']) PLEX_HOST = CFG['Plex']['plex_host'] From a5d51d6e5a752754b4d7fda70d19b3e3308a5917 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:45:38 -0500 Subject: [PATCH 104/406] Use generator exp for remote paths --- core/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index a1e47591..b3d02541 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -491,10 +491,18 @@ def configure_remote_paths(): if REMOTE_PATHS: if isinstance(REMOTE_PATHS, list): REMOTE_PATHS = ','.join(REMOTE_PATHS) # fix in case this imported as list. - REMOTE_PATHS = [tuple(item.split(',')) for item in - REMOTE_PATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\ - REMOTE_PATHS = [(local.strip(), remote.strip()) for local, remote in - REMOTE_PATHS] # strip trailing and leading whitespaces + + REMOTE_PATHS = ( + # /volume1/Public/,E:\|/volume2/share/,\\NAS\ + tuple(item.split(',')) + for item in REMOTE_PATHS.split('|') + ) + + REMOTE_PATHS = [ + # strip trailing and leading whitespaces + (local.strip(), remote.strip()) + for local, remote in REMOTE_PATHS + ] def initialize(section=None): From f67f8a32aa8fc80371705d7d158b7f0b8b0e0f17 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:48:49 -0500 Subject: [PATCH 105/406] Refactor plex configuration --- core/__init__.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index b3d02541..fbdfeb21 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -505,6 +505,28 @@ def configure_remote_paths(): ] +def configure_plex(): + global PLEX_SSL + global PLEX_HOST + global PLEX_PORT + global PLEX_TOKEN + global PLEX_SECTION + + PLEX_SSL = int(CFG['Plex']['plex_ssl']) + PLEX_HOST = CFG['Plex']['plex_host'] + PLEX_PORT = CFG['Plex']['plex_port'] + PLEX_TOKEN = CFG['Plex']['plex_token'] + PLEX_SECTION = CFG['Plex']['plex_sections'] or [] + + if PLEX_SECTION: + if isinstance(PLEX_SECTION, list): + PLEX_SECTION = ','.join(PLEX_SECTION) # fix in case this imported as list. + PLEX_SECTION = [ + tuple(item.split(',')) + for item in PLEX_SECTION.split('|') + ] + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -523,7 +545,7 @@ def initialize(section=None): PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PLEX_SSL, PLEX_HOST, PLEX_PORT, PLEX_TOKEN, PLEX_SECTION, PAR2CMD + PAR2CMD if __INITIALIZED__: return False @@ -547,16 +569,7 @@ def initialize(section=None): configure_nzbs() configure_torrents() configure_remote_paths() - - PLEX_SSL = int(CFG['Plex']['plex_ssl']) - PLEX_HOST = CFG['Plex']['plex_host'] - PLEX_PORT = CFG['Plex']['plex_port'] - PLEX_TOKEN = CFG['Plex']['plex_token'] - PLEX_SECTION = CFG['Plex']['plex_sections'] or [] - if PLEX_SECTION: - if isinstance(PLEX_SECTION, list): - PLEX_SECTION = ','.join(PLEX_SECTION) # fix in case this imported as list. - PLEX_SECTION = [tuple(item.split(',')) for item in PLEX_SECTION.split('|')] + configure_plex() devnull = open(os.devnull, 'w') try: From e0c55c4f844acb641c7e428582e26a6ae6833507 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:51:14 -0500 Subject: [PATCH 106/406] Refactor niceness configuration --- core/__init__.py | 53 ++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index fbdfeb21..c5cfd7a1 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -527,6 +527,33 @@ def configure_plex(): ] +def configure_niceness(): + global NICENESS + + devnull = open(os.devnull, 'w') + try: + subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) + except Exception: + pass + try: + subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() + try: + NICENESS.extend(['ionice', '-c{0}'.format(int(CFG['Posix']['ionice_class']))]) + except Exception: + pass + try: + if 'ionice' in NICENESS: + NICENESS.extend(['-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + else: + NICENESS.extend(['ionice', '-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + except Exception: + pass + except Exception: + pass + devnull.close() + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ @@ -541,7 +568,7 @@ def initialize(section=None): VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - NICENESS, FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ @@ -570,29 +597,7 @@ def initialize(section=None): configure_torrents() configure_remote_paths() configure_plex() - - devnull = open(os.devnull, 'w') - try: - subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) - except Exception: - pass - try: - subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() - try: - NICENESS.extend(['ionice', '-c{0}'.format(int(CFG['Posix']['ionice_class']))]) - except Exception: - pass - try: - if 'ionice' in NICENESS: - NICENESS.extend(['-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) - else: - NICENESS.extend(['ionice', '-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) - except Exception: - pass - except Exception: - pass - devnull.close() + configure_niceness() COMPRESSED_CONTAINER = [re.compile(r'.r\d{2}$', re.I), re.compile(r'.part\d+.rar$', re.I), From ddf15247e341e14cac97bba24d0301d526498958 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 11:53:29 -0500 Subject: [PATCH 107/406] Use context manager instead of assignment --- core/__init__.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index c5cfd7a1..730068a4 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -530,28 +530,27 @@ def configure_plex(): def configure_niceness(): global NICENESS - devnull = open(os.devnull, 'w') - try: - subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) - except Exception: - pass - try: - subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() + with open(os.devnull, 'w') as devnull: try: - NICENESS.extend(['ionice', '-c{0}'.format(int(CFG['Posix']['ionice_class']))]) + subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) except Exception: pass try: - if 'ionice' in NICENESS: - NICENESS.extend(['-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) - else: - NICENESS.extend(['ionice', '-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() + try: + NICENESS.extend(['ionice', '-c{0}'.format(int(CFG['Posix']['ionice_class']))]) + except Exception: + pass + try: + if 'ionice' in NICENESS: + NICENESS.extend(['-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + else: + NICENESS.extend(['ionice', '-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + except Exception: + pass except Exception: pass - except Exception: - pass - devnull.close() def initialize(section=None): From 10710ffd4cc35e11d794f1f523d98568aac27e98 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:00:00 -0500 Subject: [PATCH 108/406] Refactor container configuration --- core/__init__.py | 50 ++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 730068a4..f7c7f867 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -553,21 +553,48 @@ def configure_niceness(): pass +def configure_containers(): + global COMPRESSED_CONTAINER + global MEDIA_CONTAINER + global AUDIO_CONTAINER + global META_CONTAINER + + COMPRESSED_CONTAINER = [re.compile(r'.r\d{2}$', re.I), + re.compile(r'.part\d+.rar$', re.I), + re.compile('.rar$', re.I)] + COMPRESSED_CONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in + CFG['Extensions']['compressedExtensions']] + MEDIA_CONTAINER = CFG['Extensions']['mediaExtensions'] + AUDIO_CONTAINER = CFG['Extensions']['audioExtensions'] + META_CONTAINER = CFG['Extensions']['metaExtensions'] # .nfo,.sub,.srt + + if isinstance(COMPRESSED_CONTAINER, str): + COMPRESSED_CONTAINER = COMPRESSED_CONTAINER.split(',') + + if isinstance(MEDIA_CONTAINER, str): + MEDIA_CONTAINER = MEDIA_CONTAINER.split(',') + + if isinstance(AUDIO_CONTAINER, str): + AUDIO_CONTAINER = AUDIO_CONTAINER.split(',') + + if isinstance(META_CONTAINER, str): + META_CONTAINER = META_CONTAINER.split(',') + + def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, \ VLEVEL, \ - COMPRESSED_CONTAINER, MEDIA_CONTAINER, \ - META_CONTAINER, SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ + SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ SYS_ENCODING, \ DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - FFMPEG, FFPROBE, AUDIO_CONTAINER, EXT_CONTAINER, TORRENT_CLASS, \ + FFMPEG, FFPROBE, EXT_CONTAINER, TORRENT_CLASS, \ PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ @@ -597,22 +624,7 @@ def initialize(section=None): configure_remote_paths() configure_plex() configure_niceness() - - COMPRESSED_CONTAINER = [re.compile(r'.r\d{2}$', re.I), - re.compile(r'.part\d+.rar$', re.I), - re.compile('.rar$', re.I)] - COMPRESSED_CONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG['Extensions']['compressedExtensions']] - MEDIA_CONTAINER = CFG['Extensions']['mediaExtensions'] - AUDIO_CONTAINER = CFG['Extensions']['audioExtensions'] - META_CONTAINER = CFG['Extensions']['metaExtensions'] # .nfo,.sub,.srt - if isinstance(COMPRESSED_CONTAINER, str): - COMPRESSED_CONTAINER = COMPRESSED_CONTAINER.split(',') - if isinstance(MEDIA_CONTAINER, str): - MEDIA_CONTAINER = MEDIA_CONTAINER.split(',') - if isinstance(AUDIO_CONTAINER, str): - AUDIO_CONTAINER = AUDIO_CONTAINER.split(',') - if isinstance(META_CONTAINER, str): - META_CONTAINER = META_CONTAINER.split(',') + configure_containers() GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) From 2d0b5e706b73a9036cde8c7946a3b588c4c80243 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:19:54 -0500 Subject: [PATCH 109/406] Refactor transcoder configuration --- core/__init__.py | 130 +++++++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 44 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index f7c7f867..c84d94f1 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -581,50 +581,49 @@ def configure_containers(): META_CONTAINER = META_CONTAINER.split(',') -def initialize(section=None): - global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, \ - VLEVEL, \ - SECTIONS, ALL_FORKS, TEST_FILE, GENERALOPTS, SEVENZIP, CONCAT, VCRF, \ - __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, TRANSCODE, DEFAULTS, \ - SYS_ENCODING, \ - DUPLICATE, IGNOREEXTENSIONS, VEXTENSION, OUTPUTVIDEOPATH, PROCESSOUTPUT, VCODEC, VCODEC_ALLOW, VPRESET, \ - VFRAMERATE, VBITRATE, VRESOLUTION, ALANGUAGE, AINCLUDE, ACODEC, ACODEC_ALLOW, ABITRATE, FAILED, \ - ACODEC2, ACODEC2_ALLOW, ABITRATE2, ACODEC3, ACODEC3_ALLOW, ABITRATE3, ALLOWSUBS, SEXTRACT, SEMBED, SLANGUAGES, \ - SINCLUDE, SUBSDIR, SCODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, BURN, GETSUBS, HWACCEL, \ - FFMPEG, FFPROBE, EXT_CONTAINER, TORRENT_CLASS, \ - PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ - PID_FILE, MYAPP, ACHANNELS, ACHANNELS2, ACHANNELS3, \ - PAR2CMD - - if __INITIALIZED__: - return False - - configure_logging() - configure_process() - configure_locale() - - # init logging - logger.ntm_log_instance.init_logging() - - configure_migration() - configure_logging_part_2() - - # initialize the main SB database - main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) - - configure_general() - configure_updates() - configure_wake_on_lan() - configure_nzbs() - configure_torrents() - configure_remote_paths() - configure_plex() - configure_niceness() - configure_containers() +def configure_transcoder(): + global GETSUBS + global TRANSCODE + global DUPLICATE + global CONCAT + global IGNOREEXTENSIONS + global OUTPUTFASTSTART + global GENERALOPTS + global OUTPUTQUALITYPERCENT + global OUTPUTVIDEOPATH + global PROCESSOUTPUT + global ALANGUAGE + global AINCLUDE + global SLANGUAGES + global SINCLUDE + global SEXTRACT + global SEMBED + global SUBSDIR + global VEXTENSION + global VCODEC + global VPRESET + global VFRAMERATE + global VBITRATE + global VRESOLUTION + global VCRF + global VLEVEL + global VCODEC_ALLOW + global ACODEC + global ACODEC_ALLOW + global ACHANNELS + global ABITRATE + global ACODEC2 + global ACODEC2_ALLOW + global ACHANNELS2 + global ABITRATE2 + global ACODEC3 + global ACODEC3_ALLOW + global ACHANNELS3 + global ABITRATE3 + global SCODEC + global BURN + global HWACCEL + global ALLOWSUBS GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) @@ -906,6 +905,49 @@ def initialize(section=None): extra = [item for item in codec_alias[codec] if item not in ACODEC3_ALLOW] ACODEC3_ALLOW.extend(extra) + +def initialize(section=None): + global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ + NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ + NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ + SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, \ + SECTIONS, ALL_FORKS, TEST_FILE, SEVENZIP, \ + __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, DEFAULTS, \ + SYS_ENCODING, \ + FAILED, \ + FFMPEG, FFPROBE, EXT_CONTAINER, TORRENT_CLASS, \ + PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ + USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ + PID_FILE, MYAPP, \ + PAR2CMD + + if __INITIALIZED__: + return False + + configure_logging() + configure_process() + configure_locale() + + # init logging + logger.ntm_log_instance.init_logging() + + configure_migration() + configure_logging_part_2() + + # initialize the main SB database + main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) + + configure_general() + configure_updates() + configure_wake_on_lan() + configure_nzbs() + configure_torrents() + configure_remote_paths() + configure_plex() + configure_niceness() + configure_containers() + configure_transcoder() + PASSWORDS_FILE = CFG['passwords']['PassWordFile'] # Setup FFMPEG, FFPROBE and SEVENZIP locations From 819cf7b225e2c77aa48da271a82f3a193f1b3e22 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:33:10 -0500 Subject: [PATCH 110/406] Fix global declarations --- core/__init__.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index c84d94f1..82c5def2 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -368,6 +368,7 @@ def configure_general(): def configure_updates(): global AUTO_UPDATE + global MYAPP AUTO_UPDATE = int(CFG['General']['auto_update']) version_checker = version_check.CheckVersion() @@ -624,6 +625,7 @@ def configure_transcoder(): global BURN global HWACCEL global ALLOWSUBS + global DEFAULTS GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) @@ -907,19 +909,18 @@ def configure_transcoder(): def initialize(section=None): - global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PAR_CHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, SHOWEXTRACT, \ - NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, SYS_ARGV, \ - SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, \ - SECTIONS, ALL_FORKS, TEST_FILE, SEVENZIP, \ - __INITIALIZED__, APP_FILENAME, USER_DELAY, APP_NAME, DEFAULTS, \ - SYS_ENCODING, \ - FAILED, \ - FFMPEG, FFPROBE, EXT_CONTAINER, TORRENT_CLASS, \ - PASSWORDS_FILE, USER_DELAY, USER_SCRIPT, USER_SCRIPT_CLEAN, USER_SCRIPT_MEDIAEXTENSIONS, \ - USER_SCRIPT_PARAM, USER_SCRIPT_RUNONCE, USER_SCRIPT_SUCCESSCODES, DOWNLOAD_INFO, \ - PID_FILE, MYAPP, \ - PAR2CMD + global SHOWEXTRACT + global CATEGORIES + global SECTIONS + global SEVENZIP + global __INITIALIZED__ + global USER_DELAY + global SYS_ENCODING + global FFMPEG + global FFPROBE + global TORRENT_CLASS + global PASSWORDS_FILE + global PAR2CMD if __INITIALIZED__: return False From e85b92f1db6d67bb7b9bb2dbd3f885babd3b6e27 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:35:14 -0500 Subject: [PATCH 111/406] Refactor passwords file configuration --- core/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 82c5def2..4885c763 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -908,6 +908,12 @@ def configure_transcoder(): ACODEC3_ALLOW.extend(extra) +def configure_passwords_file(): + global PASSWORDS_FILE + + PASSWORDS_FILE = CFG['passwords']['PassWordFile'] + + def initialize(section=None): global SHOWEXTRACT global CATEGORIES @@ -948,8 +954,7 @@ def initialize(section=None): configure_niceness() configure_containers() configure_transcoder() - - PASSWORDS_FILE = CFG['passwords']['PassWordFile'] + configure_passwords_file() # Setup FFMPEG, FFPROBE and SEVENZIP locations if platform.system() == 'Windows': From 4f828e0a77624d2177c9b6cea55c3f35534c8607 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:37:57 -0500 Subject: [PATCH 112/406] Refactor torrent class configuration --- core/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 4885c763..be1fa13e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -914,6 +914,13 @@ def configure_passwords_file(): PASSWORDS_FILE = CFG['passwords']['PassWordFile'] +def configure_torrent_class(): + global TORRENT_CLASS + + # create torrent class + TORRENT_CLASS = create_torrent_class(TORRENT_CLIENT_AGENT) + + def initialize(section=None): global SHOWEXTRACT global CATEGORIES @@ -1050,8 +1057,7 @@ def initialize(section=None): CATEGORIES.extend([subsection for subsection in subsections if CFG[section][subsection].isenabled()]) CATEGORIES = list(set(CATEGORIES)) - # create torrent class - TORRENT_CLASS = create_torrent_class(TORRENT_CLIENT_AGENT) + configure_torrent_class() # finished initalizing return True From 0a58b6b6a02764cab0a5ddac8ad9e41898015432 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:40:02 -0500 Subject: [PATCH 113/406] Refactor section configuration --- core/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index be1fa13e..d6e61db1 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -921,6 +921,19 @@ def configure_torrent_class(): TORRENT_CLASS = create_torrent_class(TORRENT_CLIENT_AGENT) +def configure_sections(section): + global SECTIONS + global CATEGORIES + # check for script-defied section and if None set to allow sections + SECTIONS = CFG[ + tuple(x for x in CFG if CFG[x].sections and CFG[x].isenabled()) + if not section else (section,) + ] + for section, subsections in SECTIONS.items(): + CATEGORIES.extend([subsection for subsection in subsections if CFG[section][subsection].isenabled()]) + CATEGORIES = list(set(CATEGORIES)) + + def initialize(section=None): global SHOWEXTRACT global CATEGORIES @@ -1051,12 +1064,7 @@ def initialize(section=None): logger.warning('Failed to locate ffprobe. Video corruption detection disabled!') logger.warning('Install ffmpeg with x264 support to enable this feature ...') - # check for script-defied section and if None set to allow sections - SECTIONS = CFG[tuple(x for x in CFG if CFG[x].sections and CFG[x].isenabled()) if not section else (section,)] - for section, subsections in SECTIONS.items(): - CATEGORIES.extend([subsection for subsection in subsections if CFG[section][subsection].isenabled()]) - CATEGORIES = list(set(CATEGORIES)) - + configure_sections(section) configure_torrent_class() # finished initalizing From 3d2070e106ce7d89424048401188d9c446d3c30f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:43:02 -0500 Subject: [PATCH 114/406] Refactor utility location configuration --- core/__init__.py | 69 +++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index d6e61db1..2213276e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -934,48 +934,13 @@ def configure_sections(section): CATEGORIES = list(set(CATEGORIES)) -def initialize(section=None): +def configure_utility_locations(): global SHOWEXTRACT - global CATEGORIES - global SECTIONS global SEVENZIP - global __INITIALIZED__ - global USER_DELAY - global SYS_ENCODING global FFMPEG global FFPROBE - global TORRENT_CLASS - global PASSWORDS_FILE global PAR2CMD - if __INITIALIZED__: - return False - - configure_logging() - configure_process() - configure_locale() - - # init logging - logger.ntm_log_instance.init_logging() - - configure_migration() - configure_logging_part_2() - - # initialize the main SB database - main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) - - configure_general() - configure_updates() - configure_wake_on_lan() - configure_nzbs() - configure_torrents() - configure_remote_paths() - configure_plex() - configure_niceness() - configure_containers() - configure_transcoder() - configure_passwords_file() - # Setup FFMPEG, FFPROBE and SEVENZIP locations if platform.system() == 'Windows': FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg.exe') @@ -1064,6 +1029,38 @@ def initialize(section=None): logger.warning('Failed to locate ffprobe. Video corruption detection disabled!') logger.warning('Install ffmpeg with x264 support to enable this feature ...') + +def initialize(section=None): + global __INITIALIZED__ + + if __INITIALIZED__: + return False + + configure_logging() + configure_process() + configure_locale() + + # init logging + logger.ntm_log_instance.init_logging() + + configure_migration() + configure_logging_part_2() + + # initialize the main SB database + main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) + + configure_general() + configure_updates() + configure_wake_on_lan() + configure_nzbs() + configure_torrents() + configure_remote_paths() + configure_plex() + configure_niceness() + configure_containers() + configure_transcoder() + configure_passwords_file() + configure_utility_locations() configure_sections(section) configure_torrent_class() From 287e3aa17b55912c773d8130195b996a1ddc7bfa Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 12:48:55 -0500 Subject: [PATCH 115/406] Fix initializing constant --- core/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 2213276e..2ad98d34 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1064,5 +1064,7 @@ def initialize(section=None): configure_sections(section) configure_torrent_class() - # finished initalizing - return True + __INITIALIZED__ = True + + # finished initializing + return __INITIALIZED__ From f961c476aefbc99abaa8ae50799efc97b177661f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:06:06 -0500 Subject: [PATCH 116/406] Refactor transmission configuration --- core/__init__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 2ad98d34..a8fd8d89 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -423,6 +423,18 @@ def configure_groups(): GROUPS = None +def configure_transmission(): + global TRANSMISSION_HOST + global TRANSMISSION_PORT + global TRANSMISSION_USER + global TRANSMISSION_PASSWORD + + TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost + TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) + TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr + TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK @@ -437,10 +449,6 @@ def configure_torrents(): global UTORRENT_WEB_UI global UTORRENT_USER global UTORRENT_PASSWORD - global TRANSMISSION_HOST - global TRANSMISSION_PORT - global TRANSMISSION_USER - global TRANSMISSION_PASSWORD global DELUGE_HOST global DELUGE_PORT global DELUGE_USER @@ -468,10 +476,7 @@ def configure_torrents(): UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr - TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost - TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) - TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr - TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr + configure_transmission() DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 From 44df360fbea37202225c3f668349f536b38137d0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:07:30 -0500 Subject: [PATCH 117/406] Refactor utorrent configuration --- core/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index a8fd8d89..d598f0a4 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -423,6 +423,16 @@ def configure_groups(): GROUPS = None +def configure_utorrent(): + global UTORRENT_WEB_UI + global UTORRENT_USER + global UTORRENT_PASSWORD + + UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ + UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr + UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr + + def configure_transmission(): global TRANSMISSION_HOST global TRANSMISSION_PORT @@ -446,9 +456,6 @@ def configure_torrents(): global TORRENT_CHMOD_DIRECTORY global TORRENT_RESUME_ON_FAILURE global TORRENT_RESUME - global UTORRENT_WEB_UI - global UTORRENT_USER - global UTORRENT_PASSWORD global DELUGE_HOST global DELUGE_PORT global DELUGE_USER @@ -472,10 +479,8 @@ def configure_torrents(): TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) TORRENT_RESUME = int(CFG['Torrent']['resume']) - UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ - UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr - UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr + configure_utorrent() configure_transmission() DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost From 22dfadd65c4027ca747187915cb382af906f7b77 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:08:28 -0500 Subject: [PATCH 118/406] Refactor deluge configuration --- core/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index d598f0a4..f0a1afc9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -445,6 +445,18 @@ def configure_transmission(): TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr +def configure_deluge(): + global DELUGE_HOST + global DELUGE_PORT + global DELUGE_USER + global DELUGE_PASSWORD + + DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost + DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 + DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr + DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK @@ -456,10 +468,6 @@ def configure_torrents(): global TORRENT_CHMOD_DIRECTORY global TORRENT_RESUME_ON_FAILURE global TORRENT_RESUME - global DELUGE_HOST - global DELUGE_PORT - global DELUGE_USER - global DELUGE_PASSWORD global QBITTORRENT_HOST global QBITTORRENT_PORT global QBITTORRENT_USER @@ -482,11 +490,7 @@ def configure_torrents(): configure_utorrent() configure_transmission() - - DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost - DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 - DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr - DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr + configure_deluge() QBITTORRENT_HOST = CFG['Torrent']['qBittorrenHost'] # localhost QBITTORRENT_PORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 From 9c105061d649e6e6720df3c61c593dd64677db1c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:09:49 -0500 Subject: [PATCH 119/406] Refactor qbittorrent configuration --- core/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index f0a1afc9..a4289541 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -457,6 +457,18 @@ def configure_deluge(): DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr +def configure_qbittorrent(): + global QBITTORRENT_HOST + global QBITTORRENT_PORT + global QBITTORRENT_USER + global QBITTORRENT_PASSWORD + + QBITTORRENT_HOST = CFG['Torrent']['qBittorrenHost'] # localhost + QBITTORRENT_PORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 + QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr + QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK @@ -468,10 +480,6 @@ def configure_torrents(): global TORRENT_CHMOD_DIRECTORY global TORRENT_RESUME_ON_FAILURE global TORRENT_RESUME - global QBITTORRENT_HOST - global QBITTORRENT_PORT - global QBITTORRENT_USER - global QBITTORRENT_PASSWORD TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym @@ -491,11 +499,7 @@ def configure_torrents(): configure_utorrent() configure_transmission() configure_deluge() - - QBITTORRENT_HOST = CFG['Torrent']['qBittorrenHost'] # localhost - QBITTORRENT_PORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 - QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr - QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr + configure_qbittorrent() def configure_remote_paths(): From 218e082ec78a63099e7afa382fed31acb99a0890 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:19:13 -0500 Subject: [PATCH 120/406] Refactor sabnzbd configuration --- core/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index a4289541..e100f20e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -397,19 +397,25 @@ def configure_wake_on_lan(): wake_up() -def configure_nzbs(): - global NZB_CLIENT_AGENT +def configure_sabnzbd(): global SABNZBD_HOST global SABNZBD_PORT global SABNZBD_APIKEY - global NZB_DEFAULT_DIRECTORY - NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accommodate NzbGet SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] + + +def configure_nzbs(): + global NZB_CLIENT_AGENT + global NZB_DEFAULT_DIRECTORY + + NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] + configure_sabnzbd() + def configure_groups(): global GROUPS From 1906d626648fe3277d2c1eaed3738dd876449e14 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:20:53 -0500 Subject: [PATCH 121/406] Refactor flatenning configuration --- core/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e100f20e..a1e6512e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -475,13 +475,20 @@ def configure_qbittorrent(): QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr +def configure_flattening(): + global NOFLATTEN + + NOFLATTEN = (CFG['Torrent']['noFlatten']) + if isinstance(NOFLATTEN, str): + NOFLATTEN = NOFLATTEN.split(',') + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK global OUTPUT_DIRECTORY global TORRENT_DEFAULT_DIRECTORY global CATEGORIES - global NOFLATTEN global DELETE_ORIGINAL global TORRENT_CHMOD_DIRECTORY global TORRENT_RESUME_ON_FAILURE @@ -492,9 +499,9 @@ def configure_torrents(): OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software - NOFLATTEN = (CFG['Torrent']['noFlatten']) - if isinstance(NOFLATTEN, str): - NOFLATTEN = NOFLATTEN.split(',') + + configure_flattening() + if isinstance(CATEGORIES, str): CATEGORIES = CATEGORIES.split(',') DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) From cf0fc1296ffd56535b3e9e4f6a37a9d26e248e95 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:22:59 -0500 Subject: [PATCH 122/406] Refactor torrent categories configuration --- core/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index a1e6512e..b95c0183 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -483,12 +483,19 @@ def configure_flattening(): NOFLATTEN = NOFLATTEN.split(',') +def configure_torrent_categories(): + global CATEGORIES + + CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software + if isinstance(CATEGORIES, str): + CATEGORIES = CATEGORIES.split(',') + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK global OUTPUT_DIRECTORY global TORRENT_DEFAULT_DIRECTORY - global CATEGORIES global DELETE_ORIGINAL global TORRENT_CHMOD_DIRECTORY global TORRENT_RESUME_ON_FAILURE @@ -498,12 +505,8 @@ def configure_torrents(): USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] - CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software - configure_flattening() - - if isinstance(CATEGORIES, str): - CATEGORIES = CATEGORIES.split(',') + configure_torrent_categories() DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) From f23eccc050d248aba18142a56e9791e54e629047 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:24:06 -0500 Subject: [PATCH 123/406] Refactor torrent resuming configuration --- core/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index b95c0183..e53999ec 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -491,6 +491,14 @@ def configure_torrent_categories(): CATEGORIES = CATEGORIES.split(',') +def configure_torrent_resuming(): + global TORRENT_RESUME + global TORRENT_RESUME_ON_FAILURE + + TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) + TORRENT_RESUME = int(CFG['Torrent']['resume']) + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK @@ -498,8 +506,6 @@ def configure_torrents(): global TORRENT_DEFAULT_DIRECTORY global DELETE_ORIGINAL global TORRENT_CHMOD_DIRECTORY - global TORRENT_RESUME_ON_FAILURE - global TORRENT_RESUME TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym @@ -509,9 +515,7 @@ def configure_torrents(): configure_torrent_categories() DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) - TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) - TORRENT_RESUME = int(CFG['Torrent']['resume']) - + configure_torrent_resuming() configure_utorrent() configure_transmission() configure_deluge() From 521d2b7a050593d1898aac31fbbcc9187666bb04 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:25:25 -0500 Subject: [PATCH 124/406] Refactor torrent permission configuration --- core/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index e53999ec..3104d779 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -499,13 +499,18 @@ def configure_torrent_resuming(): TORRENT_RESUME = int(CFG['Torrent']['resume']) +def configure_torrent_permissions(): + global TORRENT_CHMOD_DIRECTORY + + TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK global OUTPUT_DIRECTORY global TORRENT_DEFAULT_DIRECTORY global DELETE_ORIGINAL - global TORRENT_CHMOD_DIRECTORY TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym @@ -514,7 +519,7 @@ def configure_torrents(): configure_flattening() configure_torrent_categories() DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) - TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) + configure_torrent_permissions() configure_torrent_resuming() configure_utorrent() configure_transmission() From 9a1be36e8bae0353e820a01a20cb50a9b71a0745 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:26:32 -0500 Subject: [PATCH 125/406] Refactor torrent deletion configuration --- core/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 3104d779..4f7afff9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -505,20 +505,25 @@ def configure_torrent_permissions(): TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) +def configure_torrent_deltetion(): + global DELETE_ORIGINAL + + DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) + + def configure_torrents(): global TORRENT_CLIENT_AGENT global USE_LINK global OUTPUT_DIRECTORY global TORRENT_DEFAULT_DIRECTORY - global DELETE_ORIGINAL TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] configure_flattening() + configure_torrent_deltetion() configure_torrent_categories() - DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) configure_torrent_permissions() configure_torrent_resuming() configure_utorrent() From 81a6d9c4fa56f8055416e20a9faad48910605038 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 19 Jan 2019 14:27:32 -0500 Subject: [PATCH 126/406] Refactor torrent linking configuration --- core/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 4f7afff9..317809a2 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -511,16 +511,21 @@ def configure_torrent_deltetion(): DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) +def configure_torrent_linking(): + global USE_LINK + + USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym + + def configure_torrents(): global TORRENT_CLIENT_AGENT - global USE_LINK global OUTPUT_DIRECTORY global TORRENT_DEFAULT_DIRECTORY TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other - USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] + configure_torrent_linking() configure_flattening() configure_torrent_deltetion() configure_torrent_categories() From 00877c2d97ec242e2182457dd829bf90406d0b44 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 20 Jan 2019 10:09:03 -0500 Subject: [PATCH 127/406] Add Medusa API --- core/__init__.py | 2 ++ core/forks.py | 1 + 2 files changed, 3 insertions(+) diff --git a/core/__init__.py b/core/__init__.py index 317809a2..136e4193 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -88,6 +88,7 @@ FORK_SICKRAGE = 'SickRage' FORK_SICKCHILL = 'SickChill' FORK_SICKBEARD_API = 'SickBeard-api' FORK_MEDUSA = 'Medusa' +FORK_MEDUSA_API = 'Medusa-api' FORK_SICKGEAR = 'SickGear' FORK_STHENO = 'Stheno' @@ -99,6 +100,7 @@ FORKS = { FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, + FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} } diff --git a/core/forks.py b/core/forks.py index ede32f9c..be81a965 100644 --- a/core/forks.py +++ b/core/forks.py @@ -22,6 +22,7 @@ def auto_fork(section, input_category): web_root = cfg.get('web_root', '') replace = { 'medusa': 'Medusa', + 'medusa-api': 'Medusa-api', 'sickbeard-api': 'SickBeard-api', 'sickgear': 'SickGear', 'sickchill': 'SickChill', From de86259bb0e42cea2fb5d7de38d9b7092510cf09 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 27 Jan 2019 22:45:04 +1300 Subject: [PATCH 128/406] fix first return parsing from HeadPhones. Fixes #1536 --- core/auto_process/music.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 58e63714..4f10c64e 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -77,7 +77,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', } res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) - if res[0] in [0, 1]: + if res.status_code in [0, 1]: return res params = { From f6e620a3fdade741b96e64e01c1c74e32a8d6c0f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 3 Feb 2019 11:14:51 -0500 Subject: [PATCH 129/406] Add Python End-of-Life detection --- TorrentToMedia.py | 3 + core/__init__.py | 26 +++++++ eol.py | 179 ++++++++++++++++++++++++++++++++++++++++++++++ nzbToMedia.py | 3 + 4 files changed, 211 insertions(+) create mode 100644 eol.py diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 31fa152c..7704f9fc 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # coding=utf-8 +import eol +eol.check() + import cleanup cleanup.clean(cleanup.FOLDER_STRUCTURE) diff --git a/core/__init__.py b/core/__init__.py index 136e4193..fbd471f7 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -13,6 +13,7 @@ import time import libs.autoload import libs.util +import eol if not libs.autoload.completed: sys.exit('Could not load vendored libraries.') @@ -1085,6 +1086,28 @@ def configure_utility_locations(): logger.warning('Install ffmpeg with x264 support to enable this feature ...') +def check_python(): + """Check End-of-Life status for Python version.""" + # Raise if end of life + eol.check() + + # Warn if within grace period + grace_period = 365 # days + eol.warn_for_status(grace_period=-grace_period) + + # Log warning if within grace period + days_left = eol.lifetime() + logger.info( + 'Python v{major}.{minor} will reach end of life in {x} days.'.format( + major=sys.version_info[0], + minor=sys.version_info[1], + x=days_left, + ) + ) + if days_left <= grace_period: + logger.warning('Please upgrade to a more recent Python version.') + + def initialize(section=None): global __INITIALIZED__ @@ -1101,6 +1124,9 @@ def initialize(section=None): configure_migration() configure_logging_part_2() + # check python version + check_python() + # initialize the main SB database main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) diff --git a/eol.py b/eol.py new file mode 100644 index 00000000..a67bfc9e --- /dev/null +++ b/eol.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +import datetime +import sys +import warnings + +__version__ = '1.0.0' + + +def date(string, fmt='%Y-%m-%d'): + """ + Convert date string to date. + + :param string: A date string + :param fmt: Format to use when parsing the date string + :return: A datetime.date + """ + return datetime.datetime.strptime(string, fmt).date() + + +# https://devguide.python.org/ +# https://devguide.python.org/devcycle/#devcycle +PYTHON_EOL = { + (3, 7): date('2023-06-27'), + (3, 6): date('2021-12-23'), + (3, 5): date('2020-09-13'), + (3, 4): date('2019-03-16'), + (3, 3): date('2017-09-29'), + (3, 2): date('2016-02-20'), + (3, 1): date('2012-04-09'), + (3, 0): date('2009-01-13'), + (2, 7): date('2020-01-01'), + (2, 6): date('2013-10-29'), +} + + +class Error(Exception): + """An error has occurred.""" + + +class LifetimeError(Error): + """Lifetime has been exceeded and upgrade is required.""" + + +class LifetimeWarning(Warning): + """Lifetime has been exceeded and is no longer supported.""" + + +def lifetime(version=None): + """ + Calculate days left till End-of-Life for a version. + + :param version: An optional tuple with version information + If a version is not provided, the current system version will be used. + :return: Days left until End-of-Life + """ + if version is None: + version = sys.version_info + major = version[0] + minor = version[1] + now = datetime.datetime.now().date() + time_left = PYTHON_EOL[(major, minor)] - now + return time_left.days + + +def expiration(version=None, grace_period=0): + """ + Calculate expiration date for a version given a grace period. + + :param version: An optional tuple with version information + If a version is not provided, the current system version will be used. + :param grace_period: An optional number of days grace period + :return: Total days till expiration + """ + days_left = lifetime(version) + return days_left + grace_period + + +def check(version=None, grace_period=0): + """ + Raise an exception if end of life has been reached and recommend upgrade. + + :param version: An optional tuple with version information + If a version is not provided, the current system version will be used. + :param grace_period: An optional number of days grace period + If a grace period is not provided, a default 60 days grace period will + be used. + :return: None + """ + try: + raise_for_status(version, grace_period) + except LifetimeError as error: + print('Please use a newer version of Python.') + print_statuses() + sys.exit(error) + + +def raise_for_status(version=None, grace_period=0): + """ + Raise an exception if end of life has been reached. + + :param version: An optional tuple with version information + If a version is not provided, the current system version will be used. + :param grace_period: An optional number of days grace period + If a grace period is not provided, a default 60 days grace period will + be used. + :return: None + """ + if version is None: + version = sys.version_info + days_left = lifetime(version) + expires = days_left + grace_period + if expires <= 0: + msg = 'Python {major}.{minor} is no longer supported.'.format( + major=version[0], + minor=version[1], + ) + raise LifetimeError(msg) + + +def warn_for_status(version=None, grace_period=0): + """ + Warn if end of life has been reached. + + :param version: An optional tuple with version information + If a version is not provided, the current system version will be used. + :param grace_period: An optional number of days grace period + :return: None + """ + if version is None: + version = sys.version_info + days_left = lifetime(version) + expires = days_left + grace_period + if expires <= 0: + msg = 'Python {major}.{minor} is no longer supported.'.format( + major=version[0], + minor=version[1], + ) + warnings.warn(msg, LifetimeWarning) + + +def print_statuses(show_expired=False): + """ + Print end-of-life statuses of known python versions. + + :param show_expired: If true also print expired python version statuses + """ + lifetimes = sorted( + (lifetime(python_version), python_version) + for python_version in PYTHON_EOL + ) + print('Python End-of-Life for current versions:') + for days_left, python_version in lifetimes: + if days_left >= 0: + print( + 'v{major}.{minor} in {remaining:>4} days'.format( + major=python_version[0], + minor=python_version[1], + remaining=days_left, + ) + ) + if not show_expired: + return + + print() + print('Python End-of-Life for expired versions:') + for days_left, python_version in lifetimes: + if days_left < 0: + print( + 'v{major}.{minor} {remaining:>4} days ago'.format( + major=python_version[0], + minor=python_version[1], + remaining=-days_left, + ) + ) + + +if __name__ == '__main__': + print_statuses(show_expired=True) diff --git a/nzbToMedia.py b/nzbToMedia.py index f6c7e5db..a353e430 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -623,6 +623,9 @@ from __future__ import print_function +import eol +eol.check() + import cleanup cleanup.clean(cleanup.FOLDER_STRUCTURE) From f47f68f699c0ba427227d2b3f885272da8e16828 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 5 Feb 2019 22:01:20 +1300 Subject: [PATCH 130/406] convert byte to string from Popen. Fix Sick* failed processing. Fixes #1545 --- core/__init__.py | 16 ++++++++-------- core/auto_process/tv.py | 6 ++++-- core/transcoder.py | 4 ++-- tests/general.py | 15 +++++++++------ 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index fbd471f7..4d935def 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1017,17 +1017,17 @@ def configure_utility_locations(): else: try: - SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip() + SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not SEVENZIP: try: - SEVENZIP = subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE).communicate()[0].strip() + SEVENZIP = subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not SEVENZIP: try: - SEVENZIP = subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE).communicate()[0].strip() + SEVENZIP = subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not SEVENZIP: @@ -1035,7 +1035,7 @@ def configure_utility_locations(): logger.warning( 'Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!') try: - PAR2CMD = subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE).communicate()[0].strip() + PAR2CMD = subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not PAR2CMD: @@ -1050,12 +1050,12 @@ def configure_utility_locations(): FFMPEG = os.path.join(FFMPEG_PATH, 'avconv') else: try: - FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip() + FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not FFMPEG: try: - FFMPEG = subprocess.Popen(['which', 'avconv'], stdout=subprocess.PIPE).communicate()[0].strip() + FFMPEG = subprocess.Popen(['which', 'avconv'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not FFMPEG: @@ -1071,12 +1071,12 @@ def configure_utility_locations(): FFPROBE = os.path.join(FFMPEG_PATH, 'avprobe') else: try: - FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip() + FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not FFPROBE: try: - FFPROBE = subprocess.Popen(['which', 'avprobe'], stdout=subprocess.PIPE).communicate()[0].strip() + FFPROBE = subprocess.Popen(['which', 'avprobe'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: pass if not FFPROBE: diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index cc31f94e..8d082aec 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -168,13 +168,15 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu for param in copy.copy(fork_params): if param == 'failed': fork_params[param] = failed - del fork_params['proc_type'] + if 'proc_type' in fork_params: + del fork_params['proc_type'] if 'type' in fork_params: del fork_params['type'] if param == 'return_data': fork_params[param] = 0 - del fork_params['quiet'] + if 'quiet' in fork_params: + del fork_params['quiet'] if param == 'type': fork_params[param] = 'manual' diff --git a/core/transcoder.py b/core/transcoder.py index 52a35292..375d9d3a 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -95,7 +95,7 @@ def get_video_details(videofile, img=None, bitbucket=None): proc = subprocess.Popen(command, stdout=subprocess.PIPE) out, err = proc.communicate() result = proc.returncode - video_details = json.loads(out) + video_details = json.loads(out.decode()) except Exception: pass if not video_details: @@ -109,7 +109,7 @@ def get_video_details(videofile, img=None, bitbucket=None): proc = subprocess.Popen(command, stdout=subprocess.PIPE) out, err = proc.communicate() result = proc.returncode - video_details = json.loads(out) + video_details = json.loads(out.decode()) except Exception: logger.error('Checking [{0}] has failed'.format(file), 'TRANSCODER') return video_details, result diff --git a/tests/general.py b/tests/general.py index ed0ef06c..ddcc7aa0 100755 --- a/tests/general.py +++ b/tests/general.py @@ -1,15 +1,18 @@ #! /usr/bin/env python2 from __future__ import print_function -from babelfish import Language -import guessit -import requests -import subliminal +import datetime +import os +import sys import core -from core import transcoder +from core import logger, main_db, transcoder +from core.auto_process import comics, games, movies, music, tv +from core.auto_process.common import ProcessResult +from core.user_scripts import external_script from core.forks import auto_fork -from core.utils import server_responding +from core.utils import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status, server_responding + # Initialize the config core.initialize() From 8a637918d65cf46ea14354b88933271a1e5f9315 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 5 Feb 2019 22:15:05 +1300 Subject: [PATCH 131/406] use list for python3 compatibility. Fixes #1545 --- core/auto_process/tv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 8d082aec..bfc1a8d9 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -216,7 +216,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu fork_params[param] = 1 # delete any unused params so we don't pass them to SB by mistake - [fork_params.pop(k) for k, v in fork_params.items() if v is None] + [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None] if status == 0: if section == 'NzbDrone' and not apikey: From 45baf797537b8efda0884e941683e38b5ed91c4e Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 9 Feb 2019 11:08:33 +1300 Subject: [PATCH 132/406] log sucessful when returning failed download to Radarr. Fixes #1546 --- core/auto_process/movies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 7b1fb8e5..c0f1ace4 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -238,11 +238,11 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', report_nzb(failure_link, client_agent) if section == 'Radarr': - logger.postprocess('FAILED: The download failed. Sending failed download to {0} for CDH processing'.format(section), section) + logger.postprocess('SUCCESS: Sending failed download to {0} for CDH processing'.format(section), section) return ProcessResult( - message='{0}: Download Failed. Sending back to {0}'.format(section), + message='{0}: Sending failed download back to {0}'.format(section), status_code=1, # Return as failed to flag this in the downloader. - ) + ) # Return failed flag, but log the event as successful. if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) From fd8452b5c6e27ce2c8a6a5bb5378be2abf34392b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 16 Feb 2019 10:17:01 -0500 Subject: [PATCH 133/406] Add exception handling for failure to return to original directory Fixes #1552 --- cleanup.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cleanup.py b/cleanup.py index b78df372..dcef7789 100644 --- a/cleanup.py +++ b/cleanup.py @@ -33,7 +33,17 @@ class WorkingDirectory(object): return self def __exit__(self, exc_type, exc_val, exc_tb): - os.chdir(self.original_directory) + try: + os.chdir(self.original_directory) + except OSError as error: + print( + 'Unable to return to {original_directory}: {error}\n' + 'Continuing in {working_directory}'.format( + original_directory=self.original_directory, + error=error, + working_directory=self.working_directory, + ) + ) def module_path(module=__file__, parent=False): From 27cfc3457719163015fcb30db3464d6da721c425 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Mon, 25 Feb 2019 19:53:54 +1300 Subject: [PATCH 134/406] add sys path config to find executables not in path. Fixes #830 --- autoProcessMedia.cfg.spec | 2 ++ core/__init__.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index ef3c362b..767d937c 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -22,6 +22,8 @@ log_env = 0 # Enable/Disable logging git output to debug nzbtomedia.log (helpful to track down update failures.) log_git = 0 + # Set to the directory to search for executables if not in default system path + sys_path = # Set to the directory where your ffmpeg/ffprobe executables are located ffmpeg_path = # Enable/Disable media file checking using ffprobe. diff --git a/core/__init__.py b/core/__init__.py index 4d935def..8d638dc4 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -191,6 +191,7 @@ GETSUBS = False TRANSCODE = None CONCAT = None FFMPEG_PATH = None +SYS_PATH = None DUPLICATE = None IGNOREEXTENSIONS = [] VEXTENSION = None @@ -352,6 +353,7 @@ def configure_general(): global GIT_BRANCH global FORCE_CLEAN global FFMPEG_PATH + global SYS_PATH global CHECK_MEDIA global SAFE_MODE global NOEXTRACTFAILED @@ -364,6 +366,7 @@ def configure_general(): GIT_BRANCH = CFG['General']['git_branch'] or 'master' FORCE_CLEAN = int(CFG['General']['force_clean']) FFMPEG_PATH = CFG['General']['ffmpeg_path'] + SYS_PATH = CFG['General']['sys_path'] CHECK_MEDIA = int(CFG['General']['check_media']) SAFE_MODE = int(CFG['General']['safe_mode']) NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) @@ -1016,6 +1019,8 @@ def configure_utility_locations(): logger.warning('Install ffmpeg with x264 support to enable this feature ...') else: + if SYS_PATH: + sys.path.append(SYS_PATH) try: SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: From 3f3e1415c9be0f3b1c5eada55238747d92833829 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 2 Mar 2019 09:03:21 +1300 Subject: [PATCH 135/406] change method of writing to system PATH. Fixes #830 --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 8d638dc4..2f4645b2 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1020,7 +1020,7 @@ def configure_utility_locations(): else: if SYS_PATH: - sys.path.append(SYS_PATH) + os.environ['PATH'] += ':'+SYS_PATH try: SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: From f82fe0ee81fde101f1658a990bd6d32f98effb34 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 8 Mar 2019 23:03:24 +1300 Subject: [PATCH 136/406] decode 7zip outut. Fixes #1561 --- core/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/transcoder.py b/core/transcoder.py index 375d9d3a..4c650dcc 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -645,7 +645,7 @@ def rip_iso(item, new_dir, bitbucket): logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) - out, err = proc.communicate() + out, err = proc.communicate().decode() file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line).groups()[0] for line in out.splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line)] combined = [] From 64862ece10e63098632fe2d27801584e18814947 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 9 Mar 2019 20:30:25 +1300 Subject: [PATCH 137/406] fix python3 parsing of .iso files. Fixes #1561 --- core/transcoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/transcoder.py b/core/transcoder.py index 4c650dcc..42bf528e 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -645,9 +645,9 @@ def rip_iso(item, new_dir, bitbucket): logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) - out, err = proc.communicate().decode() - file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line).groups()[0] for line in - out.splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line)] + out, err = proc.communicate() + file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line.decode()).groups()[0] for line in + out.splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line.decode())] combined = [] for n in range(99): concat = [] From 392967780c861d1e6f13ac82f7e29918d95c743d Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 10 Mar 2019 08:34:56 +1300 Subject: [PATCH 138/406] don't load torrent clients for nzbs. Fixes #1563 --- core/utils/torrents.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/utils/torrents.py b/core/utils/torrents.py index 810bfc42..4c152e47 100644 --- a/core/utils/torrents.py +++ b/core/utils/torrents.py @@ -12,6 +12,8 @@ from core import logger def create_torrent_class(client_agent): # Hardlink solution for Torrents tc = None + if not core.APP_NAME == 'TorrentToMedia.py': #Skip loading Torrent for NZBs. + return tc if client_agent == 'utorrent': try: From d4786e10d7317f94037caf8a0d5d43daa0a2df02 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 10 Mar 2019 20:37:57 +1300 Subject: [PATCH 139/406] rev up to 12.0.8 --- .bumpversion.cfg | 2 +- README.md | 2 +- changelog.txt | 13 +++++++++++++ core/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a1d61c63..96d2148f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.7 +current_version = 12.0.8 commit = True tag = False diff --git a/README.md b/README.md index c78ffe83..63b8626b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.7 +nzbToMedia v12.0.8 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/changelog.txt b/changelog.txt index b197e7fa..b269e118 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,18 @@ Change_LOG / History +V12.0.8 + +Refactor and Rename Modules +Add Medusa API +Fix return parsing from HeadPhones +Add Python end of life detection and reporting +Fix Py3 return from Popen (Transcoder and executable path detection) +Add variable sys_path to config (allows user to specify separate path for binary detection) +Various Py3 compatability fixes +Log successful when returning to Radarr CDH +Add exception handling when failing to return to original directory (due to permissions) +Don't load Torrent Clients when calling NZB processing + V12.0.7 Refactor utils diff --git a/core/__init__.py b/core/__init__.py index 2f4645b2..15d78bfd 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -71,7 +71,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.7' +__version__ = '12.0.8' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 941b4477..9c366acf 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.7', + version='12.0.8', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 1c63c9fe398446167425daefcd7b03186e218bcf Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 10 Mar 2019 22:01:22 +1300 Subject: [PATCH 140/406] Create CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..35d72887 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at fock_wulf@hotmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 2fc5101ef099773133c4ac0d4c02e42640b86082 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 10 Mar 2019 22:22:28 +1300 Subject: [PATCH 141/406] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..688719d1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Technical Specs** +1. Running on (Windows, Linux, NAS Model etc) '....' +2. Python version '....' +3. Download Client (NZBget, SABnbzd, Transmission) '....' +4. Intended media management (SickChill, CouchPotoato, Radarr, Sonarr) '....' + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Log** +Please provide an extract, or full debug log that indicates the issue. From ef950d8024b66efb5935a6e0619b9e15b2e1036b Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 10 Mar 2019 22:32:55 +1300 Subject: [PATCH 142/406] add Contributing guide --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..9ec23db2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing + +When contributing to this repository, please first check the issues list, current pull requests, and FAQ pages. + +While it is prefered that all interactions be made through github, the author can be contacted directly at fock_wulf@hotmail.com + +Please note we have a code of conduct, please follow it in all your interactions with the project. + +## Pull Request Process + +1. Please base all pull requests on the current nightly branch. +2. Include a description to explain what is achieved with a pull request. +3. Link any relevant issues that are closed or impacted by the pull request. +4. Please update the FAQ to reflect any new parameters, changed behaviour, or suggested configurations relevant to the changes. \ No newline at end of file From cd64014a9d2af9ddbdcdd028283ccf646aba7225 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 10:02:58 -0400 Subject: [PATCH 143/406] Refactor ISO file matching to decode process output a single time. --- core/transcoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/transcoder.py b/core/transcoder.py index 42bf528e..9abc2fae 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -646,8 +646,8 @@ def rip_iso(item, new_dir, bitbucket): print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() - file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line.decode()).groups()[0] for line in - out.splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line.decode())] + file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line).groups()[0] for line in + out.decode().splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line)] combined = [] for n in range(99): concat = [] From 28eed3bc928f54c8d8a99094e2b560575b1347f5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 10:07:22 -0400 Subject: [PATCH 144/406] Refactor ISO file matching to use regex only once per file. --- core/transcoder.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/transcoder.py b/core/transcoder.py index 9abc2fae..c53a1211 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -646,8 +646,15 @@ def rip_iso(item, new_dir, bitbucket): print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() - file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line).groups()[0] for line in - out.decode().splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line)] + file_match_gen = ( + re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line) + for line in out.decode().splitlines() + ) + file_list = [ + file_match.groups()[0] + for file_match in file_match_gen + if file_match + ] combined = [] for n in range(99): concat = [] From e1aa32aee797af5800e39eb9297f1674b2861dad Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 3 Feb 2019 12:24:34 -0500 Subject: [PATCH 145/406] Refactor downloader configuration to plugins.downloaders --- core/__init__.py | 153 ++---------------- core/plugins/__init__.py | 0 core/plugins/downloaders/__init__.py | 0 core/plugins/downloaders/configuration.py | 5 + core/plugins/downloaders/nzb/__init__.py | 0 core/plugins/downloaders/nzb/configuration.py | 15 ++ core/plugins/downloaders/torrent/__init__.py | 0 .../downloaders/torrent/configuration.py | 81 ++++++++++ core/plugins/downloaders/torrent/deluge.py | 21 +++ .../downloaders/torrent/qbittorrent.py | 23 +++ .../downloaders/torrent/transmission.py | 20 +++ .../downloaders/torrent/utils.py} | 61 ++----- core/plugins/downloaders/torrent/utorrent.py | 19 +++ core/plugins/downloaders/utils.py | 5 + core/utils/__init__.py | 1 - 15 files changed, 218 insertions(+), 186 deletions(-) create mode 100644 core/plugins/__init__.py create mode 100644 core/plugins/downloaders/__init__.py create mode 100644 core/plugins/downloaders/configuration.py create mode 100644 core/plugins/downloaders/nzb/__init__.py create mode 100644 core/plugins/downloaders/nzb/configuration.py create mode 100644 core/plugins/downloaders/torrent/__init__.py create mode 100644 core/plugins/downloaders/torrent/configuration.py create mode 100644 core/plugins/downloaders/torrent/deluge.py create mode 100644 core/plugins/downloaders/torrent/qbittorrent.py create mode 100644 core/plugins/downloaders/torrent/transmission.py rename core/{utils/torrents.py => plugins/downloaders/torrent/utils.py} (57%) create mode 100644 core/plugins/downloaders/torrent/utorrent.py create mode 100644 core/plugins/downloaders/utils.py diff --git a/core/__init__.py b/core/__init__.py index 15d78bfd..c59daa92 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -46,12 +46,21 @@ from six.moves import reload_module from core import logger, main_db, version_check, databases, transcoder from core.configuration import config +from core.plugins.downloaders.configuration import ( + configure_nzbs, + configure_torrents, + configure_torrent_class, +) +from core.plugins.downloaders.utils import ( + pause_torrent, + remove_torrent, + resume_torrent, +) from core.utils import ( RunningProcess, category_search, clean_dir, copy_link, - create_torrent_class, extract_files, flatten, get_dirs, @@ -59,13 +68,10 @@ from core.utils import ( list_media_files, make_dir, parse_args, - pause_torrent, rchmod, remove_dir, remove_read_only, - remove_torrent, restart, - resume_torrent, sanitize_name, update_download_info_status, wake_up, @@ -403,26 +409,6 @@ def configure_wake_on_lan(): wake_up() -def configure_sabnzbd(): - global SABNZBD_HOST - global SABNZBD_PORT - global SABNZBD_APIKEY - - SABNZBD_HOST = CFG['Nzb']['sabnzbd_host'] - SABNZBD_PORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accommodate NzbGet - SABNZBD_APIKEY = CFG['Nzb']['sabnzbd_apikey'] - - -def configure_nzbs(): - global NZB_CLIENT_AGENT - global NZB_DEFAULT_DIRECTORY - - NZB_CLIENT_AGENT = CFG['Nzb']['clientAgent'] # sabnzbd - NZB_DEFAULT_DIRECTORY = CFG['Nzb']['default_downloadDirectory'] - - configure_sabnzbd() - - def configure_groups(): global GROUPS @@ -435,114 +421,6 @@ def configure_groups(): GROUPS = None -def configure_utorrent(): - global UTORRENT_WEB_UI - global UTORRENT_USER - global UTORRENT_PASSWORD - - UTORRENT_WEB_UI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ - UTORRENT_USER = CFG['Torrent']['uTorrentUSR'] # mysecretusr - UTORRENT_PASSWORD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr - - -def configure_transmission(): - global TRANSMISSION_HOST - global TRANSMISSION_PORT - global TRANSMISSION_USER - global TRANSMISSION_PASSWORD - - TRANSMISSION_HOST = CFG['Torrent']['TransmissionHost'] # localhost - TRANSMISSION_PORT = int(CFG['Torrent']['TransmissionPort']) - TRANSMISSION_USER = CFG['Torrent']['TransmissionUSR'] # mysecretusr - TRANSMISSION_PASSWORD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr - - -def configure_deluge(): - global DELUGE_HOST - global DELUGE_PORT - global DELUGE_USER - global DELUGE_PASSWORD - - DELUGE_HOST = CFG['Torrent']['DelugeHost'] # localhost - DELUGE_PORT = int(CFG['Torrent']['DelugePort']) # 8084 - DELUGE_USER = CFG['Torrent']['DelugeUSR'] # mysecretusr - DELUGE_PASSWORD = CFG['Torrent']['DelugePWD'] # mysecretpwr - - -def configure_qbittorrent(): - global QBITTORRENT_HOST - global QBITTORRENT_PORT - global QBITTORRENT_USER - global QBITTORRENT_PASSWORD - - QBITTORRENT_HOST = CFG['Torrent']['qBittorrenHost'] # localhost - QBITTORRENT_PORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 - QBITTORRENT_USER = CFG['Torrent']['qBittorrentUSR'] # mysecretusr - QBITTORRENT_PASSWORD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr - - -def configure_flattening(): - global NOFLATTEN - - NOFLATTEN = (CFG['Torrent']['noFlatten']) - if isinstance(NOFLATTEN, str): - NOFLATTEN = NOFLATTEN.split(',') - - -def configure_torrent_categories(): - global CATEGORIES - - CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software - if isinstance(CATEGORIES, str): - CATEGORIES = CATEGORIES.split(',') - - -def configure_torrent_resuming(): - global TORRENT_RESUME - global TORRENT_RESUME_ON_FAILURE - - TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) - TORRENT_RESUME = int(CFG['Torrent']['resume']) - - -def configure_torrent_permissions(): - global TORRENT_CHMOD_DIRECTORY - - TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) - - -def configure_torrent_deltetion(): - global DELETE_ORIGINAL - - DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) - - -def configure_torrent_linking(): - global USE_LINK - - USE_LINK = CFG['Torrent']['useLink'] # no | hard | sym - - -def configure_torrents(): - global TORRENT_CLIENT_AGENT - global OUTPUT_DIRECTORY - global TORRENT_DEFAULT_DIRECTORY - - TORRENT_CLIENT_AGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other - OUTPUT_DIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ - TORRENT_DEFAULT_DIRECTORY = CFG['Torrent']['default_downloadDirectory'] - configure_torrent_linking() - configure_flattening() - configure_torrent_deltetion() - configure_torrent_categories() - configure_torrent_permissions() - configure_torrent_resuming() - configure_utorrent() - configure_transmission() - configure_deluge() - configure_qbittorrent() - - def configure_remote_paths(): global REMOTE_PATHS @@ -973,13 +851,6 @@ def configure_passwords_file(): PASSWORDS_FILE = CFG['passwords']['PassWordFile'] -def configure_torrent_class(): - global TORRENT_CLASS - - # create torrent class - TORRENT_CLASS = create_torrent_class(TORRENT_CLIENT_AGENT) - - def configure_sections(section): global SECTIONS global CATEGORIES @@ -1138,8 +1009,8 @@ def initialize(section=None): configure_general() configure_updates() configure_wake_on_lan() - configure_nzbs() - configure_torrents() + configure_nzbs(CFG) + configure_torrents(CFG) configure_remote_paths() configure_plex() configure_niceness() diff --git a/core/plugins/__init__.py b/core/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/plugins/downloaders/__init__.py b/core/plugins/downloaders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/plugins/downloaders/configuration.py b/core/plugins/downloaders/configuration.py new file mode 100644 index 00000000..7ea4f881 --- /dev/null +++ b/core/plugins/downloaders/configuration.py @@ -0,0 +1,5 @@ +from core.plugins.downloaders.nzb.configuration import configure_nzbs +from core.plugins.downloaders.torrent.configuration import ( + configure_torrents, + configure_torrent_class, +) diff --git a/core/plugins/downloaders/nzb/__init__.py b/core/plugins/downloaders/nzb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/plugins/downloaders/nzb/configuration.py b/core/plugins/downloaders/nzb/configuration.py new file mode 100644 index 00000000..d0883d01 --- /dev/null +++ b/core/plugins/downloaders/nzb/configuration.py @@ -0,0 +1,15 @@ +import core + + +def configure_nzbs(config): + nzb_config = config['Nzb'] + core.NZB_CLIENT_AGENT = nzb_config['clientAgent'] # sabnzbd + core.NZB_DEFAULT_DIRECTORY = nzb_config['default_downloadDirectory'] + + configure_sabnzbd(nzb_config) + + +def configure_sabnzbd(config): + core.SABNZBD_HOST = config['sabnzbd_host'] + core.SABNZBD_PORT = int(config['sabnzbd_port'] or 8080) # defaults to accommodate NzbGet + core.SABNZBD_APIKEY = config['sabnzbd_apikey'] diff --git a/core/plugins/downloaders/torrent/__init__.py b/core/plugins/downloaders/torrent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/plugins/downloaders/torrent/configuration.py b/core/plugins/downloaders/torrent/configuration.py new file mode 100644 index 00000000..92001636 --- /dev/null +++ b/core/plugins/downloaders/torrent/configuration.py @@ -0,0 +1,81 @@ +import core +from core.plugins.downloaders.torrent.utils import create_torrent_class + + +def configure_torrents(config): + torrent_config = config['Torrent'] + core.TORRENT_CLIENT_AGENT = torrent_config['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other + core.OUTPUT_DIRECTORY = torrent_config['outputDirectory'] # /abs/path/to/complete/ + core.TORRENT_DEFAULT_DIRECTORY = torrent_config['default_downloadDirectory'] + + configure_torrent_linking(torrent_config) + configure_flattening(torrent_config) + configure_torrent_deletion(torrent_config) + configure_torrent_categories(torrent_config) + configure_torrent_permissions(torrent_config) + configure_torrent_resuming(torrent_config) + configure_utorrent(torrent_config) + configure_transmission(torrent_config) + configure_deluge(torrent_config) + configure_qbittorrent(torrent_config) + + +def configure_torrent_linking(config): + core.USE_LINK = config['useLink'] # no | hard | sym + + +def configure_flattening(config): + core.NOFLATTEN = (config['noFlatten']) + if isinstance(core.NOFLATTEN, str): + core.NOFLATTEN = core.NOFLATTEN.split(',') + + +def configure_torrent_categories(config): + core.CATEGORIES = (config['categories']) # music,music_videos,pictures,software + if isinstance(core.CATEGORIES, str): + core.CATEGORIES = core.CATEGORIES.split(',') + + +def configure_torrent_resuming(config): + core.TORRENT_RESUME_ON_FAILURE = int(config['resumeOnFailure']) + core.TORRENT_RESUME = int(config['resume']) + + +def configure_torrent_permissions(config): + core.TORRENT_CHMOD_DIRECTORY = int(str(config['chmodDirectory']), 8) + + +def configure_torrent_deletion(config): + core.DELETE_ORIGINAL = int(config['deleteOriginal']) + + +def configure_utorrent(config): + core.UTORRENT_WEB_UI = config['uTorrentWEBui'] # http://localhost:8090/gui/ + core.UTORRENT_USER = config['uTorrentUSR'] # mysecretusr + core.UTORRENT_PASSWORD = config['uTorrentPWD'] # mysecretpwr + + +def configure_transmission(config): + core.TRANSMISSION_HOST = config['TransmissionHost'] # localhost + core.TRANSMISSION_PORT = int(config['TransmissionPort']) + core.TRANSMISSION_USER = config['TransmissionUSR'] # mysecretusr + core.TRANSMISSION_PASSWORD = config['TransmissionPWD'] # mysecretpwr + + +def configure_deluge(config): + core.DELUGE_HOST = config['DelugeHost'] # localhost + core.DELUGE_PORT = int(config['DelugePort']) # 8084 + core.DELUGE_USER = config['DelugeUSR'] # mysecretusr + core.DELUGE_PASSWORD = config['DelugePWD'] # mysecretpwr + + +def configure_qbittorrent(config): + core.QBITTORRENT_HOST = config['qBittorrenHost'] # localhost + core.QBITTORRENT_PORT = int(config['qBittorrentPort']) # 8080 + core.QBITTORRENT_USER = config['qBittorrentUSR'] # mysecretusr + core.QBITTORRENT_PASSWORD = config['qBittorrentPWD'] # mysecretpwr + + +def configure_torrent_class(): + # create torrent class + core.TORRENT_CLASS = create_torrent_class(core.TORRENT_CLIENT_AGENT) diff --git a/core/plugins/downloaders/torrent/deluge.py b/core/plugins/downloaders/torrent/deluge.py new file mode 100644 index 00000000..69ae2000 --- /dev/null +++ b/core/plugins/downloaders/torrent/deluge.py @@ -0,0 +1,21 @@ +from synchronousdeluge.client import DelugeClient + +import core +from core import logger + + +def configure_client(): + agent = 'deluge' + host = core.DELUGE_HOST + port = core.DELUGE_PORT + user = core.DELUGE_USER + password = core.DELUGE_PASSWORD + + logger.debug('Connecting to {0}: http://{1}:{2}'.format(agent, host, port)) + client = DelugeClient() + try: + client.connect(host, port, user, password) + except Exception: + logger.error('Failed to connect to Deluge') + else: + return client diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py new file mode 100644 index 00000000..37671fea --- /dev/null +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -0,0 +1,23 @@ +from qbittorrent import Client as qBittorrentClient + +import core +from core import logger + + +def configure_client(): + agent = 'qbittorrent' + host = core.QBITTORRENT_HOST + port = core.QBITTORRENT_PORT + user = core.QBITTORRENT_USER + password = core.QBITTORRENT_PASSWORD + + logger.debug( + 'Connecting to {0}: http://{1}:{2}'.format(agent, host, port) + ) + client = qBittorrentClient('http://{0}:{1}/'.format(host, port)) + try: + client.login(user, password) + except Exception: + logger.error('Failed to connect to qBittorrent') + else: + return client diff --git a/core/plugins/downloaders/torrent/transmission.py b/core/plugins/downloaders/torrent/transmission.py new file mode 100644 index 00000000..d122aaf0 --- /dev/null +++ b/core/plugins/downloaders/torrent/transmission.py @@ -0,0 +1,20 @@ +from transmissionrpc.client import Client as TransmissionClient + +import core +from core import logger + + +def configure_client(): + agent = 'transmission' + host = core.TRANSMISSION_HOST + port = core.TRANSMISSION_PORT + user = core.TRANSMISSION_USER + password = core.TRANSMISSION_PASSWORD + + logger.debug('Connecting to {0}: http://{1}:{2}'.format(agent, host, port)) + try: + client = TransmissionClient(host, port, user, password) + except Exception: + logger.error('Failed to connect to Transmission') + else: + return client diff --git a/core/utils/torrents.py b/core/plugins/downloaders/torrent/utils.py similarity index 57% rename from core/utils/torrents.py rename to core/plugins/downloaders/torrent/utils.py index 4c152e47..be2db4e8 100644 --- a/core/utils/torrents.py +++ b/core/plugins/downloaders/torrent/utils.py @@ -1,55 +1,28 @@ import time -from qbittorrent import Client as qBittorrentClient -from synchronousdeluge.client import DelugeClient -from transmissionrpc.client import Client as TransmissionClient -from utorrent.client import UTorrentClient - import core from core import logger +from .deluge import configure_client as deluge_client +from .qbittorrent import configure_client as qbittorrent_client +from .transmission import configure_client as transmission_client +from .utorrent import configure_client as utorrent_client + +torrent_clients = { + 'deluge': deluge_client, + 'qbittorrent': qbittorrent_client, + 'transmission': transmission_client, + 'utorrent': utorrent_client, +} + def create_torrent_class(client_agent): - # Hardlink solution for Torrents - tc = None - if not core.APP_NAME == 'TorrentToMedia.py': #Skip loading Torrent for NZBs. - return tc + if not core.APP_NAME == 'TorrentToMedia.py': + return # Skip loading Torrent for NZBs. - if client_agent == 'utorrent': - try: - logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENT_WEB_UI)) - tc = UTorrentClient(core.UTORRENT_WEB_UI, core.UTORRENT_USER, core.UTORRENT_PASSWORD) - except Exception: - logger.error('Failed to connect to uTorrent') - - if client_agent == 'transmission': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format( - client_agent, core.TRANSMISSION_HOST, core.TRANSMISSION_PORT)) - tc = TransmissionClient(core.TRANSMISSION_HOST, core.TRANSMISSION_PORT, - core.TRANSMISSION_USER, - core.TRANSMISSION_PASSWORD) - except Exception: - logger.error('Failed to connect to Transmission') - - if client_agent == 'deluge': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGE_HOST, core.DELUGE_PORT)) - tc = DelugeClient() - tc.connect(host=core.DELUGE_HOST, port=core.DELUGE_PORT, username=core.DELUGE_USER, - password=core.DELUGE_PASSWORD) - except Exception: - logger.error('Failed to connect to Deluge') - - if client_agent == 'qbittorrent': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENT_HOST, core.QBITTORRENT_PORT)) - tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENT_HOST, core.QBITTORRENT_PORT)) - tc.login(core.QBITTORRENT_USER, core.QBITTORRENT_PASSWORD) - except Exception: - logger.error('Failed to connect to qBittorrent') - - return tc + client = torrent_clients.get(client_agent) + if client: + return client() def pause_torrent(client_agent, input_hash, input_id, input_name): diff --git a/core/plugins/downloaders/torrent/utorrent.py b/core/plugins/downloaders/torrent/utorrent.py new file mode 100644 index 00000000..95422bfe --- /dev/null +++ b/core/plugins/downloaders/torrent/utorrent.py @@ -0,0 +1,19 @@ +from utorrent.client import UTorrentClient + +import core +from core import logger + + +def configure_client(): + agent = 'utorrent' + web_ui = core.UTORRENT_WEB_UI + user = core.UTORRENT_USER + password = core.UTORRENT_PASSWORD + + logger.debug('Connecting to {0}: {1}'.format(agent, web_ui)) + try: + client = UTorrentClient(web_ui, user, password) + except Exception: + logger.error('Failed to connect to uTorrent') + else: + return client diff --git a/core/plugins/downloaders/utils.py b/core/plugins/downloaders/utils.py new file mode 100644 index 00000000..a84d270b --- /dev/null +++ b/core/plugins/downloaders/utils.py @@ -0,0 +1,5 @@ +from core.plugins.downloaders.torrent.utils import ( + pause_torrent, + remove_torrent, + resume_torrent, +) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 0083fbbd..b662e9ae 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -45,7 +45,6 @@ from core.utils.paths import ( ) from core.utils.processes import RunningProcess, restart from core.utils.subtitles import import_subs -from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() From 1d75439441dece2be2f315240e1797cfd2409a4f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 11:09:26 -0400 Subject: [PATCH 146/406] Refactor utils.nzb to plugins.downloaders.nzb.utils --- core/auto_process/movies.py | 18 ++++++++++++++++-- core/auto_process/tv.py | 17 +++++++++++++++-- .../downloaders/nzb/utils.py} | 0 core/utils/__init__.py | 1 - 4 files changed, 31 insertions(+), 5 deletions(-) rename core/{utils/nzbs.py => plugins/downloaders/nzb/utils.py} (100%) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index c0f1ace4..98ce62ad 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -8,9 +8,23 @@ import requests import core from core import logger, transcoder -from core.auto_process.common import command_complete, completed_download_handling, ProcessResult +from core.auto_process.common import ( + ProcessResult, + command_complete, + completed_download_handling, +) +from core.plugins.downloaders.nzb.utils import report_nzb from core.scene_exceptions import process_all_exceptions -from core.utils import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding +from core.utils import ( + convert_to_ascii, + find_download, + find_imdbid, + import_subs, + list_media_files, + remote_dir, + remove_dir, + server_responding, +) requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index bfc1a8d9..3b9b46d3 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -10,10 +10,23 @@ import requests import core from core import logger, transcoder -from core.auto_process.common import command_complete, completed_download_handling, ProcessResult +from core.auto_process.common import ( + ProcessResult, + command_complete, + completed_download_handling, +) from core.forks import auto_fork +from core.plugins.downloaders.nzb.utils import report_nzb from core.scene_exceptions import process_all_exceptions -from core.utils import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding +from core.utils import ( + convert_to_ascii, + flatten, + import_subs, + list_media_files, + remote_dir, + remove_dir, + server_responding, +) requests.packages.urllib3.disable_warnings() diff --git a/core/utils/nzbs.py b/core/plugins/downloaders/nzb/utils.py similarity index 100% rename from core/utils/nzbs.py rename to core/plugins/downloaders/nzb/utils.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index b662e9ae..6817860a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -20,7 +20,6 @@ from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, server_responding, test_connection, wake_on_lan, wake_up from core.utils.notifications import plex_update -from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( parse_args, parse_deluge, From e12f2724e62b6abd74769f8d1390446350889754 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 10:58:17 -0400 Subject: [PATCH 147/406] Refactor plex configuration to plugins.plex --- core/__init__.py | 27 +++------------------------ core/plugins/plex.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 core/plugins/plex.py diff --git a/core/__init__.py b/core/__init__.py index c59daa92..9468f5a1 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -11,9 +11,9 @@ import subprocess import sys import time +import eol import libs.autoload import libs.util -import eol if not libs.autoload.completed: sys.exit('Could not load vendored libraries.') @@ -56,6 +56,7 @@ from core.plugins.downloaders.utils import ( remove_torrent, resume_torrent, ) +from core.plugins.plex import configure_plex from core.utils import ( RunningProcess, category_search, @@ -443,28 +444,6 @@ def configure_remote_paths(): ] -def configure_plex(): - global PLEX_SSL - global PLEX_HOST - global PLEX_PORT - global PLEX_TOKEN - global PLEX_SECTION - - PLEX_SSL = int(CFG['Plex']['plex_ssl']) - PLEX_HOST = CFG['Plex']['plex_host'] - PLEX_PORT = CFG['Plex']['plex_port'] - PLEX_TOKEN = CFG['Plex']['plex_token'] - PLEX_SECTION = CFG['Plex']['plex_sections'] or [] - - if PLEX_SECTION: - if isinstance(PLEX_SECTION, list): - PLEX_SECTION = ','.join(PLEX_SECTION) # fix in case this imported as list. - PLEX_SECTION = [ - tuple(item.split(',')) - for item in PLEX_SECTION.split('|') - ] - - def configure_niceness(): global NICENESS @@ -1012,7 +991,7 @@ def initialize(section=None): configure_nzbs(CFG) configure_torrents(CFG) configure_remote_paths() - configure_plex() + configure_plex(CFG) configure_niceness() configure_containers() configure_transcoder() diff --git a/core/plugins/plex.py b/core/plugins/plex.py new file mode 100644 index 00000000..180cca66 --- /dev/null +++ b/core/plugins/plex.py @@ -0,0 +1,19 @@ +import core + + +def configure_plex(config): + core.PLEX_SSL = int(config['Plex']['plex_ssl']) + core.PLEX_HOST = config['Plex']['plex_host'] + core.PLEX_PORT = config['Plex']['plex_port'] + core.PLEX_TOKEN = config['Plex']['plex_token'] + plex_section = config['Plex']['plex_sections'] or [] + + if plex_section: + if isinstance(plex_section, list): + plex_section = ','.join(plex_section) # fix in case this imported as list. + plex_section = [ + tuple(item.split(',')) + for item in plex_section.split('|') + ] + + core.PLEX_SECTION = plex_section From 76b5c06a33964d9cd7909413bcdb16398f60a4a8 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 11:15:31 -0400 Subject: [PATCH 148/406] Refactor utils.notifications.plex_update to plugins.plex.plex_update --- core/plugins/plex.py | 27 +++++++++++++++++++++++++++ core/utils/__init__.py | 1 - core/utils/notifications.py | 30 ------------------------------ 3 files changed, 27 insertions(+), 31 deletions(-) delete mode 100644 core/utils/notifications.py diff --git a/core/plugins/plex.py b/core/plugins/plex.py index 180cca66..6d851e85 100644 --- a/core/plugins/plex.py +++ b/core/plugins/plex.py @@ -1,4 +1,7 @@ +import requests + import core +from core import logger def configure_plex(config): @@ -17,3 +20,27 @@ def configure_plex(config): ] core.PLEX_SECTION = plex_section + + +def plex_update(category): + if core.FAILED: + return + url = '{scheme}://{host}:{port}/library/sections/'.format( + scheme='https' if core.PLEX_SSL else 'http', + host=core.PLEX_HOST, + port=core.PLEX_PORT, + ) + section = None + if not core.PLEX_SECTION: + return + logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') + for item in core.PLEX_SECTION: + if item[0] == category: + section = item[1] + + if section: + url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEX_TOKEN) + requests.get(url, timeout=(60, 120), verify=False) + logger.debug('Plex Library has been refreshed.', 'PLEX') + else: + logger.debug('Could not identify section for plex update', 'PLEX') diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 6817860a..88b9692f 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -19,7 +19,6 @@ from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, server_responding, test_connection, wake_on_lan, wake_up -from core.utils.notifications import plex_update from core.utils.parsers import ( parse_args, parse_deluge, diff --git a/core/utils/notifications.py b/core/utils/notifications.py deleted file mode 100644 index ddad0c1b..00000000 --- a/core/utils/notifications.py +++ /dev/null @@ -1,30 +0,0 @@ -import requests - -import core -from core import logger - - -def plex_update(category): - if core.FAILED: - return - url = '{scheme}://{host}:{port}/library/sections/'.format( - scheme='https' if core.PLEX_SSL else 'http', - host=core.PLEX_HOST, - port=core.PLEX_PORT, - ) - section = None - if not core.PLEX_SECTION: - return - logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') - for item in core.PLEX_SECTION: - if item[0] == category: - section = item[1] - - if section: - url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEX_TOKEN) - requests.get(url, timeout=(60, 120), verify=False) - logger.debug('Plex Library has been refreshed.', 'PLEX') - else: - logger.debug('Could not identify section for plex update', 'PLEX') - - From b6db785c9221638c152a10a5c0555ba24f19b8e3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 11:28:54 -0400 Subject: [PATCH 149/406] Refactor utils.subtitles to plugins.subtitles --- core/auto_process/movies.py | 2 +- core/auto_process/tv.py | 2 +- core/{utils => plugins}/subtitles.py | 0 core/user_scripts.py | 3 ++- core/utils/__init__.py | 1 - 5 files changed, 4 insertions(+), 4 deletions(-) rename core/{utils => plugins}/subtitles.py (100%) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 98ce62ad..b7fe3a66 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -14,12 +14,12 @@ from core.auto_process.common import ( completed_download_handling, ) from core.plugins.downloaders.nzb.utils import report_nzb +from core.plugins.subtitles import import_subs from core.scene_exceptions import process_all_exceptions from core.utils import ( convert_to_ascii, find_download, find_imdbid, - import_subs, list_media_files, remote_dir, remove_dir, diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 3b9b46d3..c4a68f59 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -17,11 +17,11 @@ from core.auto_process.common import ( ) from core.forks import auto_fork from core.plugins.downloaders.nzb.utils import report_nzb +from core.plugins.subtitles import import_subs from core.scene_exceptions import process_all_exceptions from core.utils import ( convert_to_ascii, flatten, - import_subs, list_media_files, remote_dir, remove_dir, diff --git a/core/utils/subtitles.py b/core/plugins/subtitles.py similarity index 100% rename from core/utils/subtitles.py rename to core/plugins/subtitles.py diff --git a/core/user_scripts.py b/core/user_scripts.py index 83f17d60..5d447670 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -5,7 +5,8 @@ from subprocess import Popen import core from core import logger, transcoder -from core.utils import import_subs, list_media_files, remove_dir +from core.plugins.subtitles import import_subs +from core.utils import list_media_files, remove_dir def external_script(output_destination, torrent_name, torrent_label, settings): diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 88b9692f..470fe4fd 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -42,7 +42,6 @@ from core.utils.paths import ( remove_read_only, ) from core.utils.processes import RunningProcess, restart -from core.utils.subtitles import import_subs requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() From 9f7f28d54eb4157ea92eb8e8476e23d9029e5c54 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 20:35:05 -0400 Subject: [PATCH 150/406] Fix missed commits during refactor --- TorrentToMedia.py | 3 ++- nzbToMedia.py | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 7704f9fc..74f37165 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -15,8 +15,9 @@ import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv from core.auto_process.common import ProcessResult +from core.plugins.plex import plex_update from core.user_scripts import external_script -from core.utils import char_replace, convert_to_ascii, plex_update, replace_links +from core.utils import char_replace, convert_to_ascii, replace_links from six import text_type diff --git a/nzbToMedia.py b/nzbToMedia.py index a353e430..d91f413f 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -637,8 +637,14 @@ import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv from core.auto_process.common import ProcessResult +from core.plugins.downloaders.nzb.utils import get_nzoid +from core.plugins.plex import plex_update from core.user_scripts import external_script -from core.utils import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status +from core.utils import ( + char_replace, clean_dir, convert_to_ascii, + extract_files, get_dirs, get_download_info, + update_download_info_status, +) try: text_type = unicode From a669c983b7f912c69e1c28c8d27bed64be344eb2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 10 Mar 2019 20:45:01 -0400 Subject: [PATCH 151/406] Fix absolute imports for qbittorrent and utorrent in Python 2.7 --- core/plugins/downloaders/torrent/qbittorrent.py | 2 ++ core/plugins/downloaders/torrent/utorrent.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py index 37671fea..ff92512c 100644 --- a/core/plugins/downloaders/torrent/qbittorrent.py +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from qbittorrent import Client as qBittorrentClient import core diff --git a/core/plugins/downloaders/torrent/utorrent.py b/core/plugins/downloaders/torrent/utorrent.py index 95422bfe..634e8ca5 100644 --- a/core/plugins/downloaders/torrent/utorrent.py +++ b/core/plugins/downloaders/torrent/utorrent.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from utorrent.client import UTorrentClient import core From f5891459c28c912fa3660966583edfefbe2b3da5 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 11 Mar 2019 22:40:59 +1300 Subject: [PATCH 152/406] Set up CI with Azure Pipelines (#1573) * Set up CI with Azure Pipelines * test all python versions * rename test file and set to run from subdir. --- azure-pipelines.yml | 58 ++++++++++++++++++++++++++++++++ tests/general.py | 72 ---------------------------------------- tests/test_initialize.py | 33 ++++++++++++++++++ 3 files changed, 91 insertions(+), 72 deletions(-) create mode 100644 azure-pipelines.yml delete mode 100755 tests/general.py create mode 100755 tests/test_initialize.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..1a68748d --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,58 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +jobs: + +- job: 'Test' + pool: + vmImage: 'Ubuntu-16.04' + strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + maxParallel: 4 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + + - script: python -m pip install --upgrade pip && pip install -r libs/requirements.txt + displayName: 'Install dependencies' + + - script: | + pip install pytest + pytest tests --doctest-modules --junitxml=junit/test-results.xml + displayName: 'pytest' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() + +- job: 'Publish' + dependsOn: 'Test' + pool: + vmImage: 'Ubuntu-16.04' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + architecture: 'x64' + + - script: python setup.py sdist + displayName: 'Build sdist' diff --git a/tests/general.py b/tests/general.py deleted file mode 100755 index ddcc7aa0..00000000 --- a/tests/general.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python2 -from __future__ import print_function - -import datetime -import os -import sys - -import core -from core import logger, main_db, transcoder -from core.auto_process import comics, games, movies, music, tv -from core.auto_process.common import ProcessResult -from core.user_scripts import external_script -from core.forks import auto_fork -from core.utils import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status, server_responding - - -# Initialize the config -core.initialize() - -# label = core.TORRENT_CLASS.core.get_torrent_status('f33a9c4b15cbd9170722d700069af86746817ade', ['label']).get()['label'] -# print(label) - -if transcoder.is_video_good(core.TEST_FILE, 0): - print('FFPROBE Works') -else: - print('FFPROBE FAILED') - -test = core.CFG['SickBeard', 'NzbDrone']['tv'].isenabled() -print(test) -section = core.CFG.findsection('tv').isenabled() -print(section) -print(len(section)) -fork, fork_params = auto_fork('SickBeard', 'tv') - -if server_responding('http://127.0.0.1:5050'): - print('CouchPotato Running') -if server_responding('http://127.0.0.1:7073'): - print('SickBeard Running') -if server_responding('http://127.0.0.1:8181'): - print('HeadPhones Running') -if server_responding('http://127.0.0.1:8085'): - print('Gamez Running') -if server_responding('http://127.0.0.1:8090'): - print('Mylar Running') - -lan = 'pt' -lan = Language.fromalpha2(lan) -print(lan.alpha3) -vidName = '/volume1/Public/Movies/A Few Good Men/A Few Good Men(1992).mkv' -inputName = 'in.the.name.of.ben.hur.2016.bdrip.x264-rusted.nzb' -guess = guessit.guessit(inputName) -if guess: - # Movie Title - title = None - if 'title' in guess: - title = guess['title'] - # Movie Year - year = None - if 'year' in guess: - year = guess['year'] - url = 'http://www.omdbapi.com' - r = requests.get(url, params={'y': year, 't': title}, verify=False, timeout=(60, 300)) - results = r.json() - print(results) - -subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) -languages = set() -languages.add(lan) -video = subliminal.scan_video(vidName) -subtitles = subliminal.download_best_subtitles({video}, languages) -subliminal.save_subtitles(video, subtitles[video]) -del core.MYAPP diff --git a/tests/test_initialize.py b/tests/test_initialize.py new file mode 100755 index 00000000..8985f84a --- /dev/null +++ b/tests/test_initialize.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python +from __future__ import print_function + +import datetime +import os +import sys +sys.path.extend(["..","."]) + +import eol +eol.check() + +import cleanup +cleanup.clean(cleanup.FOLDER_STRUCTURE) + +import core +from core import logger, main_db +from core.auto_process import comics, games, movies, music, tv +from core.auto_process.common import ProcessResult +from core.plugins.downloaders.nzb.utils import get_nzoid +from core.plugins.plex import plex_update +from core.user_scripts import external_script +from core.utils import ( + char_replace, clean_dir, convert_to_ascii, + extract_files, get_dirs, get_download_info, + update_download_info_status, +) + +# Initialize the config +core.initialize() +del core.MYAPP + +def test_answer(): + assert core.CHECK_MEDIA == 1 From 410aab4c588536aee8cb6ed8c369d277392cb15d Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 12 Mar 2019 18:55:37 +1300 Subject: [PATCH 153/406] improve tests (#1574) improve tests --- azure-pipelines.yml | 2 +- tests/test_initialize.py | 55 +++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1a68748d..3f857b99 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r libs/requirements.txt + - script: python -m pip install --upgrade pip displayName: 'Install dependencies' - script: | diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 8985f84a..612261d3 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -1,33 +1,48 @@ #! /usr/bin/env python from __future__ import print_function - import datetime import os import sys -sys.path.extend(["..","."]) -import eol -eol.check() +def test_eol(): + import eol + eol.check() -import cleanup -cleanup.clean(cleanup.FOLDER_STRUCTURE) +def test_cleanup(): + import cleanup + cleanup.clean(cleanup.FOLDER_STRUCTURE) + +def test_import_core(): + import core + from core import logger, main_db + +def test_import_core_auto_process(): + from core.auto_process import comics, games, movies, music, tv + from core.auto_process.common import ProcessResult + +def test_import_core_plugins(): + from core.plugins.downloaders.nzb.utils import get_nzoid + from core.plugins.plex import plex_update + +def test_import_core_user_scripts(): + from core.user_scripts import external_script + +def test_import_six(): + from six import text_type + +def test_import_core_utils(): + from core.utils import ( + char_replace, clean_dir, convert_to_ascii, + extract_files, get_dirs, get_download_info, + update_download_info_status, replace_links, + ) import core from core import logger, main_db -from core.auto_process import comics, games, movies, music, tv -from core.auto_process.common import ProcessResult -from core.plugins.downloaders.nzb.utils import get_nzoid -from core.plugins.plex import plex_update -from core.user_scripts import external_script -from core.utils import ( - char_replace, clean_dir, convert_to_ascii, - extract_files, get_dirs, get_download_info, - update_download_info_status, -) -# Initialize the config -core.initialize() -del core.MYAPP +def test_initial(): + core.initialize() + del core.MYAPP -def test_answer(): +def test_core_parameters(): assert core.CHECK_MEDIA == 1 From 742d482020e16970ab992baabe72f95cc74ba202 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 13 Mar 2019 07:40:35 +1300 Subject: [PATCH 154/406] cleanup supporting files. --- .../CODE_OF_CONDUCT.md | 0 CONTRIBUTING.md => .github/CONTRIBUTING.md | 0 .../bug_report.md => ISSUE_TEMPLATE.md} | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 28 + README.md => .github/README.md | 0 changelog.txt | 769 ------------------ 6 files changed, 29 insertions(+), 770 deletions(-) rename CODE_OF_CONDUCT.md => .github/CODE_OF_CONDUCT.md (100%) rename CONTRIBUTING.md => .github/CONTRIBUTING.md (100%) rename .github/{ISSUE_TEMPLATE/bug_report.md => ISSUE_TEMPLATE.md} (88%) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md rename README.md => .github/README.md (100%) delete mode 100644 changelog.txt diff --git a/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to .github/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE.md similarity index 88% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .github/ISSUE_TEMPLATE.md index 688719d1..323376b4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE.md @@ -14,7 +14,7 @@ A clear and concise description of what the bug is. 1. Running on (Windows, Linux, NAS Model etc) '....' 2. Python version '....' 3. Download Client (NZBget, SABnbzd, Transmission) '....' -4. Intended media management (SickChill, CouchPotoato, Radarr, Sonarr) '....' +4. Intended Media Management (SickChill, CouchPotoato, Radarr, Sonarr) '....' **Expected behavior** A clear and concise description of what you expected to happen. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..89eb5514 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +# Description + +Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. + +Fixes # (issue) + +## Type of change + +Please delete options that are not relevant. + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] This change requires a documentation update + +# How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration + +**Test Configuration**: + +# Checklist: +- [ ] I have based this chnage on the nightly branch +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/README.md b/.github/README.md similarity index 100% rename from README.md rename to .github/README.md diff --git a/changelog.txt b/changelog.txt deleted file mode 100644 index b269e118..00000000 --- a/changelog.txt +++ /dev/null @@ -1,769 +0,0 @@ -Change_LOG / History - -V12.0.8 - -Refactor and Rename Modules -Add Medusa API -Fix return parsing from HeadPhones -Add Python end of life detection and reporting -Fix Py3 return from Popen (Transcoder and executable path detection) -Add variable sys_path to config (allows user to specify separate path for binary detection) -Various Py3 compatability fixes -Log successful when returning to Radarr CDH -Add exception handling when failing to return to original directory (due to permissions) -Don't load Torrent Clients when calling NZB processing - -V12.0.7 - -Refactor utils -Fix git subprocess -Fix cleanup script output -Add extra logging for fork detection -Additional code clean up - -V12.0.6 - -Hotfix for Manual Torrent run results. - -V12.0.5 - -Proper fix for source cleaner - -V12.0.4 - -Hotfix missed commit for source cleaner - -V12.0.3 - -Hotfix cleaning for source installs - -V12.0.2 - -Fix missed ProcessResult - -V12.0.1 - -Added Python 3 support -Updated all dependencies -Major code refactoring -Various bug fixes -Hotfix NZBGet not working without comment - -V12.0.0 - -NOTE: -- This release contains major backwards-incompatible changes to the internal API -- Windows users will need to manually install pywin32 - -Add Python 3 support -Add cleanup script for post-update cleanup -Update all dependencies -Move vendored packages in `core` to `libs` -Move common libs to `libs/common` -Move custom libs to `libs/custom` -Move Python 2 libs to `libs/py2` -Move Windows libs to `libs/windows` -Fix PEP8 -Add feature to make libs importable -Add feature to auto-update libs -Add path parent option to module path and default to using local path -Update invisible.cmd to return errorlevel -Update invisible.vbs to return exit code of 7zip -Update extractor.py for correct return code -Added debugging to extractor -Add option for windows extraction debugging -Remove surplus debug -Fix handling of None Password file -Fix invisible windows extraction -Fix execution of extraction -Start vbs directly from extractor -Delete invisible.cmd -Use args instead of Wscript.Arguments -Fix postprocessing of failed / bad downloads (#1091) -Fix release is None -Fix UnRAR failing - -V11.8.1 12/29/2018 - -Fix cleanup for nzbToMedia installed as a git submodule - -V11.8.0 12/28/2018 - -Add version information -Add bumpversion support -Fix automatic cleanup script - -V11.7 12/25/2018 - -Merry Christmas and Happy Holidays! - -Add cleanup script to clean up bytecode -Add automatic cleanup on update - -NOTE: Cleanup will force-run every time during a transitional period to minimize issues with upcoming refactoring - -V11.06 11/03/2018 - -updates to incorporate importMode for NzbDrone/Sonarr and Radarr. -Correct typo(s) for "Lidarr" category. -only pass id to CP if release id found. -fix issue with no release id and no imdbid. -Fixed NZBGet save of Lidarr config. -improve logging for imdb id lookup. -fix minor description error. -add better logging of movie name when added to CP. -attempt to clean up Liddar api commands. -update to use Mylar api. -set Torrent move-sym option to force SickRage process_method. -add rmDir import for HeadPhones processing. -change sickrage and sickchill names and modify api process to work with multiple sick* forks. -add NZBGet WebUI set of delete failed for HP. -fix qbittorrent to delete permanently (remove files on delete). - -V11.05 27/06/2018 - -Add qBittorrent support. -Add SickGear support. -Add SiCKRAGE api support. -Fix for single file download. -Diable media check for failed HeadPhones downloads. -Added Lidarr flow. Still awaiting confirmation of api interface commands and return. - -V11.04 30/12/2017 - -do not embed .sub. -add proper check of sub streams #1150 and filter out commentary. -traverse audiostreams in reverse. -add catch for OMDB api errors. -convert all listdir functions to unicode. -perform extraction, corruption checks, and transcoding when no server. -fix list indices errors when no fork set. -fix CP server responding test. Add trailing /. -use basestring to match unicode path in transcoder. -attempt autofork even if no username set. -allow long paths in Cleandir. -add Radarr handling. -minor fix for transcoder. -fix non-iterable type. -fix logging error. -DownloadedMovieScan updated to DownloadedMoviesScan. -add check to exception rename to not over-write exisiting. -don't try and process when no api/user. -Added omdbapikey functionality -force sonarr processing to "move". -already extracted archive not skipped. -fix text for keep_archive. -try to avoid spaces in outputdir. -change subtitle logging level. -Increase shutil copy buffer length from 4KB to 512KB. -improve user script media extension handling -add par2 rename/repair (linux only). - -V11.03 15/01/2017 - -Add -o to output path for 7zip. -Try album directory then parent directory for HeadPhones variants. -Prevent duplication of audio tracks in Transcoder. -Update uTorrent Client interface. -Updated to use force_next for SickRage to prevent postprocessing in queue. - -V11.02 30/11/2016 - -Added default "MKV-SD" -Added VideoResolution in nzbGet. -Fix Headphones direcotry parsing. -Remove proc_type when failed. -Added option "no_extract_failed" -Updated beautifulsoup 4 module. -Check for existence of codec_type key when counting streams. -Added default fallback for sabnzbd port = 8080. - -V11.01 30/10/2016 - -Updated external modules and changed config to dict. -Started making code python 3 compatible. -Fixed auto-fork detection for new Sick* branches. -Fixed invalid indexing scope for TorrentToMedia. -Add Medusa fork and new param "ignore_subs". -Added check for language tag size, convert 3 letter language codes. -Fixed guessit call to allow guessit to work of full file path. -Add the ability to set octal permissions on the processed files prior to handing it off to Sickrage/Couchpotato. -Catch errors if not audio codec name. -Allow manual scans to continue. -Revert to 7zip if others missing. -Fixed int conversion base 8 from string or int. -Added more logging to server tests. -Added MKV-SD Profile. -Check for preferred codec even if not preferred language. -Don't convert VobSub to mov_text. - -V10.15 29/05/2016 - -Don't copy archives when set to extract. -Specifically check for failed download handing regardless of fork. -sort Media file results by pathlength. -Synchronize changed SickRage directory param. -Don't remove release group information from base folder. -Don't add imdb id to file name when move-sym in use. -Fix string and integer concat error. - -V10.14 13/03/2016 - -Add option move-sym to create symlink to renamed files. -Transmission comment fix. -Prevent int errors in chmod. -Fix urllib warnings. -Create unique directory in output incase of rename error in sick/couch. -Add -strict -2 to dts codec. -Added support to handle archives in SickRage. -Report Downloader failures to SickRage. -Continue on encoding detection failure. -Strip trailing and leading whitespaces from `mount_points`. -Also check sabnzbd history for nzoid. -Add generic run mode (manually enter parameters for execution). - -V10.13 11/12/2015 - -Always add -strict -2 to aac codec. -Add "delete_on" for SickRage. -Add https handling for SABnzbd. -Added the ability to chmod Torrent diretory before processing. -Add option to not resume failed torrent. -Add Option to not resume successful torrent. -Add procees name to final SABnzbd message. -Fix SSL warnings forcomic processing. -Add .ts to mediaExtensions. -Don't update plex on failed. -Add option to preserve archive files after extraction. -Force_Clean doesn't over-ride delete_failed. -Added support for SickRageTV and SickRage branches. - -V10.12 21/09/2015 - -Updated Requests Module to Latest Version. Works with Python 2.7.10 -Add .img files to transcoder extraction routines. - -V10.11 28/05/2015 - -Use socket to verify if running on Linux. Prevents issues with stale pid. -Add timeouts and improve single instance handling. -Prevent Scale Up. -Improve regex for rename script. -Improve safe rename functionality. -Ignore .bts extensions. -Don't process output when no transcoding needed. -Ignore Thumbs.db on manual run. -Rename nzbtomedia to core. To prevent erros on non-case sensitive file systems. -Mark as bad if no media files found. -Increase server responding timeout. -Don't use last modified entry for CP renamer when no imdb id found. -Add plex library update. - -V10.10 29/01/2015 - -Fix error when extracting on windows. (added import of subprocess) -Fix subtitles download and emdedding. - -V10.9 19/01/2015 - -Prevent Errors when trying next release from CouchPotato (CouchPotato failed handling) -Prevent check for status change when using Manage scan (CouchPotato) -Better Tooltip for "host" in NZBGet settings. -Continue if failed to connect to Torrent Client. -Fixed resolution settings in Transcoder. -Make Windows Linking and extraction invisible. - -V10.8 15/12/2014 - -Impacts All -Removed "stand alone" scripts DeleteSamples and ResetDateTimes. These are now in https://github.com/clinton-hall/GetScripts -Removed chp.exe and replaced with vb script. -Improved Sonarr(NZBDrone) CDH support. -Use folder Permissions to set permissions for sub directories and files following extract. -Added support fro new SickRage Login. - -Impacts NZBs -Get NZOID from SABnzbd for better release matching. - -Impacts Torrents -Now gets Label from Deluge. -Changed SSL version for updated Deluge (0.3.11+) - -Impacts Transcoding -Fixed reported bugs. -Fix Audio mapping. -Fix Subtitle mapping from external files. -Fixed scaling errors. - -V10.7 06/10/2014 - -Impacts All -Add Transcoding of iso/images and VIDEO_TS structures. -Improved multiple session handling. -Improve NZBDrone handling (including Torrent Branch). -Multiple bug-fixes. - -Impacts NZBs -Add custom "group" replacements to allow better subtitle searching. - -Impacts Torrents -Add Vuze Torrent Client support. - -V10.6 26/08/2014 - -Impacts All -Bug Fixes. - -Impacts NZBs -Added FailureLink style feedback to dognzb for failed and corrupt downloads. - -V10.5 05/08/2014 - -Impacts All -Bug Fixes for Transcoder. -Support for lib-av as well as ffmpeg. -Fixed SickBeard aut-fork detection. - -V10.4 30/07/2014 - -Impacts All -Supress printed messages from extractor. -Allow no sub languages to be specified. -Ignore hdmv_pgs_subtitle codecs in transcoder. -Fix remote directory use with HeadPhones. -Only use nice and ionice when available. - -Impacts NZBs -Cleaner exit logging for SABnzbd. - -Impacts Torrents -Improved manual run handling. - -V10.3 15/07/2014 - -Impacts All -Fix auto-fork to identify default fork. - -V10.2 15/07/2014 - -Impacts All -Bug Fixes. -If extracting files and extraction not successful, return Failure and Don't delete archives. - -V10.1 11/07/2014 - -Impacts All -Improved Transcoder -Minor Bug Fixes -Now accepts Number of Audio Channels for Transcoder options. -Userscript can perform video corruption check first. -Improved extraction. Extract all subdirs and multiple "unique" archives in a directory. -Check if already running and wait for complete before continuing. - -Impacts NZBs -Allow UserScript for NZBs - -Impacts Torrents -Do Extraction Before Flatten - - -V10.0 03/07/2014 - -Impacts All -Changed to python2 (some systems now come with python = python3 as default). -Major changes to Transcoder. Only copy streams where possible. -Pre-defined Transcode options for some devices. -Added log_env option to capture environment variables. -Improved remote directory handling. -Various fixes. - -V9.3 09/06/2014 - -Impacts Torrents -Allow Headphones to remove torrents and data after processing. -Delete torrent if uselink = move -Added forceClean for outputDir. Works in file permissions prevent CP/SB from moving files. -Ignore .x264 from archive "part" checks. -Changed handling of TPB/Pistachitos SB forks. Default is to link/extract here. Disabled by Torrent_NoLink = 1. -Changed handling for HeadPhones Now that HeadPhones allows process directory to be defined. -Restructured Flow and streamlines process - -Impacts NZBs -Fix setting of Mylar config from NZBGet. -Created sheel scripts to nzbTo{App}. All now call the common nzbToMedia.py - -Impacts All -Changes to Couchpotato API for [nosql] added. Keeps aligned with current CouchPotato develop branch. -Add Auto Detection of SickBeard Fork. Thanks @echel0n -Added config class, re-coded migratecfg, misc bugfixes and code cleanup. -Added dynamic timeout based on directory size. -Added process_Method for SickBeard. -Changed configuration migrate process. -Major structure and process re-format. -Improved Manual Call Handling -Now prints github version into log when available. -Changed log location and format. -Added autoUpdate option via git. -All calls now use requests, not urllib. -All details now saved into Database. Can be used for more features later ;) -Improved status checking to ensure we only cleanup when successfully processed. - -Huge Thanks @echel0n - -V9.2 05/03/2014 - -Impacts All -Change default "wait_for" to 5 mins. CouchPotato can take more than 2 minutes to return on renamer.scan request. -Added SickBeard "wait_for" to bw customizable to prevent unwanted timeouts. -Fixed ascii conversion of directory name. -Added list of common sample ids and a way to set deletion of All media files less than the sample file size limit. -Added urlquote to dirName for CouchPotato (allows special characters in directory name) - -Impacts NZBs -Fix Error with manual run of nzbToMedia -Make sure SickBeard receives the individula download dir. -Added option to set SickBeard extraction as either Downlaoder or Destination (SickBeard). -Fixed Health Check handling for NZBGet. - -Impacts Torrents -Added option to run userscript once only (on directory). -Added Option to not flatten specific categories. -Added rtorrent integration. -Fixes for HeadPhones use (no flatten), no move/sym, and fix move back to original. - -V9.1 24/01/2014 - -Impacts All -Don't wait to verify status change in CouchPotato when no initial status (manual run) -Now use "wait_for" timing as socket timeout on the renamer.scan. It appears to now be delayed in confirming success. - -V9.0 19/01/2014 - -Impacts NZBs -SABnzbd 0.7.17+ now uses 8 arguments, not 7. These scripts now support the extra argument. - -Impacts Torrents -Always pause before processing. -Moved delete to end of routine, only when succesful process occurs. -Don't flatten hp category (in case multi cd album) -Added UserScript to be called for un-categorized downloads and other defined categories. -Added Torrent Hash to Deluge to assist with movie ID. -Added passwords option to attempt extraction od passworded archives. - -Impacts All -Added default socket timeout to prevent script hanging when the destination servers don't respond to http requests. -Made processing Category Centric as an option for people running multiple versions of SickBeard and CouchPotato etc. -Added TPB version of SickBeard processing. This now uses a fork pass-in instead of failed_fork. -Added new option to convert files, directories, and parameters to ASCII. To be used if you regularly download "foreign" titles and have problems with CP/SB. -Now only parse results from CouchPotato 50 at a time to prevent error with large wanted list. - -V8.5 05/10/2013 - -Impacts Torrents -Added Transmission RPC client. -Now pauses and resumes or removes from transmission. -Added debugging of input arguments from torrent clients. - -Impacts NZBs -Removed obsolete NZBget (pre V11) code. - -Impacts All. -Fixed HeadPhones processing. -Fixed movie parsing in CPS api. - -V8.4 14/09/2013 - -Impacts Torrents -Don't include 720p or 1080p as parts for extracting. -Extracts all sub-folders. -Added option to Move files. -Fix for single file torrents linked to subfolder of same name. - -Impacts All -Added option for SickBeard delay (for forks that use 1 minute check. -Updated to new api call in CouchPotato (movie.searcher.try_next) - - -V8.3 11/07/2013 - -Impacts All -Allow use of experimental AAC codec in transcoder. -Remove username and password when api key is used. -Add .m4v as media -Added ResetDateTime.py -Manual Opion for Mylar script. -Fixes for Gamez script. - -Impacts NZBs -Added option to remove folder path when CouchPotato on different system to downlaoder. -NZBGet v11.0 stable now current. - - -V8.2 26/05/2013 - -Impacts All -Add option to set the "wait_for" period. This is how long the script waits to see if the movie changes status in CouchPotato. -minSampleSize now moved to [extensions] section and availabe for nzbs and torrents. -New option in transcoder to use "niceness" on Linux. -Remove excess logging from transcoder. - -Impacts NZBs -Added Flatten of input directory and test for media files (including sample deletion) in autoProcessTV - -Impacts Torrents -Fixed Delete_Original option -Fix type which caused crash if not sickbeard or couchpotato. - - -V8.1 04/05/2013 - -Impacts All -Improved exception logging for error conditions - -Impacts Torrents -Fixed an import error when extracting - -Impacts NZBs -Fixed passthrough of inputName from NZBGet to pass the .nzb extension (required for SickBeard's failed fork) - - -V8.0 28/04/2013 - -Impacts All -Added download_id pass through for CouchPotato release matching -Uses single directory scanning for CouchPotato renamer -Matches imdb_id, download_id, clientAgent with CPS database - -Impacts NZB -Addeed direct configuration support via nzbget webUI (nzbget v11+) -All nzb scripts are now directly callabale in nzbget v11 -Settings made in nzbget webUI will be applied to the auotPorcessMedia.cfg when the scripts are run from nzbget. -Fixed TLS support for NZBGet email notifications (for V10 support) - -V7.1 28/03/2013 - -Impacts Torrents -Added test for chp.exe. If not found, calls 7zip directly -Added test for multi-part archives. Will only extract part1 - -Impacts NZB -Fixed failed download handling from nzbget (won't delete or move root!!!) -Fixed sendEmail for nzbget to use html with
line breaks - -V7.0 21/03/2013 - -Impacts Torrents -Added option to delete torrent and original files after processing (utorrent) - -Impacts NZB -Added nzbget windows script (to be compiled) -Changed nzbget folders to previous X.X, current-stable, testing X.X format -Fix nzbget change directory failure problem -Improved nzbget logging -Add logging to nzbget email notification -Synchronised v10 to latest nzbget testing scripts -Added failed download folder for failed downloads in nzbget -Added option to delete failed in nzbget -Created a single nzbToMedia.py script for all categories (will be the only nzb script compiled for windows) - -Impacts All -Added rotating log file handler -Added ffmpeg transcoder -Added CouchPotato status check to provide confirmation of renamer complete -CouchPotato status check will timeout after 2 minutes in case something goes wrong -Improved logging. -Improved scen exception handling. -Major changes to code layout -Better efficiency -Added support for Mylar, Gamez, and HeadPhones -Moved many of the "support" files to the autoProcess directory so that they aren't visible (looks neater) -Added migration tool to update .cfg file on first run following update. - -V6.0 03/03/2013 - -Impacts Torrents -Bundled 7zip binaries and created extraction functions. -Now pauses uTorrent seeding before calling renamer in SickBeard/CouchPotatoServer -uTorrent Resumes seeding after files (hardlinks) have been renamed - -Impacts NZB -Added local file logging. - -Impacts All -Added scene exception handling. Currently for "QoQ" -Improved code layout. - - -V5.1 22/02/2013 - -Improved category search to loop through directory structure. -Added support for deluge and potentially other Torrent clients. -uTorrent now must pass "utorrent" before "%D" "%N" "%L" -added test for date modified (less than 5 mins ago) if root directory and no torrent name. -".cp(ttxxxxxx)" tag preserved in directory name for CPS renaming. -All changes affect Torrent handling. Should not impact NZB handling. - -V5.0 20/02/2013 - -Fixed Extarction and Hard-Linking support in TorrentToMedia -Added new config options for movie file extensions, metadata extensions, compressed file extensions. -Added braid to sync linktastic. -Windows Builds now run without console displaying. -All changes affect Torrent handling. Should not impact NZB handling. - -V4.3 17/02/2013 - -Added Logger in TorrentToMedia.py -Added nzbget V10.0 script. -Delete sample files in nzbget postprocessing -Single Version for all files. - -V4.2 12/02/2013 - -Fixes to TorrentToMedia - -V4.1 02/02/2013 - -Added Torrent Support (µTorrent and Transmission). -Added manual run option for nzbToSickBeard. -Changed nzbGet script to use move not copy and remove. -Merged all .cfg scripts into one (autoProcessMedia.cfg). -Made all scripts execitable (755) on github. -Added category limits for email support in nzbget. -Fixed issue with replacements (of paths) in email messages in nzbget. - -V4.0 21/12/2012 - -Changed name from nzbToCouchPotato to nzbToMedia; Now supports mltiple post-processing from two nzb download clients. -Added email support for nzbget. -Version printing now for each of the nzbTo* scripts. -Added "custom" post-process support in nzbget. -Added post-process script output logging in nzbget. - -V3.2 11/12/2012 - -Added failed handling from NZBGet. Thanks to schumi2004. -Also added support for the "failed download" development branch of SickBeard from https://github.com/Tolstyak/Sick-Beard.git - -V3.1 02/12/2012 - -Added conversion to ensure the status passed to the autoProcessTV and autoProcessMovie is always handled as an integer. - -V3.0 30/11/2012 - -Changed name from sabToCouchPotato to nzbToCouchPotato as this now included NZBGet support. -Packaged the NZBGet postprocess files as well as modified version of nzbToSickBeard (from sabToSickBeard). - - -V2.2 05/10/2012 - -Re-wrote the failed downlaod handling to just search for the imdb ttXXXX identifier (as received from the nzb name) -Now issues only two api calls. movie.list and searcher.try_next - -Should be more robust with regards changes to CPS and also utilises less resources (i.e. less api call and and less processing). - - -V2.1 04/10/2012 - -detected a change in the movie release info format. Fixed the script to work with new format. - - -V2.0 04/10/2012 - -Fixed an issue with the failed download handling in that the status id for "snatched" can be different on each installation. now performs a status.list via api to verify the status. - -Also including a version print (currently 2.0... yeah original I know) so you know if you are current. - -removed the multiple versions. The former _recue version will perform the standard renamer only if "postprocess only verified downloads" (default) is enabled in SABnzbd. Also, the "unix" version works fine in Windows, only the "dos" version gave issue in Linux. In other words, this one version should work for all systems. -For historical reasons, the former download stats apply to the old versions: -sabToCouchPotato-dos - downloaded 143 times -sabToCouchPotato-unix - downloaded 205 times -sabToCouchPotato_recue - downloaded 105 times - -Also updated the Windows Build to include the same changes. I have removed the link to the linux build as this didn't work on all systems and it really shouldn't be necessary. Let me know if you need this updated. - - -V1.9 18/09/2012 - -compiled (build) versions of sabToSickBeard and sabToCouchPotato added for both Linux and Windows. links at top of post. - - -V1.9 16/09/2012 - -Added a compiled .exe version for windows. Should prevent the "python not recognised" issue and allow this to be used in conjunction with the windows build on systems that do not have python installed. - -This is the full (_recue version) if sabnzbd is set to post ptocess only verified jobs, this will not recue and will function as a standard renamer. - - -V1.9 27/08/2012 - -Following the latest CPS update on the master branch, this script is not really needed as CPS actually polls the SABnzbd api and does the same as this script (internally). - -However, if you have any issues with CPS constantly downloading the same movies, or filling the log with polling SABnzbd for completed movies, or otherwise prefer to use this method, then you can still use this script and make the following changes in CPS: -Settings, renamer, run every (advanced) = set to 1440 (or some longer interval) -Settings, renamer, next On_failed = off -Settings, downloaders, SABnzbd, Delete failed = off. - -V1.9 06/08/2012 - -Also added the integer handling of status in the sabToSickBeard.py script to prevent SickBeard trying to postprocess a failed TV download. Only impacts the _recue version - - -V1.8 05/08/2012 - -Modified the _recue version as SABnzbd 0.7.3 now appears to pass the "status" variable as a string not an integer!!! (or i had it wrong on first attempt :~) -This causes the old script to identify completed downloads as failed and recues the next download! - -The fix here should work with any conceivable subsequent updates in that I now make the sys.argv[7] an integer before passing it. if the variable already is an integer, this shouldn't cause any issues. - -status = int(sys.argv[7]) -autoProcessMovie.process(sys.argv[1], sys.argv[2], status) - - -V1.7 02/08/2012 - -Added a new version sabToCouchPotato_recue -This works the same as the other versions, but includes support for recuing failed downloads. -This is new, and only tested once (with success ) at my end. - -To get this to run you will need to uncheck the "post-process only verified jobs" option in SABnzbd. Also, to avoid issues with SickBeard postprocessing, I have included a modified postprocessing for SickBeard that just checks for failed status and then exits (the SickBeard Team are currently working on failed download handling and I will hopefully make this script work with that in the future) - -This re-cue works as follows: -Performs an api call to CPS to get a list of all wanted movies (with all data including the releases and status etc) -It finds the nzbname (from SABnzbd) in the json list returned from the api call (movie.list) and identifies the movie id and release id. -It performs an api call to make the release as "ignore" and then performs another api call to refresh the movie. -If another (next best) release that meets your criteria is already available it will send that to SABnzbd, otherwise it will wait until a new release becomes availabe. - -I have left the old versions here for now for those who don't want to try this. Also, if you don't uncheck the "post-process only verified jobs" in SABnzbd this code will perform the same as the previous versions. - -The next issue to tackle (if this works) is automating the deletion of failed download files in SABnzbd.... but I figured this was a start. - - -V1.6 22/07/2012 - -no functionality change, but providing scripts in both unix and dos format to prevent exit(127) errors. -if you are using windows, use the dos format. if you are using linux, use the unix format and unzip the files in linux. - - -V1.5 17/07/2012 - -add back the web_root parameter to set the URL base. - - -V1.4 17/07/2012 - -Have uploaded the latest version. -changes -Removed support for a movie.downlaoded api call that was only used in a seperate branch and is not expected to be merged. -Modified the passthrough to allow a manual call to this script (i.e. does not need to be called from SABnzbd). -Have added a helpfile that explains the setup options in a bit more detail. -Modified the .cfg.sample file to use 60 as a default delay and now specify that 60 should be your minimum to ensure the renamer.scan finds newly extracted movies. - - -V1.3 and earlier were not fully tracked, as the script itself (not files) was posted on the QNAP forums. - - - - - - From 8745af26298f995487ba3a21d74185c57fc158e2 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 13 Mar 2019 07:54:21 +1300 Subject: [PATCH 155/406] update to v12.0.9 --- .bumpversion.cfg | 2 +- .github/README.md | 4 ++-- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 96d2148f..eb16c596 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.8 +current_version = 12.0.9 commit = True tag = False diff --git a/.github/README.md b/.github/README.md index 63b8626b..0757be3c 100644 --- a/.github/README.md +++ b/.github/README.md @@ -1,5 +1,5 @@ -nzbToMedia v12.0.8 -================== +nzbToMedia +========== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) when using one of the popular NZB download clients like [SABnzbd](http://sabnzbd.org/ "SABnzbd") and [NZBGet](http://nzbget.sourceforge.net/ "NZBGet") on low performance systems like a NAS. diff --git a/core/__init__.py b/core/__init__.py index 9468f5a1..a68a31a7 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -78,7 +78,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.8' +__version__ = '12.0.9' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 9c366acf..6818d769 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.8', + version='12.0.9', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 9b31482ce3851b813b5731289ef0f9a8ac323e64 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 12 Mar 2019 16:48:50 -0400 Subject: [PATCH 156/406] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 89eb5514..d47031a9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi **Test Configuration**: # Checklist: -- [ ] I have based this chnage on the nightly branch +- [ ] I have based this change on the nightly branch - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation From 6aee6baf6e163c5e915c88102ddb17ae2fa9822c Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 14 Mar 2019 20:02:40 +1300 Subject: [PATCH 157/406] fix cleanup --- cleanup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cleanup.py b/cleanup.py index dcef7789..d313f2b4 100644 --- a/cleanup.py +++ b/cleanup.py @@ -17,6 +17,7 @@ FOLDER_STRUCTURE = { 'core': [ 'auto_process', 'extractor', + 'plugins', 'utils', ], } From ac7e0b702a3bca5bf298f3dce332bb8ce74db32e Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 14 Mar 2019 20:28:53 +1300 Subject: [PATCH 158/406] update to 12.0.10 --- .bumpversion.cfg | 2 +- core/__init__.py | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index eb16c596..31885a91 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.9 +current_version = 12.0.10 commit = True tag = False diff --git a/core/__init__.py b/core/__init__.py index a68a31a7..3087ff55 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -78,7 +78,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.9' +__version__ = '12.0.10' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 6818d769..803f430b 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.9', + version='12.0.10', license='GPLv3', description='Efficient on demand post processing', long_description=""" @@ -42,8 +42,8 @@ setup( 4. Continuously polling a folder for changes can prevent the drive from going to sleep 5. Continuous polling may encounter file access/permissions issues 6. Continuous polling may miss a folder change, causing the PVR app to wait forever - 6. An on-demand post-processing script is able to utilize additional functionality such as ffprobe media checking to test for bad video files. - 7. On-demand scripts can be tweaked to allow for delays with slow hardware + 7. An on-demand post-processing script is able to utilize additional functionality such as ffprobe media checking to test for bad video files. + 8. On-demand scripts can be tweaked to allow for delays with slow hardware nzbToMedia is an on-demand post-processing script and was created out of a demand for more efficient post-processing on low-performance hardware. Many features have been added so higher performance hardware can benefit too. From 19c3e1fd8591e8876615578f67d058111955a3a7 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 15 Mar 2019 20:42:21 +1300 Subject: [PATCH 159/406] remove .encode which creates byte vs string comparison issues. Fixes #1582 --- TorrentToMedia.py | 48 ++++++++++++++++++------------------ core/utils/identification.py | 16 ++++++------ core/utils/naming.py | 8 +++--- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 74f37165..bd1166f4 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -60,14 +60,14 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass + #try: + # input_directory = input_directory.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format (input_directory, input_name, input_category)) @@ -125,10 +125,10 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - try: - output_destination = output_destination.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # output_destination = output_destination.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass if output_destination in input_directory: output_destination = input_directory @@ -170,10 +170,10 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - try: - target_file = target_file.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # target_file = target_file.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass if root == 1: if not found_file: logger.debug('Looking for {0} in: {1}'.format(input_name, inputFile)) @@ -350,15 +350,15 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - try: - dir_name = dir_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # dir_name = dir_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass input_name = os.path.basename(dir_name) - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except UnicodeError: + # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) diff --git a/core/utils/identification.py b/core/utils/identification.py index c2dc0352..7a48ec87 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -90,14 +90,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - try: - input_name = input_name.encode(core.SYS_ENCODING) - except Exception: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except Exception: - pass + #try: + # input_name = input_name.encode(core.SYS_ENCODING) + #except Exception: + # pass + #try: + # input_directory = input_directory.encode(core.SYS_ENCODING) + #except Exception: + # pass if input_directory is None: # =Nothing to process here. return input_directory, input_name, input_category, root diff --git a/core/utils/naming.py b/core/utils/naming.py index 352d51ba..8b3e6971 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -20,10 +20,10 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - try: - name = name.encode(core.SYS_ENCODING) - except Exception: - pass + #try: + # name = name.encode(core.SYS_ENCODING) + #except Exception: + # pass return name From a3db8fb4b657a733cba2abeb9609862ffadb6c9c Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Wed, 27 Mar 2019 10:09:47 +1300 Subject: [PATCH 160/406] Test 1 (#1586) * add transcoder tests --- azure-pipelines.yml | 3 +++ tests/test_transcoder.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100755 tests/test_transcoder.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3f857b99..a609703b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,6 +32,9 @@ jobs: - script: python -m pip install --upgrade pip displayName: 'Install dependencies' + - script: sudo apt-get install ffmpeg + displayName: 'Install ffmpeg' + - script: | pip install pytest pytest tests --doctest-modules --junitxml=junit/test-results.xml diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py new file mode 100755 index 00000000..929e7f23 --- /dev/null +++ b/tests/test_transcoder.py @@ -0,0 +1,14 @@ +#! /usr/bin/env python +from __future__ import print_function +import datetime +import os +import sys +import json +import time +import requests + +import core +from core import logger, transcoder + +def test_transcoder_check(): + assert transcoder.is_video_good(core.TEST_FILE, 0) == True From aee3b151c0f2a7f34c705d91d94af9c720a96e7b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 29 Mar 2019 09:50:43 +1300 Subject: [PATCH 161/406] Lazylib 1 (#1587) * add support for LazyLibrarian. Fixes #1223 --- TorrentToMedia.py | 5 +- autoProcessMedia.cfg.spec | 25 +++++++++ core/auto_process/books.py | 75 ++++++++++++++++++++++++++ core/configuration.py | 15 ++++++ nzbToCouchPotato.py | 2 +- nzbToGamez.py | 2 +- nzbToLazyLibrarian.py | 104 +++++++++++++++++++++++++++++++++++++ nzbToMedia.py | 40 +++++++++++++- 8 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 core/auto_process/books.py create mode 100755 nzbToLazyLibrarian.py diff --git a/TorrentToMedia.py b/TorrentToMedia.py index bd1166f4..06d36e1a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -13,7 +13,7 @@ import sys import core from core import logger, main_db -from core.auto_process import comics, games, movies, music, tv +from core.auto_process import comics, games, movies, music, tv, books from core.auto_process.common import ProcessResult from core.plugins.plex import plex_update from core.user_scripts import external_script @@ -256,6 +256,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp result = comics.process(section_name, output_destination, input_name, status, client_agent, input_category) elif section_name == 'Gamez': result = games.process(section_name, output_destination, input_name, status, client_agent, input_category) + elif section_name == 'LazyLibrarian': + result = books.process(section_name, output_destination, input_name, status, client_agent, input_category) + plex_update(input_category) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 767d937c..b54c640d 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -282,6 +282,31 @@ ##### Set to path where download client places completed downloads locally for this category watch_dir = +[LazyLibrarian] + #### autoProcessing for Games + #### games - category that gets called for post-processing with Gamez + [[books]] + enabled = 0 + apikey = + host = localhost + port = 5299 + ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### + ssl = 0 + web_root = + # Enable/Disable linking for Torrents + Torrent_NoLink = 0 + keep_archive = 1 + extract = 1 + # Set this to minimum required size to consider a media file valid (in MB) + minSize = 0 + # Enable/Disable deleting ignored files (samples and invalid media files) + delete_ignored = 0 + ##### Enable if LazyLibrarian is on a remote server for this category + remote_path = 0 + ##### Set to path where download client places completed downloads locally for this category + watch_dir = + + [Network] # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. MountPoints = /volume1/Public/,E:\|/volume2/share/,\\NAS\ diff --git a/core/auto_process/books.py b/core/auto_process/books.py new file mode 100644 index 00000000..b05032db --- /dev/null +++ b/core/auto_process/books.py @@ -0,0 +1,75 @@ +# coding=utf-8 + +import os +import shutil + +import requests + +import core +from core import logger +from core.auto_process.common import ProcessResult +from core.utils import convert_to_ascii, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + status = int(status) + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + protocol = 'https://' if ssl else 'http://' + + url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) + if not server_responding(url): + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + params = { + 'api_key': apikey, + 'cmd': 'forceProcess', + 'dir': dir_name, + } + + logger.debug('Opening URL: {0}'.format(url), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {1}'.format(section, section), + status_code=1, + ) + + result = r.json() + logger.postprocess('{0}'.format(result), section) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif result['success']: + logger.postprocess('SUCCESS: ForceProcess for {0} has been started in LazyLibrarian'.format(dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + else: + logger.error('FAILED: ForceProcess of {0} has Failed in LazyLibrarian'.format(dir_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, + ) diff --git a/core/configuration.py b/core/configuration.py index 31ea4852..216a1f05 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -383,6 +383,21 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new[section][os.environ[env_cat_key]][option] = value cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + section = 'LazyLibrarian' + env_cat_key = 'NZBPO_LLCATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'REMOTE_PATH'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_LL{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + section = 'NzbDrone' env_cat_key = 'NZBPO_NDCATEGORY' env_keys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 12223106..be7fb4f3 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -4,7 +4,7 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to CouchPotato. # # This script sends the download to your automated media management servers. # diff --git a/nzbToGamez.py b/nzbToGamez.py index d1559e72..42d0a2f5 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -4,7 +4,7 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to Gamez. # # This script sends the download to your automated media management servers. # diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py new file mode 100755 index 00000000..96ab86c8 --- /dev/null +++ b/nzbToLazyLibrarian.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# coding=utf-8 +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Post-Process to LazyLibrarian. +# +# This script sends the download to your automated media management servers. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +# +### OPTIONS ### + +## General + +# Auto Update nzbToMedia (0, 1). +# +# Set to 1 if you want nzbToMedia to automatically check for and update to the latest version +#auto_update=0 + +# Safe Mode protection of DestDir (0, 1). +# +# Enable/Disable a safety check to ensure we don't process all downloads in the default_downloadDirectory by mistake. +#safe_mode=1 + +## LazyLibrarian + +# LazyLibrarian script category. +# +# category that gets called for post-processing with LazyLibrarian. +#llCategory=games + +# LazyLibrarian api key. +#llapikey= + +# LazyLibrarian host. +# +# The ipaddress for your LazyLibrarian server. e.g For the Same system use localhost or 127.0.0.1 +#llhost=localhost + +# LazyLibrarian port. +#llport=5299 + +# LazyLibrarian uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#llssl=0 + +# LazyLibrarian web_root +# +# set this if using a reverse proxy. +#llweb_root= + +# LazyLibrarian watch directory. +# +# set this to where your LazyLibrarian completed downloads are. +#llwatch_dir= + +## Posix + +# Niceness for external tasks Extractor and Transcoder. +# +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +#niceness=10 + +# ionice scheduling class (0, 1, 2, 3). +# +# Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. +#ionice_class=2 + +# ionice scheduling class data. +# +# Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. +#ionice_classdata=4 + +## WakeOnLan + +# use WOL (0, 1). +# +# set to 1 to send WOL broadcast to the mac and test the server (e.g. xbmc) on the host and port specified. +#wolwake=0 + +# WOL MAC +# +# enter the mac address of the system to be woken. +#wolmac=00:01:2e:2D:64:e1 + +# Set the Host and Port of a server to verify system has woken. +#wolhost=192.168.1.37 +#wolport=80 + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import sys + +import nzbToMedia + +section = 'LazyLibrarian' +result = nzbToMedia.main(sys.argv, section) +sys.exit(result) diff --git a/nzbToMedia.py b/nzbToMedia.py index d91f413f..c663916c 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -4,7 +4,8 @@ ############################################################################## ### NZBGET POST-PROCESSING SCRIPT ### -# Post-Process to CouchPotato, SickBeard, NzbDrone, Mylar, Gamez, HeadPhones. +# Post-Process to CouchPotato, SickBeard, Sonarr, Mylar, Gamez, HeadPhones, +# LazyLibrarian, Radarr, Lidarr # # This script sends the download to your automated media management servers. # @@ -410,6 +411,39 @@ # Enable to replace local path with the path as per the mountPoints below. #gzremote_path=0 +## LazyLibrarian + +# LazyLibrarian script category. +# +# category that gets called for post-processing with LazyLibrarian. +#llCategory=games + +# LazyLibrarian api key. +#llapikey= + +# LazyLibrarian host. +# +# The ipaddress for your LazyLibrarian server. e.g For the Same system use localhost or 127.0.0.1 +#llhost=localhost + +# LazyLibrarian port. +#llport=5299 + +# LazyLibrarian uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#llssl=0 + +# LazyLibrarian web_root +# +# set this if using a reverse proxy. +#llweb_root= + +# LazyLibrarian watch directory. +# +# set this to where your LazyLibrarian completed downloads are. +#llwatch_dir= + ## Network # Network Mount Points (Needed for remote path above) @@ -635,7 +669,7 @@ import sys import core from core import logger, main_db -from core.auto_process import comics, games, movies, music, tv +from core.auto_process import comics, games, movies, music, tv, books from core.auto_process.common import ProcessResult from core.plugins.downloaders.nzb.utils import get_nzoid from core.plugins.plex import plex_update @@ -763,6 +797,8 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d result = comics.process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == 'Gamez': result = games.process(section_name, input_directory, input_name, status, client_agent, input_category) + elif section_name == 'LazyLibrarian': + result = books.process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == 'UserScript': result = external_script(input_directory, input_name, input_category, section[usercat]) else: From 1597763d3007dad4c013aefa93220f1febf1c082 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 29 Mar 2019 10:38:59 +1300 Subject: [PATCH 162/406] minor fix for LazyLibrarian api. --- autoProcessMedia.cfg.spec | 4 ++-- core/auto_process/books.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index b54c640d..30a399a0 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -283,8 +283,8 @@ watch_dir = [LazyLibrarian] - #### autoProcessing for Games - #### games - category that gets called for post-processing with Gamez + #### autoProcessing for LazyLibrarian + #### books - category that gets called for post-processing with LazyLibrarian [[books]] enabled = 0 apikey = diff --git a/core/auto_process/books.py b/core/auto_process/books.py index b05032db..c029d06f 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -36,12 +36,11 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_name, dir_name = convert_to_ascii(input_name, dir_name) params = { - 'api_key': apikey, + 'apikey': apikey, 'cmd': 'forceProcess', 'dir': dir_name, } - - logger.debug('Opening URL: {0}'.format(url), section) + logger.debug('Opening URL: {0} with params: {1}'.format(url, params), section) try: r = requests.get(url, params=params, verify=False, timeout=(30, 300)) @@ -52,8 +51,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', status_code=1, ) - result = r.json() - logger.postprocess('{0}'.format(result), section) + logger.postprocess('{0}'.format(r.text), section) if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) @@ -61,7 +59,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), status_code=1, ) - elif result['success']: + elif r.text == 'OK': logger.postprocess('SUCCESS: ForceProcess for {0} has been started in LazyLibrarian'.format(dir_name), section) return ProcessResult( message='{0}: Successfully post-processed {1}'.format(section, input_name), From 809e6420393464d65945931d5083d7de01c3c3c8 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sat, 30 Mar 2019 08:47:20 +1300 Subject: [PATCH 163/406] fix LL default branch. --- nzbToLazyLibrarian.py | 2 +- nzbToMedia.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index 96ab86c8..0a26013b 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -31,7 +31,7 @@ # LazyLibrarian script category. # # category that gets called for post-processing with LazyLibrarian. -#llCategory=games +#llCategory=books # LazyLibrarian api key. #llapikey= diff --git a/nzbToMedia.py b/nzbToMedia.py index c663916c..8d969d6a 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -416,7 +416,7 @@ # LazyLibrarian script category. # # category that gets called for post-processing with LazyLibrarian. -#llCategory=games +#llCategory=books # LazyLibrarian api key. #llapikey= From f20e1e4f0d3366c35ff075735c5c802cef8c2b51 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 11:45:04 -0400 Subject: [PATCH 164/406] Add pywin32 to setup.py install_requires on Windows --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 803f430b..b2f78439 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,9 @@ setup( author_email='fock_wulf@hotmail.com', url='https://github.com/clinton-hall/nzbToMedia', packages=['core'], + install_requires=[ + 'pywin32;platform_system=="Windows"', + ], classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers 'Development Status :: 5 - Production/Stable', From a531f4480e33dc720568da6ac38487e8b62333c6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:13:11 -0400 Subject: [PATCH 165/406] Add source install cleanup test --- azure-pipelines.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a609703b..485f214f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,6 +40,13 @@ jobs: pytest tests --doctest-modules --junitxml=junit/test-results.xml displayName: 'pytest' + - script: + rm -rf .git + python cleanup.py + python TorrentToMedia.py + python nzbToMedia.py + displayName: 'Test source install cleanup' + - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-results.xml' From 02813a6eaf865157433dadfccaf9ee651c64e6d7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:39:12 -0400 Subject: [PATCH 166/406] Add source install cleanup test --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 485f214f..96321a5b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,7 @@ jobs: pytest tests --doctest-modules --junitxml=junit/test-results.xml displayName: 'pytest' - - script: + - script: | rm -rf .git python cleanup.py python TorrentToMedia.py From 16b7c1149511528972751d1b986558ead0b188be Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:45:07 -0400 Subject: [PATCH 167/406] Force cleanup errors for confirming CI test --- cleanup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cleanup.py b/cleanup.py index d313f2b4..43dc5775 100644 --- a/cleanup.py +++ b/cleanup.py @@ -15,10 +15,6 @@ FOLDER_STRUCTURE = { 'win', ], 'core': [ - 'auto_process', - 'extractor', - 'plugins', - 'utils', ], } From f5fdc145774b936b592d33ad523a43ec2a8eb204 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 31 Mar 2019 12:49:32 -0400 Subject: [PATCH 168/406] Revert "Force cleanup errors for confirming CI test" This reverts commit 16b7c1149511528972751d1b986558ead0b188be. --- cleanup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cleanup.py b/cleanup.py index 43dc5775..d313f2b4 100644 --- a/cleanup.py +++ b/cleanup.py @@ -15,6 +15,10 @@ FOLDER_STRUCTURE = { 'win', ], 'core': [ + 'auto_process', + 'extractor', + 'plugins', + 'utils', ], } From 825b48a6c121a12f398dea3ea23410155383e737 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 4 Apr 2019 11:34:25 +1300 Subject: [PATCH 169/406] add h265 to MKV profile allow. Fixes #1592 --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 3087ff55..56f08bce 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -746,7 +746,7 @@ def configure_transcoder(): }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, From 822603d0211d6b2ac0c4daf9a20a8882b21128a1 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:10:38 -0400 Subject: [PATCH 170/406] Add tox.ini --- tox.ini | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..a18abb54 --- /dev/null +++ b/tox.ini @@ -0,0 +1,54 @@ +; a generative tox configuration, see: https://tox.readthedocs.io/en/latest/config.html#generative-envlist + +[tox] +envlist = + clean, + check, + {py27, py35, py36, py37}, + report + +[testenv] +basepython = + py27: {env:TOXPYTHON:python2.7} + py35: {env:TOXPYTHON:python3.5} + py36: {env:TOXPYTHON:python3.6} + py37: {env:TOXPYTHON:python3.7} + {clean,check,report,codecov}: {env:TOXPYTHON:python3} +setenv = + PYTHONPATH={toxinidir}/tests + PYTHONUNBUFFERED=yes +passenv = + * +usedevelop = false +skip_install = true +deps = + pytest + pytest-travis-fold + pytest-cov + pywin32 ; sys.platform == 'win32' +commands = + {posargs:pytest --cov --cov-report=term-missing tests} + +[coverage:run] +omit = + libs/* + +[testenv:codecov] +deps = + codecov +skip_install = true +commands = + coverage xml --ignore-errors + codecov [] + +[testenv:report] +deps = coverage +skip_install = true +commands = + coverage report + coverage html + +[testenv:clean] +commands = coverage erase +skip_install = true +deps = coverage From a8d1cc4fe9f02005443d73ef0097c635713d1181 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 17:11:16 -0400 Subject: [PATCH 171/406] Add flake8 quality checks to tox.ini --- tox.ini | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tox.ini b/tox.ini index a18abb54..2684e857 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,30 @@ deps = commands = {posargs:pytest --cov --cov-report=term-missing tests} +[flake8] +max-line-length = 79 +verbose = 2 +statistics = True +ignore = +; -- flake8 -- +; E501 line too long + E501 + +per-file-ignores = +; F401 imported but unused +; E402 module level import not at top of file + core/__init__.py: E402, F401 + core/utils/__init__.py: F401 + core/plugins/downloaders/configuration.py: F401 + core/plugins/downloaders/utils.py: F401 + +[testenv:check] +deps = + flake8 +skip_install = true +commands = + flake8 core tests setup.py + [coverage:run] omit = libs/* From 90090d7e02ed8b24306e219b38836f3456a1276a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:36:23 -0400 Subject: [PATCH 172/406] Fix flake8 E117 over-indented --- core/auto_process/movies.py | 32 ++++++++++++++++---------------- core/version_check.py | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index b7fe3a66..ff1222bf 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -373,22 +373,22 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', except Exception: pass elif scan_id: - url = '{0}/{1}'.format(base_url, scan_id) - command_status = command_complete(url, params, headers, section) - if command_status: - logger.debug('The Scan command return status: {0}'.format(command_status), section) - if command_status in ['completed']: - logger.debug('The Scan command has completed successfully. Renaming was successful.', section) - return ProcessResult( - message='{0}: Successfully post-processed {1}'.format(section, input_name), - status_code=0, - ) - elif command_status in ['failed']: - logger.debug('The Scan command has failed. Renaming was not successful.', section) - # return ProcessResult( - # message='{0}: Failed to post-process {1}'.format(section, input_name), - # status_code=1, - # ) + url = '{0}/{1}'.format(base_url, scan_id) + command_status = command_complete(url, params, headers, section) + if command_status: + logger.debug('The Scan command return status: {0}'.format(command_status), section) + if command_status in ['completed']: + logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status in ['failed']: + logger.debug('The Scan command has failed. Renaming was not successful.', section) + # return ProcessResult( + # message='{0}: Failed to post-process {1}'.format(section, input_name), + # status_code=1, + # ) if not os.path.isdir(dir_name): logger.postprocess('SUCCESS: Input Directory [{0}] has been processed and removed'.format( diff --git a/core/version_check.py b/core/version_check.py index 53945e1f..d865b9f3 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -199,8 +199,8 @@ class GitUpdateManager(UpdateManager): logger.log(u'{cmd} : returned successful'.format(cmd=cmd), logger.DEBUG) exit_status = 0 elif core.LOG_GIT and exit_status in (1, 128): - logger.log(u'{cmd} returned : {output}'.format - (cmd=cmd, output=output), logger.DEBUG) + logger.log(u'{cmd} returned : {output}'.format + (cmd=cmd, output=output), logger.DEBUG) else: if core.LOG_GIT: logger.log(u'{cmd} returned : {output}, treat as error for now'.format From 87e813f06280c5b46a15f367f5e148a2181c0227 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:04:50 -0400 Subject: [PATCH 173/406] Fix flake8 E126 continuation line over-indented for hanging indent --- core/forks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/forks.py b/core/forks.py index be81a965..897a1fd9 100644 --- a/core/forks.py +++ b/core/forks.py @@ -42,7 +42,8 @@ def auto_fork(section, input_category): logger.info('Attempting to verify {category} fork'.format (category=input_category)) url = '{protocol}{host}:{port}{root}/api/rootfolder'.format( - protocol=protocol, host=host, port=port, root=web_root) + protocol=protocol, host=host, port=port, root=web_root + ) headers = {'X-Api-Key': apikey} try: r = requests.get(url, headers=headers, stream=True, verify=False) @@ -65,10 +66,12 @@ def auto_fork(section, input_category): if apikey: url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess'.format( - protocol=protocol, host=host, port=port, root=web_root, apikey=apikey) + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey + ) else: url = '{protocol}{host}:{port}{root}/home/postprocess/'.format( - protocol=protocol, host=host, port=port, root=web_root) + protocol=protocol, host=host, port=port, root=web_root + ) # attempting to auto-detect fork try: From 07ad515b5066c6bbc53f639d4679647b01c02790 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:18:31 -0400 Subject: [PATCH 174/406] Fix flake8 E226 missing whitespace around arithmetic operator --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 56f08bce..0a224f94 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -870,7 +870,7 @@ def configure_utility_locations(): else: if SYS_PATH: - os.environ['PATH'] += ':'+SYS_PATH + os.environ['PATH'] += ':' + SYS_PATH try: SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip().decode() except Exception: From 8a22f20a8b189e149a858547a97a807c1e1bf828 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:10:20 -0400 Subject: [PATCH 175/406] Fix flake8 E241 multiple spaces after ':' --- core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/__init__.py b/core/__init__.py index 0a224f94..40b9d54f 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -108,7 +108,7 @@ FORKS = { FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, - FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, + FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} } From 5f633b931aaa0508dae655872446429ce0531ac3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:27:39 -0400 Subject: [PATCH 176/406] Fix flake8 E261 at least two spaces before inline comment --- core/auto_process/movies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index ff1222bf..76d49b78 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -256,7 +256,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', return ProcessResult( message='{0}: Sending failed download back to {0}'.format(section), status_code=1, # Return as failed to flag this in the downloader. - ) # Return failed flag, but log the event as successful. + ) # Return failed flag, but log the event as successful. if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) From a571fc31224ef0a86fa54c110c32bafc3517d96f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:42:46 -0400 Subject: [PATCH 177/406] Fix flake8 E265 block comment should start with '# ' --- core/utils/identification.py | 8 ++++---- core/utils/naming.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 7a48ec87..5aada2c1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -90,13 +90,13 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass - #try: + # try: # input_directory = input_directory.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass if input_directory is None: # =Nothing to process here. diff --git a/core/utils/naming.py b/core/utils/naming.py index 8b3e6971..ab8d9174 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -20,9 +20,9 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - #try: + # try: # name = name.encode(core.SYS_ENCODING) - #except Exception: + # except Exception: # pass return name From 032f7456f9656072a510eacbe86f41014fba3984 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:52:41 -0400 Subject: [PATCH 178/406] Fix flake8 E302 expected 2 blank lines, found 1 --- tests/test_initialize.py | 8 ++++++++ tests/test_transcoder.py | 1 + 2 files changed, 9 insertions(+) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 612261d3..ecc11331 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -8,28 +8,35 @@ def test_eol(): import eol eol.check() + def test_cleanup(): import cleanup cleanup.clean(cleanup.FOLDER_STRUCTURE) + def test_import_core(): import core from core import logger, main_db + def test_import_core_auto_process(): from core.auto_process import comics, games, movies, music, tv from core.auto_process.common import ProcessResult + def test_import_core_plugins(): from core.plugins.downloaders.nzb.utils import get_nzoid from core.plugins.plex import plex_update + def test_import_core_user_scripts(): from core.user_scripts import external_script + def test_import_six(): from six import text_type + def test_import_core_utils(): from core.utils import ( char_replace, clean_dir, convert_to_ascii, @@ -44,5 +51,6 @@ def test_initial(): core.initialize() del core.MYAPP + def test_core_parameters(): assert core.CHECK_MEDIA == 1 diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 929e7f23..15b8f4aa 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -10,5 +10,6 @@ import requests import core from core import logger, transcoder + def test_transcoder_check(): assert transcoder.is_video_good(core.TEST_FILE, 0) == True From 8e6e2d16470c698929f1dfad70fc557989e8a389 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:28:36 -0400 Subject: [PATCH 179/406] Fix flake8 E305 expected 2 blank lines after class or function definition, found 1 --- tests/test_initialize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index ecc11331..90759dbd 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -44,6 +44,7 @@ def test_import_core_utils(): update_download_info_status, replace_links, ) + import core from core import logger, main_db From d20879843064f44acac0451b2fde991e8e8d0d55 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:50:11 -0400 Subject: [PATCH 180/406] Fix flake8 E402 module level import not at top of file --- tests/test_initialize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 90759dbd..ea25826e 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -4,6 +4,10 @@ import datetime import os import sys +import core +from core import logger, main_db + + def test_eol(): import eol eol.check() @@ -45,9 +49,6 @@ def test_import_core_utils(): ) -import core -from core import logger, main_db - def test_initial(): core.initialize() del core.MYAPP From faa378f7875f7f1cd20d99f6edd72a90efb888cd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:30:35 -0400 Subject: [PATCH 181/406] Fix flake8 E712 comparison to True should be 'if cond is True:' or 'if cond:' --- tests/test_transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 15b8f4aa..c027aee8 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -12,4 +12,4 @@ from core import logger, transcoder def test_transcoder_check(): - assert transcoder.is_video_good(core.TEST_FILE, 0) == True + assert transcoder.is_video_good(core.TEST_FILE, 0) is True From 644a11118c7d05e606234a1ca7ec6169b81c46f5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 13:07:24 -0400 Subject: [PATCH 182/406] Fix flake8 F401 imported but unused --- core/auto_process/books.py | 3 --- core/utils/identification.py | 1 - core/utils/naming.py | 1 - tests/test_initialize.py | 23 ++++++----------------- tests/test_transcoder.py | 8 +------- 5 files changed, 7 insertions(+), 29 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index c029d06f..7d843f5e 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -1,8 +1,5 @@ # coding=utf-8 -import os -import shutil - import requests import core diff --git a/core/utils/identification.py b/core/utils/identification.py index 5aada2c1..9029f7e4 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -5,7 +5,6 @@ import guessit import requests from six import text_type -import core from core import logger from core.utils.naming import sanitize_name diff --git a/core/utils/naming.py b/core/utils/naming.py index ab8d9174..b54e3258 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -1,5 +1,4 @@ import re -import core def sanitize_name(name): diff --git a/tests/test_initialize.py b/tests/test_initialize.py index ea25826e..dffcfe76 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -1,11 +1,7 @@ #! /usr/bin/env python from __future__ import print_function -import datetime -import os -import sys import core -from core import logger, main_db def test_eol(): @@ -19,34 +15,27 @@ def test_cleanup(): def test_import_core(): - import core - from core import logger, main_db + pass def test_import_core_auto_process(): - from core.auto_process import comics, games, movies, music, tv - from core.auto_process.common import ProcessResult + pass def test_import_core_plugins(): - from core.plugins.downloaders.nzb.utils import get_nzoid - from core.plugins.plex import plex_update + pass def test_import_core_user_scripts(): - from core.user_scripts import external_script + pass def test_import_six(): - from six import text_type + pass def test_import_core_utils(): - from core.utils import ( - char_replace, clean_dir, convert_to_ascii, - extract_files, get_dirs, get_download_info, - update_download_info_status, replace_links, - ) + pass def test_initial(): diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index c027aee8..148156b0 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -1,14 +1,8 @@ #! /usr/bin/env python from __future__ import print_function -import datetime -import os -import sys -import json -import time -import requests import core -from core import logger, transcoder +from core import transcoder def test_transcoder_check(): From 0350521b874c7d38340a8f19dbb1b19e53cbcaf6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:37:23 -0400 Subject: [PATCH 183/406] Fix flake8 W291 trailing whitespace --- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index f2c53c45..42473ab8 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -171,7 +171,7 @@ def par2(dirname): cmd = '' for item in command: cmd = '{cmd} {item}'.format(cmd=cmd, item=item) - logger.debug('calling command:{0}'.format(cmd), 'PAR2') + logger.debug('calling command:{0}'.format(cmd), 'PAR2') try: proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) proc.communicate() diff --git a/core/transcoder.py b/core/transcoder.py index c53a1211..edb98761 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -828,7 +828,7 @@ def transcode_directory(dir_name): if not os.listdir(text_type(new_dir)): # this is an empty directory and we didn't transcode into it. os.rmdir(new_dir) new_dir = dir_name - if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB + if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB new_dir = dir_name bitbucket.close() return final_result, new_dir diff --git a/setup.py b/setup.py index b2f78439..86d82b89 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( version='12.0.10', license='GPLv3', description='Efficient on demand post processing', - long_description=""" + long_description=""" nzbToMedia ========== From 697df555ec36367f718eec568ed9fd8976239f96 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 12:44:44 -0400 Subject: [PATCH 184/406] Fix flake8 W293 blank line contains whitespace --- core/auto_process/books.py | 2 +- core/logger.py | 2 +- setup.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index 7d843f5e..8ba74a43 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -49,7 +49,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', ) logger.postprocess('{0}'.format(r.text), section) - + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) return ProcessResult( diff --git a/core/logger.py b/core/logger.py index 3305a96e..d73b501b 100644 --- a/core/logger.py +++ b/core/logger.py @@ -132,7 +132,7 @@ class NTMRotatingLogHandler(object): """ Returns a numbered log file name depending on i. If i==0 it just uses logName, if not it appends it to the extension (blah.log.3 for i == 3) - + i: Log number to ues """ diff --git a/setup.py b/setup.py index 86d82b89..fed3a4be 100644 --- a/setup.py +++ b/setup.py @@ -24,18 +24,18 @@ setup( long_description=""" nzbToMedia ========== - + Efficient on demand post processing ----------------------------------- - + A PVR app needs to know when a download is ready for post-processing. There are two methods: - + 1. On-demand post-processing script (e.g. sabToSickBeard.py or nzbToMedia.py): A script in the downloader runs once at the end of the download job and notifies the PVR app that the download is complete. - + 2. Continuous folder scanning: The PVR app frequently polls download folder(s) for completed downloads. - + On-demand is superior, for several reasons: - + 1. The PVR app is notified only once, exactly when the download is ready for post-processing 2. The PVR app does not have to wait for the next poll interval before it starts processing 3. Continuous polling is not as efficient and is more stressful on low performance hardware @@ -46,7 +46,7 @@ setup( 8. On-demand scripts can be tweaked to allow for delays with slow hardware nzbToMedia is an on-demand post-processing script and was created out of a demand for more efficient post-processing on low-performance hardware. Many features have been added so higher performance hardware can benefit too. - + Many issues that users have with folder scanning can be fixed by switching to on-demand. A whole class of support issues can be eliminated by using nzbToMedia. """, author='Clinton Hall', From 0bcbabd6816719ac9dd1a9f3807f6273623d133e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:05:03 -0400 Subject: [PATCH 185/406] Add flake8-commas to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 2684e857..74cf8e25 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,7 @@ per-file-ignores = [testenv:check] deps = flake8 + flake8-commas skip_install = true commands = flake8 core tests setup.py From 14b2aa6bf46dc2b14db96ed6ac59e274196b8d6e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:03:25 -0400 Subject: [PATCH 186/406] Fix flake8-commas C812 missing trailing comma --- core/__init__.py | 36 +++++++++---------- core/auto_process/comics.py | 2 +- core/auto_process/common.py | 2 +- core/auto_process/games.py | 2 +- core/auto_process/music.py | 6 ++-- core/databases.py | 4 +-- core/forks.py | 6 ++-- core/logger.py | 4 +-- core/main_db.py | 8 ++--- .../downloaders/torrent/qbittorrent.py | 2 +- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- setup.py | 2 +- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 40b9d54f..6b4e40e9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -110,7 +110,7 @@ FORKS = { FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, - FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} + FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} @@ -653,7 +653,7 @@ def configure_transcoder(): codec_alias = { 'libx264': ['libx264', 'h264', 'h.264', 'AVC', 'MPEG-4'], 'libmp3lame': ['libmp3lame', 'mp3'], - 'libfaac': ['libfaac', 'aac', 'faac'] + 'libfaac': ['libfaac', 'aac', 'faac'], } transcode_defaults = { 'iPad': { @@ -662,7 +662,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPad-1080p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -670,7 +670,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPad-720p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -678,7 +678,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Apple-TV': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -686,7 +686,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPod': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -694,7 +694,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'iPhone': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -702,7 +702,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'PS3': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -710,7 +710,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'xbox': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -718,7 +718,7 @@ def configure_transcoder(): 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-480p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -726,7 +726,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-720p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -734,7 +734,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'Roku-1080p': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -742,7 +742,7 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 160000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, @@ -750,7 +750,7 @@ def configure_transcoder(): 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'mp4-scene-release': { 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': 19, 'VLEVEL': '3.1', @@ -758,7 +758,7 @@ def configure_transcoder(): 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, - 'SCODEC': 'mov_text' + 'SCODEC': 'mov_text', }, 'MKV-SD': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': '1200k', 'VCRF': None, 'VLEVEL': None, @@ -766,8 +766,8 @@ def configure_transcoder(): 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, - 'SCODEC': 'mov_text' - } + 'SCODEC': 'mov_text', + }, } if DEFAULTS and DEFAULTS in transcode_defaults: VEXTENSION = transcode_defaults[DEFAULTS]['VEXTENSION'] @@ -957,7 +957,7 @@ def check_python(): major=sys.version_info[0], minor=sys.version_info[1], x=days_left, - ) + ), ) if days_left <= grace_period: logger.warning('Please upgrade to a more recent Python version.') diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 7049704b..1669d8d8 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -60,7 +60,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', logger.error('Unable to open URL', section) return ProcessResult( message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), - status_code=1 + status_code=1, ) if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: logger.error('Server returned status {0}'.format(r.status_code), section) diff --git a/core/auto_process/common.py b/core/auto_process/common.py index 8da83485..8c3d9ef2 100644 --- a/core/auto_process/common.py +++ b/core/auto_process/common.py @@ -17,7 +17,7 @@ class ProcessResult(object): def __str__(self): return 'Processing {0}: {1}'.format( 'succeeded' if bool(self) else 'failed', - self.message + self.message, ) def __repr__(self): diff --git a/core/auto_process/games.py b/core/auto_process/games.py index c412a690..410b82a3 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -46,7 +46,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', 'api_key': apikey, 'mode': 'UPDATEREQUESTEDSTATUS', 'db_id': gamez_id, - 'status': download_status + 'status': download_status, } logger.debug('Opening URL: {0}'.format(url), section) diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 4f10c64e..2f614ee1 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -73,7 +73,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': remote_dir(dir_name) if remote_path else dir_name + 'dir': remote_dir(dir_name) if remote_path else dir_name, } res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) @@ -83,7 +83,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0] + 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0], } res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) @@ -187,7 +187,7 @@ def get_status(url, apikey, dir_name): params = { 'apikey': apikey, - 'cmd': 'getHistory' + 'cmd': 'getHistory', } logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params)) diff --git a/core/databases.py b/core/databases.py index 07a78a4d..0803b9fc 100644 --- a/core/databases.py +++ b/core/databases.py @@ -33,7 +33,7 @@ class InitialSchema(main_db.SchemaUpgrade): queries = [ 'CREATE TABLE db_version (db_version INTEGER);', 'CREATE TABLE downloads (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));', - 'INSERT INTO db_version (db_version) VALUES (2);' + 'INSERT INTO db_version (db_version) VALUES (2);', ] for query in queries: self.connection.action(query) @@ -59,7 +59,7 @@ class InitialSchema(main_db.SchemaUpgrade): 'INSERT INTO downloads2 SELECT * FROM downloads;', 'DROP TABLE IF EXISTS downloads;', 'ALTER TABLE downloads2 RENAME TO downloads;', - 'INSERT INTO db_version (db_version) VALUES (2);' + 'INSERT INTO db_version (db_version) VALUES (2);', ] for query in queries: self.connection.action(query) diff --git a/core/forks.py b/core/forks.py index 897a1fd9..400d7e50 100644 --- a/core/forks.py +++ b/core/forks.py @@ -42,7 +42,7 @@ def auto_fork(section, input_category): logger.info('Attempting to verify {category} fork'.format (category=input_category)) url = '{protocol}{host}:{port}{root}/api/rootfolder'.format( - protocol=protocol, host=host, port=port, root=web_root + protocol=protocol, host=host, port=port, root=web_root, ) headers = {'X-Api-Key': apikey} try: @@ -66,11 +66,11 @@ def auto_fork(section, input_category): if apikey: url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess'.format( - protocol=protocol, host=host, port=port, root=web_root, apikey=apikey + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey, ) else: url = '{protocol}{host}:{port}{root}/home/postprocess/'.format( - protocol=protocol, host=host, port=port, root=web_root + protocol=protocol, host=host, port=port, root=web_root, ) # attempting to auto-detect fork diff --git a/core/logger.py b/core/logger.py index d73b501b..fcc306e2 100644 --- a/core/logger.py +++ b/core/logger.py @@ -85,7 +85,7 @@ class NTMRotatingLogHandler(object): console.setFormatter(DispatchingFormatter( {'nzbtomedia': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), 'postprocess': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), - 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S') + 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), }, logging.Formatter('%(message)s'), )) @@ -122,7 +122,7 @@ class NTMRotatingLogHandler(object): file_handler.setFormatter(DispatchingFormatter( {'nzbtomedia': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), 'postprocess': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), - 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S') + 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), }, logging.Formatter('%(message)s'), )) diff --git a/core/main_db.py b/core/main_db.py index 12c21c7e..241b3adf 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -183,9 +183,9 @@ class DBConnection(object): 'WHERE {conditions}'.format( table=table_name, params=', '.join(gen_params(value_dict)), - conditions=' AND '.join(gen_params(key_dict)) + conditions=' AND '.join(gen_params(key_dict)), ), - items + items, ) if self.connection.total_changes == changes_before: @@ -194,9 +194,9 @@ class DBConnection(object): 'VALUES ({values})'.format( table=table_name, columns=', '.join(map(text_type, value_dict.keys())), - values=', '.join(['?'] * len(value_dict.values())) + values=', '.join(['?'] * len(value_dict.values())), ), - list(value_dict.values()) + list(value_dict.values()), ) def table_info(self, table_name): diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py index ff92512c..61e84a7d 100644 --- a/core/plugins/downloaders/torrent/qbittorrent.py +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -14,7 +14,7 @@ def configure_client(): password = core.QBITTORRENT_PASSWORD logger.debug( - 'Connecting to {0}: http://{1}:{2}'.format(agent, host, port) + 'Connecting to {0}: http://{1}:{2}'.format(agent, host, port), ) client = qBittorrentClient('http://{0}:{1}/'.format(host, port)) try: diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 42473ab8..d62df0ba 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -25,7 +25,7 @@ media_list = [r'\.s\d{2}e\d{2}\.', r'\.1080[pi]\.', r'\.720p\.', r'\.576[pi]', r r'\.internal\.', r'\bac3\b', r'\.ntsc\.', r'\.pal\.', r'\.secam\.', r'\bdivx\b', r'\bxvid\b'] media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) garbage_name = re.compile(r'^[a-zA-Z0-9]*$') -char_replace = [[r'(\w)1\.(\w)', r'\1i\2'] +char_replace = [[r'(\w)1\.(\w)', r'\1i\2'], ] diff --git a/core/transcoder.py b/core/transcoder.py index edb98761..6dc9ce32 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -672,7 +672,7 @@ def rip_iso(item, new_dir, bitbucket): combined.extend(concat) continue name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, ) new_files.append({item: {'name': name, 'files': concat}}) if core.CONCAT: diff --git a/setup.py b/setup.py index fed3a4be..36e97020 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ from setuptools import setup def read(*names, **kwargs): with io.open( os.path.join(os.path.dirname(__file__), *names), - encoding=kwargs.get('encoding', 'utf8') + encoding=kwargs.get('encoding', 'utf8'), ) as fh: return fh.read() From c5244df5104c9c4daa9bb4418ab94319a8e177cb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:04:46 -0400 Subject: [PATCH 187/406] Fix flake8-commas C819 trailing comma prohibited --- core/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/logger.py b/core/logger.py index fcc306e2..f7fbf9c4 100644 --- a/core/logger.py +++ b/core/logger.py @@ -87,7 +87,7 @@ class NTMRotatingLogHandler(object): 'postprocess': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), 'db': logging.Formatter('[%(asctime)s] [%(levelname)s]::%(message)s', '%H:%M:%S'), }, - logging.Formatter('%(message)s'), )) + logging.Formatter('%(message)s'))) # add the handler to the root logger logging.getLogger('nzbtomedia').addHandler(console) @@ -124,7 +124,7 @@ class NTMRotatingLogHandler(object): 'postprocess': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), 'db': logging.Formatter('%(asctime)s %(levelname)-8s::%(message)s', '%Y-%m-%d %H:%M:%S'), }, - logging.Formatter('%(message)s'), )) + logging.Formatter('%(message)s'))) return file_handler From 2995c7f39171060b43069c11c01e0c9328b2749e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 19:04:11 -0400 Subject: [PATCH 188/406] Add flake8-quotes to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 74cf8e25..77e5ae63 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,7 @@ per-file-ignores = deps = flake8 flake8-commas + flake8-quotes skip_install = true commands = flake8 core tests setup.py From 94c42dbd8a89662b3891b368b1d269c004657882 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:21:26 -0400 Subject: [PATCH 189/406] Fix flake8-quotes Q000 Remove bad quotes --- core/__init__.py | 2 +- core/auto_process/tv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 6b4e40e9..85aeaf27 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -110,7 +110,7 @@ FORKS = { FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, - FORK_STHENO: {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None}, + FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index c4a68f59..3f8f27c5 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -266,7 +266,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu if apikey: url = '{0}{1}:{2}{3}/api/{4}/?cmd=postprocess'.format(protocol, host, port, web_root, apikey) elif fork == 'Stheno': - url = "{0}{1}:{2}{3}/home/postprocess/process_episode".format(protocol, host, port, web_root) + url = '{0}{1}:{2}{3}/home/postprocess/process_episode'.format(protocol, host, port, web_root) else: url = '{0}{1}:{2}{3}/home/postprocess/processEpisode'.format(protocol, host, port, web_root) elif section == 'NzbDrone': From f2964296c5b83c225a6e148bd940a9e02eb3c82a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 19:19:11 -0400 Subject: [PATCH 190/406] Add flake8-comprehensions to tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 77e5ae63..56d21272 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,7 @@ per-file-ignores = deps = flake8 flake8-commas + flake8-comprehensions flake8-quotes skip_install = true commands = From b9c7eec834e6a90402b68accf99137c9444ceea2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:34:46 -0400 Subject: [PATCH 191/406] Fix flake8-comprehensions C403 Unnecessary list comprehension --- core/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/transcoder.py b/core/transcoder.py index 6dc9ce32..d4b7e2f2 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -713,7 +713,7 @@ def combine_vts(vts_path): def combine_cd(combine): new_files = [] - for item in set([re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine]): + for item in {re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine}: concat = '' for n in range(99): files = [file for file in combine if From 169fcaae4a565765bcb70080d376ff4bef705817 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:35:23 -0400 Subject: [PATCH 192/406] Fix flake8-comprehensions C407 Unnecessary list comprehension --- core/utils/paths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/paths.py b/core/utils/paths.py index dba119f5..fe049ae7 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -67,10 +67,10 @@ def remote_dir(path): def get_dir_size(input_path): prepend = partial(os.path.join, input_path) - return sum([ + return sum( (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) for f in map(prepend, os.listdir(text_type(input_path))) - ]) + ) def remove_empty_folders(path, remove_root=True): From a8043d025948fd54d13dda0fb7e42e9cfa996e87 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 22:59:51 -0400 Subject: [PATCH 193/406] Add flake8-docstrings to tox.ini --- tox.ini | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tox.ini b/tox.ini index 56d21272..2111ddc6 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,22 @@ ignore = ; E501 line too long E501 +; -- flake8-docstrings -- +; D100 Missing docstring in public module +; D101 Missing docstring in public class +; D102 Missing docstring in public method +; D103 Missing docstring in public function +; D104 Missing docstring in public package +; D105 Missing docstring in magic method +; D107 Missing docstring in __init__ +; D200 One-line docstring should fit on one line with quotes +; D202 No blank lines allowed after function docstring +; D205 1 blank line required between summary line and description +; D400 First line should end with a period +; D401 First line should be in imperative mood +; D402 First line should not be the function's "signature" + D100, D101, D102, D103, D104, D105, D107 + per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file @@ -51,6 +67,7 @@ deps = flake8 flake8-commas flake8-comprehensions + flake8-docstrings flake8-quotes skip_install = true commands = From 4dd58afaf60ac4b8c9cc42a7b7459084ded60521 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:45:48 -0400 Subject: [PATCH 194/406] Fix flake8-docstrings D200 One-line docstring should fit on one line with quotes --- core/github_api.py | 8 ++------ core/logger.py | 4 +--- core/version_check.py | 8 ++------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/core/github_api.py b/core/github_api.py index 6e44f9f3..b6fb0856 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -4,9 +4,7 @@ import requests class GitHub(object): - """ - Simple api wrapper for the Github API v3. - """ + """Simple api wrapper for the Github API v3.""" def __init__(self, github_repo_user, github_repo, branch='master'): @@ -15,9 +13,7 @@ class GitHub(object): self.branch = branch def _access_api(self, path, params=None): - """ - Access the API at the path given and with the optional params given. - """ + """Access API at given an API path and optional parameters.""" url = 'https://api.github.com/{path}'.format(path='/'.join(path)) data = requests.get(url, params=params, verify=False) return data.json() if data.ok else [] diff --git a/core/logger.py b/core/logger.py index f7fbf9c4..701881cf 100644 --- a/core/logger.py +++ b/core/logger.py @@ -111,9 +111,7 @@ class NTMRotatingLogHandler(object): self.close_log(old_handler) def _config_handler(self): - """ - Configure a file handler to log at file_name and return it. - """ + """Configure a file handler to log at file_name and return it.""" file_handler = logging.FileHandler(self.log_file_path, encoding='utf-8') diff --git a/core/version_check.py b/core/version_check.py index d865b9f3..b39a8b08 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -19,9 +19,7 @@ from core import github_api as github, logger class CheckVersion(object): - """ - Version check class meant to run as a thread object with the SB scheduler. - """ + """Version checker that runs in a thread with the SB scheduler.""" def __init__(self): self.install_type = self.find_install_type() @@ -435,9 +433,7 @@ class SourceUpdateManager(UpdateManager): return def update(self): - """ - Downloads the latest source tarball from github and installs it over the existing version. - """ + """Download and install latest source tarball from github.""" tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format( org=self.github_repo_user, repo=self.github_repo, branch=self.branch) version_path = os.path.join(core.APP_ROOT, u'version.txt') From 777bc7e35d821cb26c89c2828d54c64aae805d5a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 14:48:25 -0400 Subject: [PATCH 195/406] Fix flake8-docstrings D202 No blank lines allowed after function docstring --- core/logger.py | 3 --- core/utils/naming.py | 2 -- core/version_check.py | 6 ------ 3 files changed, 11 deletions(-) diff --git a/core/logger.py b/core/logger.py index 701881cf..243e3bde 100644 --- a/core/logger.py +++ b/core/logger.py @@ -112,7 +112,6 @@ class NTMRotatingLogHandler(object): def _config_handler(self): """Configure a file handler to log at file_name and return it.""" - file_handler = logging.FileHandler(self.log_file_path, encoding='utf-8') file_handler.setLevel(DB) @@ -133,7 +132,6 @@ class NTMRotatingLogHandler(object): i: Log number to ues """ - return self.log_file_path + ('.{0}'.format(i) if i else '') def _num_logs(self): @@ -142,7 +140,6 @@ class NTMRotatingLogHandler(object): Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ - cur_log = 0 while os.path.isfile(self._log_file_name(cur_log)): cur_log += 1 diff --git a/core/utils/naming.py b/core/utils/naming.py index b54e3258..27ea8c02 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -12,7 +12,6 @@ def sanitize_name(name): >>> sanitize_name('.a.b..') 'a.b' """ - # remove bad chars from the filename name = re.sub(r'[\\/*]', '-', name) name = re.sub(r'[:\'<>|?]', '', name) @@ -34,7 +33,6 @@ def clean_file_name(filename): Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: """ - filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) diff --git a/core/version_check.py b/core/version_check.py index b39a8b08..a5c98abf 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -45,7 +45,6 @@ class CheckVersion(object): 'git': running from source using git 'source': running from source without git """ - # check if we're a windows build if os.path.isdir(os.path.join(core.APP_ROOT, u'.git')): install_type = 'git' @@ -62,7 +61,6 @@ class CheckVersion(object): force: if true the VERSION_NOTIFY setting will be ignored and a check will be forced """ - if not core.VERSION_NOTIFY and not force: logger.log(u'Version checking is disabled, not checking for the newest version') return False @@ -215,7 +213,6 @@ class GitUpdateManager(UpdateManager): Returns: True for success or False for failure """ - output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable if exit_status == 0 and output: @@ -245,7 +242,6 @@ class GitUpdateManager(UpdateManager): Uses git commands to check if there is a newer version that the provided commit hash. If there is a newer version it sets _num_commits_behind. """ - self._newest_commit_hash = None self._num_commits_behind = 0 self._num_commits_ahead = 0 @@ -325,7 +321,6 @@ class GitUpdateManager(UpdateManager): Calls git pull origin in order to update Sick Beard. Returns a bool depending on the call's success. """ - output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable if exit_status == 0: @@ -385,7 +380,6 @@ class SourceUpdateManager(UpdateManager): commit_hash: hash that we're checking against """ - self._num_commits_behind = 0 self._newest_commit_hash = None From 1d7dba8aebb9078e3c68aecbd6fd63bc3be94266 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:02:50 -0400 Subject: [PATCH 196/406] Fix flake8-docstrings D205 1 blank line required between summary line and description --- core/logger.py | 6 ++++-- core/main_db.py | 2 ++ core/utils/naming.py | 7 ++++--- core/version_check.py | 20 ++++++++++++++------ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/core/logger.py b/core/logger.py index 243e3bde..c569bfb3 100644 --- a/core/logger.py +++ b/core/logger.py @@ -127,8 +127,10 @@ class NTMRotatingLogHandler(object): def _log_file_name(self, i): """ - Returns a numbered log file name depending on i. If i==0 it just uses logName, if not it appends - it to the extension (blah.log.3 for i == 3) + Returns a numbered log file name depending on i. + + If i==0 it just uses logName, if not it appends it to the extension + e.g. (blah.log.3 for i == 3) i: Log number to ues """ diff --git a/core/main_db.py b/core/main_db.py index 241b3adf..14a75d21 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -14,6 +14,8 @@ from core import logger def db_filename(filename='nzbtomedia.db', suffix=None): """ + Return the correct location of the database file. + @param filename: The sqlite database filename to use. If not specified, will be made to be nzbtomedia.db @param suffix: The suffix to append to the filename. A '.' will be added diff --git a/core/utils/naming.py b/core/utils/naming.py index 27ea8c02..964fb6fd 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -3,6 +3,8 @@ import re def sanitize_name(name): """ + Remove bad chars from the filename. + >>> sanitize_name('a/b/c') 'a-b-c' >>> sanitize_name('abc') @@ -12,7 +14,6 @@ def sanitize_name(name): >>> sanitize_name('.a.b..') 'a.b' """ - # remove bad chars from the filename name = re.sub(r'[\\/*]', '-', name) name = re.sub(r'[:\'<>|?]', '', name) @@ -27,8 +28,8 @@ def sanitize_name(name): def clean_file_name(filename): - """Cleans up nzb name by removing any . and _ - characters, along with any trailing hyphens. + """ + Cleans up nzb name by removing any . and _ characters and trailing hyphens. Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: diff --git a/core/version_check.py b/core/version_check.py index a5c98abf..ac7fdd6d 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -239,8 +239,11 @@ class GitUpdateManager(UpdateManager): def _check_github_for_update(self): """ - Uses git commands to check if there is a newer version that the provided - commit hash. If there is a newer version it sets _num_commits_behind. + Check Github for a new version. + + Uses git commands to check if there is a newer version than + the provided commit hash. If there is a newer version it + sets _num_commits_behind. """ self._newest_commit_hash = None self._num_commits_behind = 0 @@ -318,8 +321,10 @@ class GitUpdateManager(UpdateManager): def update(self): """ - Calls git pull origin in order to update Sick Beard. Returns a bool depending - on the call's success. + Check git for a new version. + + Calls git pull origin in order to update Sick Beard. + Returns a bool depending on the call's success. """ output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable @@ -375,8 +380,11 @@ class SourceUpdateManager(UpdateManager): def _check_github_for_update(self): """ - Uses pygithub to ask github if there is a newer version that the provided - commit hash. If there is a newer version it sets Sick Beard's version text. + Check Github for a new version. + + Uses pygithub to ask github if there is a newer version than + the provided commit hash. If there is a newer version it sets + Sick Beard's version text. commit_hash: hash that we're checking against """ From 6f6c9bcc9d5732d543550b7c9eb57e0068564faa Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:16:06 -0400 Subject: [PATCH 197/406] Fix flake8-docstrings D400 First line should end with a period --- core/logger.py | 2 +- core/utils/files.py | 2 +- core/utils/paths.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/logger.py b/core/logger.py index c569bfb3..dba98947 100644 --- a/core/logger.py +++ b/core/logger.py @@ -138,7 +138,7 @@ class NTMRotatingLogHandler(object): def _num_logs(self): """ - Scans the log folder and figures out how many log files there are already on disk + Scans the log folder and figures out how many log files there are already on disk. Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ diff --git a/core/utils/files.py b/core/utils/files.py index 895125e1..edb968ae 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -88,7 +88,7 @@ def is_min_size(input_name, min_size): def is_archive_file(filename): - """Check if the filename is allowed for the Archive""" + """Check if the filename is allowed for the Archive.""" for regext in core.COMPRESSED_CONTAINER: if regext.search(filename): return regext.split(filename)[0] diff --git a/core/utils/paths.py b/core/utils/paths.py index fe049ae7..cdb2c3ff 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -74,7 +74,7 @@ def get_dir_size(input_path): def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders""" + """Function to remove empty folders.""" if not os.path.isdir(path): return From 267d8d16322f168b01464a373d18e1e0c5247af2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 23:04:04 -0400 Subject: [PATCH 198/406] Add flake8-bugbear to tox.ini --- tox.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tox.ini b/tox.ini index 2111ddc6..c614d8d3 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,12 @@ commands = max-line-length = 79 verbose = 2 statistics = True +select = +; -- flake8-bugbear -- +; B902 Invalid first argument used for instance method. +; B903 Data class should either be immutable or use __slots__ to save memory. + B902, B903 + ignore = ; -- flake8 -- ; E501 line too long @@ -65,6 +71,7 @@ per-file-ignores = [testenv:check] deps = flake8 + flake8-bugbear flake8-commas flake8-comprehensions flake8-docstrings From e00b5cc1952808d87c8647f1216ac1f1215f4e7a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:43:56 -0400 Subject: [PATCH 199/406] Fix flake8-bugbear B010 Do not call setattr with a constant attribute value, it is not any safer than normal property access. --- core/logger.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/logger.py b/core/logger.py index df5a5120..bdb7e683 100644 --- a/core/logger.py +++ b/core/logger.py @@ -6,6 +6,7 @@ import sys import threading import core +import functools # number of log files to keep NUM_LOGS = 3 @@ -199,9 +200,8 @@ class NTMRotatingLogHandler(object): ntm_logger = logging.getLogger('nzbtomedia') pp_logger = logging.getLogger('postprocess') db_logger = logging.getLogger('db') - setattr(pp_logger, 'postprocess', lambda *args: pp_logger.log(POSTPROCESS, *args)) - setattr(db_logger, 'db', lambda *args: db_logger.log(DB, *args)) - + pp_logger.postprocess = functools.partial(pp_logger.log, POSTPROCESS) + db_logger.db = functools.partial(db_logger.log, DB) try: if log_level == DEBUG: if core.LOG_DEBUG == 1: From 4c8e896bbb9843232f949a2d4ff351e9325e4e10 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 16:02:23 -0400 Subject: [PATCH 200/406] Fix flake8-bugbear B007 Loop control variable not used within the loop body. --- core/auto_process/movies.py | 2 +- core/configuration.py | 4 ++-- core/scene_exceptions.py | 2 +- core/transcoder.py | 2 +- core/user_scripts.py | 4 ++-- core/utils/encoding.py | 4 ++-- core/version_check.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 76d49b78..4d2a4b2f 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -504,7 +504,7 @@ def get_release(base_url, imdb_id=None, download_id=None, release_id=None): # Narrow results by removing old releases by comparing their last_edit field if len(results) > 1: for id1, x1 in results.items(): - for id2, x2 in results.items(): + for x2 in results.values(): try: if x2['last_edit'] > x1['last_edit']: results.pop(id1) diff --git a/core/configuration.py b/core/configuration.py index 216a1f05..387f9b72 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -136,10 +136,10 @@ class ConfigObj(configobj.ConfigObj, Section): subsections = {} # gather all new-style and old-style sub-sections - for newsection, newitems in CFG_NEW.items(): + for newsection in CFG_NEW: if CFG_NEW[newsection].sections: subsections.update({newsection: CFG_NEW[newsection].sections}) - for section, items in CFG_OLD.items(): + for section in CFG_OLD: if CFG_OLD[section].sections: subsections.update({section: CFG_OLD[section].sections}) for option, value in CFG_OLD[section].items(): diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index d62df0ba..0b70c52c 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -121,7 +121,7 @@ def reverse_filename(filename, dirname, name): def rename_script(dirname): rename_file = '' - for directory, directories, files in os.walk(dirname): + for directory, _, files in os.walk(dirname): for file in files: if re.search(r'(rename\S*\.(sh|bat)$)', file, re.IGNORECASE): rename_file = os.path.join(directory, file) diff --git a/core/transcoder.py b/core/transcoder.py index d4b7e2f2..f0047dad 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -519,7 +519,7 @@ def get_subs(file): sub_ext = ['.srt', '.sub', '.idx'] name = os.path.splitext(os.path.split(file)[1])[0] path = os.path.split(file)[0] - for directory, directories, filenames in os.walk(path): + for directory, _, filenames in os.walk(path): for filename in filenames: filepaths.extend([os.path.join(directory, filename)]) subfiles = [item for item in filepaths if os.path.splitext(item)[1] in sub_ext and name in item] diff --git a/core/user_scripts.py b/core/user_scripts.py index 5d447670..fbc58614 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -47,7 +47,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): logger.info('Corrupt video file found {0}. Deleting.'.format(video), 'USERSCRIPT') os.unlink(video) - for dirpath, dirnames, filenames in os.walk(output_destination): + for dirpath, _, filenames in os.walk(output_destination): for file in filenames: file_path = core.os.path.join(dirpath, file) @@ -102,7 +102,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): final_result += result num_files_new = 0 - for dirpath, dirnames, filenames in os.walk(output_destination): + for _, _, filenames in os.walk(output_destination): for file in filenames: file_name, file_extension = os.path.splitext(file) diff --git a/core/utils/encoding.py b/core/utils/encoding.py index ca19e054..cbcc3113 100644 --- a/core/utils/encoding.py +++ b/core/utils/encoding.py @@ -68,14 +68,14 @@ def convert_to_ascii(input_name, dir_name): if 'NZBOP_SCRIPTDIR' in os.environ: print('[NZB] DIRECTORY={0}'.format(dir_name)) - for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): + for dirname, dirnames, _ in os.walk(dir_name, topdown=False): for subdirname in dirnames: encoded, subdirname2 = char_replace(subdirname) if encoded: logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) - for dirname, dirnames, filenames in os.walk(dir_name): + for dirname, _, filenames in os.walk(dir_name): for filename in filenames: encoded, filename2 = char_replace(filename) if encoded: diff --git a/core/version_check.py b/core/version_check.py index 23384a44..e7dc5578 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -487,7 +487,7 @@ class SourceUpdateManager(UpdateManager): # walk temp folder and move files to main folder logger.log(u'Moving files from {source} to {destination}'.format (source=content_dir, destination=core.APP_ROOT)) - for dirname, dirnames, filenames in os.walk(content_dir): # @UnusedVariable + for dirname, _, filenames in os.walk(content_dir): # @UnusedVariable dirname = dirname[len(content_dir) + 1:] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) From 10b2eab3c571b42fef29b3f5531a23375f980075 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 15:20:23 -0400 Subject: [PATCH 201/406] Fix flake8-docstrings D401 First line should be in imperative mood --- core/github_api.py | 4 ++-- core/logger.py | 4 ++-- core/utils/naming.py | 2 +- core/utils/paths.py | 2 +- core/version_check.py | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/github_api.py b/core/github_api.py index b6fb0856..b1a4f566 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -20,7 +20,7 @@ class GitHub(object): def commits(self): """ - Uses the API to get a list of the 100 most recent commits from the specified user/repo/branch, starting from HEAD. + Get the 100 most recent commits from the specified user/repo/branch, starting from HEAD. user: The github username of the person whose repo you're querying repo: The repo name to query @@ -35,7 +35,7 @@ class GitHub(object): def compare(self, base, head, per_page=1): """ - Uses the API to get a list of compares between base and head. + Get compares between base and head. user: The github username of the person whose repo you're querying repo: The repo name to query diff --git a/core/logger.py b/core/logger.py index dba98947..df5a5120 100644 --- a/core/logger.py +++ b/core/logger.py @@ -127,7 +127,7 @@ class NTMRotatingLogHandler(object): def _log_file_name(self, i): """ - Returns a numbered log file name depending on i. + Return a numbered log file name depending on i. If i==0 it just uses logName, if not it appends it to the extension e.g. (blah.log.3 for i == 3) @@ -138,7 +138,7 @@ class NTMRotatingLogHandler(object): def _num_logs(self): """ - Scans the log folder and figures out how many log files there are already on disk. + Scan the log folder and figure out how many log files there are already on disk. Returns: The number of the last used file (eg. mylog.log.3 would return 3). If there are no logs it returns -1 """ diff --git a/core/utils/naming.py b/core/utils/naming.py index 964fb6fd..64cd6d70 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -29,7 +29,7 @@ def sanitize_name(name): def clean_file_name(filename): """ - Cleans up nzb name by removing any . and _ characters and trailing hyphens. + Clean up nzb name by removing any . and _ characters and trailing hyphens. Is basically equivalent to replacing all _ and . with a space, but handles decimal numbers in string, for example: diff --git a/core/utils/paths.py b/core/utils/paths.py index cdb2c3ff..a2a96996 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -74,7 +74,7 @@ def get_dir_size(input_path): def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders.""" + """Remove empty folders.""" if not os.path.isdir(path): return diff --git a/core/version_check.py b/core/version_check.py index ac7fdd6d..23384a44 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -38,7 +38,7 @@ class CheckVersion(object): def find_install_type(self): """ - Determines how this copy of SB was installed. + Determine how this copy of SB was installed. returns: type of installation. Possible values are: 'win': any compiled windows build @@ -55,7 +55,7 @@ class CheckVersion(object): def check_for_new_version(self, force=False): """ - Checks the internet for a newer version. + Check the internet for a newer version. returns: bool, True for new version or False for no new version. @@ -207,7 +207,7 @@ class GitUpdateManager(UpdateManager): def _find_installed_version(self): """ - Attempts to find the currently installed version of Sick Beard. + Attempt to find the currently installed version of Sick Beard. Uses git show to get commit version. From 72140e939ced3b85bcdc7d52f1d0357849ff307b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 5 Apr 2019 16:15:21 -0400 Subject: [PATCH 202/406] Fix flake8-bugbear B902 Invalid first argument used for instance method. --- core/configuration.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/configuration.py b/core/configuration.py index 387f9b72..2d34c8bc 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -13,17 +13,17 @@ from core import logger class Section(configobj.Section, object): - def isenabled(section): + def isenabled(self): # checks if subsection enabled, returns true/false if subsection specified otherwise returns true/false in {} - if not section.sections: + if not self.sections: try: - value = list(ConfigObj.find_key(section, 'enabled'))[0] + value = list(ConfigObj.find_key(self, 'enabled'))[0] except Exception: value = 0 if int(value) == 1: - return section + return self else: - to_return = copy.deepcopy(section) + to_return = copy.deepcopy(self) for section_name, subsections in to_return.items(): for subsection in subsections: try: @@ -40,8 +40,8 @@ class Section(configobj.Section, object): return to_return - def findsection(section, key): - to_return = copy.deepcopy(section) + def findsection(self, key): + to_return = copy.deepcopy(self) for subsection in to_return: try: value = list(ConfigObj.find_key(to_return[subsection], key))[0] From 0788a754cbd9d66c3be95e0e74edd06dff9d93cc Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:15:07 -0400 Subject: [PATCH 203/406] Fix code quality checks to run all desired tests Fixes #1602 --- tox.ini | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index c614d8d3..cf22519c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,12 +33,6 @@ commands = max-line-length = 79 verbose = 2 statistics = True -select = -; -- flake8-bugbear -- -; B902 Invalid first argument used for instance method. -; B903 Data class should either be immutable or use __slots__ to save memory. - B902, B903 - ignore = ; -- flake8 -- ; E501 line too long @@ -78,7 +72,15 @@ deps = flake8-quotes skip_install = true commands = +; ** PRIMARY TESTS ** +; Run flake8 tests (with plugins) using default test selections flake8 core tests setup.py +; ** SELECTIVE TESTS ** +; Run flake8 tests (with plugins) for specific optional codes defined below +; -- flake8-bugbear -- +; B902 Invalid first argument used for instance method. +; B903 Data class should be immutable or use __slots__ to save memory. + flake8 core tests setup.py --select=B902,B903 [coverage:run] omit = From e7179dde1c8115233edadb36dc61776ea793d79a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:38:43 -0400 Subject: [PATCH 204/406] Temporarily disable some flake8 ignores for testing --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf22519c..089c95f8 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ statistics = True ignore = ; -- flake8 -- ; E501 line too long - E501 +; E501 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -57,7 +57,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - core/__init__.py: E402, F401 +; core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 From 28ff74d0c8e8ce585e1452b58e84f745e5737fe4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:42:18 -0400 Subject: [PATCH 205/406] Revert "Temporarily disable some flake8 ignores for testing" This reverts commit e7179dde1c8115233edadb36dc61776ea793d79a. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 089c95f8..cf22519c 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ statistics = True ignore = ; -- flake8 -- ; E501 line too long -; E501 + E501 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -57,7 +57,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file -; core/__init__.py: E402, F401 + core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 From 8736642e78929be4ff00a3ef3c5189b83f4d1b11 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:19:35 -0400 Subject: [PATCH 206/406] Fix code quality checks to run on project root and custom libs Fixes #1600 Fixes #1601 --- tox.ini | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf22519c..e9a5bcc1 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,15 @@ commands = max-line-length = 79 verbose = 2 statistics = True +exclude = + .github/ + .tox/ + .pytest_cache/ + htmlcov/ + logs/ + libs/common + libs/win + libs/py2 ignore = ; -- flake8 -- ; E501 line too long @@ -74,13 +83,13 @@ skip_install = true commands = ; ** PRIMARY TESTS ** ; Run flake8 tests (with plugins) using default test selections - flake8 core tests setup.py + flake8 ; ** SELECTIVE TESTS ** ; Run flake8 tests (with plugins) for specific optional codes defined below ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. - flake8 core tests setup.py --select=B902,B903 + flake8 --select=B902,B903 [coverage:run] omit = From 077f04bc53aadcc193c8032d0e15ef57357f725e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:56:50 -0400 Subject: [PATCH 207/406] Fix flake8 E265 block comment should start with '# ' Ignore for NZBGET scripts --- TorrentToMedia.py | 24 ++++++++++++------------ tox.ini | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 06d36e1a..dd7c363a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -60,13 +60,13 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass - #try: + # try: # input_directory = input_directory.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format @@ -125,9 +125,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - #try: + # try: # output_destination = output_destination.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass if output_destination in input_directory: @@ -170,9 +170,9 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - #try: + # try: # target_file = target_file.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass if root == 1: if not found_file: @@ -353,14 +353,14 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - #try: + # try: # dir_name = dir_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass input_name = os.path.basename(dir_name) - #try: + # try: # input_name = input_name.encode(core.SYS_ENCODING) - #except UnicodeError: + # except UnicodeError: # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, diff --git a/tox.ini b/tox.ini index e9a5bcc1..f9ca222c 100644 --- a/tox.ini +++ b/tox.ini @@ -66,6 +66,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file + nzbTo*.py: E265 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From 9dd25f96b2485794d0c8e66cd3b9c4cf5b62b514 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 12:58:31 -0400 Subject: [PATCH 208/406] Fix flake8 E266 too many leading '#' for block comment Ignore for NZBGET scripts --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f9ca222c..97f17b47 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - nzbTo*.py: E265 + nzbTo*.py: E265, E266 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From daa9819798b4747383d06e6f3a6172a989eabdac Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:06:25 -0400 Subject: [PATCH 209/406] Fix flake8 F401 item imported but unused --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 97f17b47..2230329e 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,7 @@ per-file-ignores = core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 core/plugins/downloaders/utils.py: F401 + libs/custom/synchronousdeluge/__init__.py: F401 [testenv:check] deps = From 98e8fd581ae9eff7d634a466b9961cece524376a Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:08:31 -0400 Subject: [PATCH 210/406] Fix flake8 E303 too many blank lines --- TorrentToMedia.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index dd7c363a..555b101d 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -259,7 +259,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp elif section_name == 'LazyLibrarian': result = books.process(section_name, output_destination, input_name, status, client_agent, input_category) - plex_update(input_category) if result.status_code != 0: From 9527a2bd677c400d4b5259db1c5625a469f20afe Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:16:35 -0400 Subject: [PATCH 211/406] Fix flake8 E402 module level import not at top of file --- TorrentToMedia.py | 17 ++++++++++------- nzbToMedia.py | 12 ++++++------ tox.ini | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 555b101d..fcd3bac5 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -1,16 +1,15 @@ #!/usr/bin/env python # coding=utf-8 -import eol -eol.check() - -import cleanup -cleanup.clean(cleanup.FOLDER_STRUCTURE) - import datetime import os import sys +import eol +import cleanup +eol.check() +cleanup.clean(cleanup.FOLDER_STRUCTURE) + import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv, books @@ -18,7 +17,11 @@ from core.auto_process.common import ProcessResult from core.plugins.plex import plex_update from core.user_scripts import external_script from core.utils import char_replace, convert_to_ascii, replace_links -from six import text_type + +try: + text_type = unicode +except NameError: + text_type = str def process_torrent(input_directory, input_name, input_category, input_hash, input_id, client_agent): diff --git a/nzbToMedia.py b/nzbToMedia.py index 8d969d6a..4f43329c 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -657,16 +657,16 @@ from __future__ import print_function -import eol -eol.check() - -import cleanup -cleanup.clean(cleanup.FOLDER_STRUCTURE) - import datetime import os import sys +import eol +import cleanup + +eol.check() +cleanup.clean(cleanup.FOLDER_STRUCTURE) + import core from core import logger, main_db from core.auto_process import comics, games, movies, music, tv, books diff --git a/tox.ini b/tox.ini index 2230329e..fd7a997b 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,8 @@ ignore = per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file - nzbTo*.py: E265, E266 + nzbTo*.py: E265, E266, E402 + TorrentToMedia.py: E402 core/__init__.py: E402, F401 core/utils/__init__.py: F401 core/plugins/downloaders/configuration.py: F401 From 90602bf154aa20355466f2bc7d4bcb3d7d866a56 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:17:55 -0400 Subject: [PATCH 212/406] Fix flake8 W293 blank line contains whitespace --- libs/custom/synchronousdeluge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index fbafe6fe..e658debc 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -1,7 +1,7 @@ # coding=utf-8 """A synchronous implementation of the Deluge RPC protocol based on gevent-deluge by Christopher Rosell. - + https://github.com/chrippa/gevent-deluge Example usage: From 181675722d7dad38bb08751c896de8ac75253e9e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:23:24 -0400 Subject: [PATCH 213/406] Fix flake8 W291 trailing whitespace --- libs/custom/utorrent/upload.py | 2 +- nzbToCouchPotato.py | 8 ++++---- nzbToHeadPhones.py | 2 +- nzbToLidarr.py | 14 +++++++------- nzbToMedia.py | 8 ++++---- nzbToMylar.py | 2 +- nzbToNzbDrone.py | 14 +++++++------- nzbToRadarr.py | 14 +++++++------- nzbToSickBeard.py | 8 ++++---- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/libs/custom/utorrent/upload.py b/libs/custom/utorrent/upload.py index 03d58bcc..1ccbcd37 100644 --- a/libs/custom/utorrent/upload.py +++ b/libs/custom/utorrent/upload.py @@ -36,7 +36,7 @@ class MultiPartForm(object): # Build a list of lists, each containing "lines" of the # request. Each part is separated by a boundary string. # Once the list is built, return a string where each - # line is separated by '\r\n'. + # line is separated by '\r\n'. parts = [] part_boundary = '--' + self.boundary diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index be7fb4f3..9869b0e1 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -99,7 +99,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -134,7 +134,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -216,7 +216,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -227,7 +227,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index 34103b0c..34fac2e3 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -101,7 +101,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## WakeOnLan diff --git a/nzbToLidarr.py b/nzbToLidarr.py index d8a20dc8..230d8b9d 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -84,7 +84,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -119,7 +119,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -145,7 +145,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -201,20 +201,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToMedia.py b/nzbToMedia.py index 4f43329c..1d479d95 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -450,7 +450,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -485,7 +485,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -567,7 +567,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -578,7 +578,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame diff --git a/nzbToMylar.py b/nzbToMylar.py index 8fe166e3..d16b357a 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -92,7 +92,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## WakeOnLan diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index b7d0ac33..df551472 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -89,7 +89,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -124,7 +124,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -150,7 +150,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -206,20 +206,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToRadarr.py b/nzbToRadarr.py index 446585bc..fa2aa26e 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -94,7 +94,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -129,7 +129,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages = eng,spa,fra # Transcode (0, 1). @@ -155,7 +155,7 @@ # outputVideoPath. # # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. -#outputVideoPath = +#outputVideoPath = # processOutput (0,1). # @@ -211,20 +211,20 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow = +#VideoCodecAllow = #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 #outputVideoBitrate=800k #outputAudioCodec=libmp3lame -#AudioCodecAllow = +#AudioCodecAllow = #outputAudioBitrate=128k #outputQualityPercent = 0 #outputAudioTrack2Codec = libfaac -#AudioCodec2Allow = +#AudioCodec2Allow = #outputAudioTrack2Bitrate = 128k #outputAudioOtherCodec = libmp3lame -#AudioOtherCodecAllow = +#AudioOtherCodecAllow = #outputAudioOtherBitrate = 128k #outputSubtitleCodec = diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 3f1b300d..d342cf0d 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -100,7 +100,7 @@ # # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ -#mountPoints= +#mountPoints= ## Extensions @@ -135,7 +135,7 @@ # subLanguages. # -# subLanguages. create a list of languages in the order you want them in your subtitles. +# subLanguages. create a list of languages in the order you want them in your subtitles. #subLanguages=eng,spa,fra # Transcode (0, 1). @@ -217,7 +217,7 @@ # ffmpeg output settings. #outputVideoExtension=.mp4 #outputVideoCodec=libx264 -#VideoCodecAllow= +#VideoCodecAllow= #outputVideoResolution=720:-1 #outputVideoPreset=medium #outputVideoFramerate=24 @@ -228,7 +228,7 @@ #outputAudioBitrate=640k #outputQualityPercent= #outputAudioTrack2Codec=libfaac -#AudioCodec2Allow= +#AudioCodec2Allow= #outputAudioTrack2Channels=2 #outputAudioTrack2Bitrate=160k #outputAudioOtherCodec=libmp3lame From 70fa47394ee3abffdf0ce0985d358550737886d7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:26:13 -0400 Subject: [PATCH 214/406] Fix flake8-docstrings D202 No blank lines allowed after function docstring --- libs/custom/synchronousdeluge/client.py | 1 - libs/custom/utorrent/client.py | 1 - 2 files changed, 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index aa180bf7..ec519687 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -141,7 +141,6 @@ class DelugeClient(object): :param username: str, the username to login with :param password: str, the password to login with """ - # Connect transport self.transfer.connect((host, port)) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 2be51c6d..613d94a7 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -32,7 +32,6 @@ class UTorrentClient(object): def _make_opener(self, realm, base_url, username, password): """uTorrent API need HTTP Basic Auth and cookie support for token verify.""" - auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm=realm, uri=base_url, From f98b39cdbb0d5bdaa2f49651e21ccc446f155f00 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:27:31 -0400 Subject: [PATCH 215/406] Fix flake8-docstrings D204 1 blank line required after class docstring --- cleanup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cleanup.py b/cleanup.py index d313f2b4..b4cf650e 100644 --- a/cleanup.py +++ b/cleanup.py @@ -25,6 +25,7 @@ FOLDER_STRUCTURE = { class WorkingDirectory(object): """Context manager for changing current working directory.""" + def __init__(self, new, original=None): self.working_directory = new self.original_directory = os.getcwd() if original is None else original From 73e47466b4b86ffc19ec117da2184b40c99419eb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:30:40 -0400 Subject: [PATCH 216/406] Fix flake8-docstrings D205 1 blank line required between summary line and description --- libs/custom/synchronousdeluge/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index e658debc..b0283e83 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -1,7 +1,8 @@ # coding=utf-8 -"""A synchronous implementation of the Deluge RPC protocol - based on gevent-deluge by Christopher Rosell. +""" +A synchronous implementation of the Deluge RPC protocol. +Based on gevent-deluge by Christopher Rosell: https://github.com/chrippa/gevent-deluge Example usage: From 093f49d5aab84d0319dde02cbeb47887329f6af2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:32:06 -0400 Subject: [PATCH 217/406] Fix flake8-docstrings D401 First line should be in imperative mood --- libs/custom/synchronousdeluge/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index ec519687..817aeba1 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -18,7 +18,7 @@ RPC_EVENT = 3 class DelugeClient(object): def __init__(self): - """A deluge client session.""" + """Create a deluge client session.""" self.transfer = DelugeTransfer() self.modules = [] self._request_counter = 0 @@ -134,7 +134,7 @@ class DelugeClient(object): return response def connect(self, host="127.0.0.1", port=58846, username="", password=""): - """Connects to a daemon process. + """Connect to a daemon process. :param host: str, the hostname of the daemon :param port: int, the port of the daemon From eec977d9096becd66b58afb257a165ad79f8d30e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:33:20 -0400 Subject: [PATCH 218/406] Fix flake8-docstrings D403 First word of the first line should be properly capitalized --- libs/custom/utorrent/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 613d94a7..fe7e53da 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -31,7 +31,7 @@ class UTorrentClient(object): # TODO refresh token, when necessary def _make_opener(self, realm, base_url, username, password): - """uTorrent API need HTTP Basic Auth and cookie support for token verify.""" + """HTTP Basic Auth and cookie support for token verification.""" auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm=realm, uri=base_url, From 81c50efcd6b22651a2b8109a9d1b9367964d77b7 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:37:17 -0400 Subject: [PATCH 219/406] Fix flake8-commas C813 missing trailing comma in Python 3 --- cleanup.py | 2 +- eol.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cleanup.py b/cleanup.py index b4cf650e..f0d4e6d1 100644 --- a/cleanup.py +++ b/cleanup.py @@ -44,7 +44,7 @@ class WorkingDirectory(object): original_directory=self.original_directory, error=error, working_directory=self.working_directory, - ) + ), ) diff --git a/eol.py b/eol.py index a67bfc9e..5504df7e 100644 --- a/eol.py +++ b/eol.py @@ -157,7 +157,7 @@ def print_statuses(show_expired=False): major=python_version[0], minor=python_version[1], remaining=days_left, - ) + ), ) if not show_expired: return @@ -171,7 +171,7 @@ def print_statuses(show_expired=False): major=python_version[0], minor=python_version[1], remaining=-days_left, - ) + ), ) From d608000345f8d6f8a899ca394e0865c8e514bb91 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:38:27 -0400 Subject: [PATCH 220/406] Fix flake8-commas C819 trailing comma prohibited --- libs/custom/utorrent/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index fe7e53da..5d3e1fec 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -60,25 +60,25 @@ class UTorrentClient(object): return self._action(params) def start(self, *hashes): - params = [('action', 'start'), ] + params = [('action', 'start')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def stop(self, *hashes): - params = [('action', 'stop'), ] + params = [('action', 'stop')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def pause(self, *hashes): - params = [('action', 'pause'), ] + params = [('action', 'pause')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def forcestart(self, *hashes): - params = [('action', 'forcestart'), ] + params = [('action', 'forcestart')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) @@ -124,13 +124,13 @@ class UTorrentClient(object): self._action(params) def remove(self, *hashes): - params = [('action', 'remove'), ] + params = [('action', 'remove')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) def removedata(self, *hashes): - params = [('action', 'removedata'), ] + params = [('action', 'removedata')] for cur_hash in hashes: params.append(('hash', cur_hash)) return self._action(params) From 99159acd8033a22dc518479cc7aaf31c480ce840 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:39:48 -0400 Subject: [PATCH 221/406] Fix flake8-bugbear B007 Loop control variable not used within the loop body. --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index fcd3bac5..e3665c5e 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -281,7 +281,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # remove torrent if core.USE_LINK == 'move-sym' and not core.DELETE_ORIGINAL == 1: logger.debug('Checking for sym-links to re-direct in: {0}'.format(input_directory)) - for dirpath, dirs, files in os.walk(input_directory): + for dirpath, _, files in os.walk(input_directory): for file in files: logger.debug('Checking symlink: {0}'.format(os.path.join(dirpath, file))) replace_links(os.path.join(dirpath, file)) From 9f52406d45c4a9830190f87b13ea95ef6baf411c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 13:44:33 -0400 Subject: [PATCH 222/406] Fix flake8-quotes Q000 Remove bad quotes --- libs/custom/synchronousdeluge/__init__.py | 6 ++-- libs/custom/synchronousdeluge/client.py | 36 ++++++++++----------- libs/custom/synchronousdeluge/exceptions.py | 2 +- libs/custom/synchronousdeluge/transfer.py | 4 +-- libs/custom/utorrent/client.py | 4 +-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index b0283e83..ad90cfe8 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -19,6 +19,6 @@ Example usage: from .exceptions import DelugeRPCError -__title__ = "synchronous-deluge" -__version__ = "0.1" -__author__ = "Christian Dale" +__title__ = 'synchronous-deluge' +__version__ = '0.1' +__author__ = 'Christian Dale' diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 817aeba1..9194ebe8 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -9,7 +9,7 @@ from .exceptions import DelugeRPCError from .protocol import DelugeRPCRequest, DelugeRPCResponse from .transfer import DelugeTransfer -__all__ = ["DelugeClient"] +__all__ = ['DelugeClient'] RPC_RESPONSE = 1 RPC_ERROR = 2 @@ -24,35 +24,35 @@ class DelugeClient(object): self._request_counter = 0 def _get_local_auth(self): - username = password = "" + username = password = '' if platform.system() in ('Windows', 'Microsoft'): - app_data_path = os.environ.get("APPDATA") + app_data_path = os.environ.get('APPDATA') if not app_data_path: from six.moves import winreg hkey = winreg.OpenKey( winreg.HKEY_CURRENT_USER, - "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders", + 'Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders', ) - app_data_reg = winreg.QueryValueEx(hkey, "AppData") + app_data_reg = winreg.QueryValueEx(hkey, 'AppData') app_data_path = app_data_reg[0] winreg.CloseKey(hkey) - auth_file = os.path.join(app_data_path, "deluge", "auth") + auth_file = os.path.join(app_data_path, 'deluge', 'auth') else: from xdg.BaseDirectory import save_config_path try: - auth_file = os.path.join(save_config_path("deluge"), "auth") + auth_file = os.path.join(save_config_path('deluge'), 'auth') except OSError: return username, password if os.path.exists(auth_file): for line in open(auth_file): - if line.startswith("#"): + if line.startswith('#'): # This is a comment line continue line = line.strip() try: - lsplit = line.split(":") + lsplit = line.split(':') except Exception: continue @@ -63,13 +63,13 @@ class DelugeClient(object): else: continue - if username == "localclient": + if username == 'localclient': return username, password - return "", "" + return '', '' def _create_module_method(self, module, method): - fullname = "{0}.{1}".format(module, method) + fullname = '{0}.{1}'.format(module, method) def func(obj, *args, **kwargs): return self.remote_call(fullname, *args, **kwargs) @@ -80,18 +80,18 @@ class DelugeClient(object): def _introspect(self): def splitter(value): - return value.split(".") + return value.split('.') self.modules = [] - methods = self.remote_call("daemon.get_method_list").get() + methods = self.remote_call('daemon.get_method_list').get() methodmap = defaultdict(dict) for module, method in imap(splitter, methods): methodmap[module][method] = self._create_module_method(module, method) for module, methods in methodmap.items(): - clsname = "DelugeModule{0}".format(module.capitalize()) + clsname = 'DelugeModule{0}'.format(module.capitalize()) cls = type(clsname, (), methods) setattr(self, module, cls()) self.modules.append(module) @@ -133,7 +133,7 @@ class DelugeClient(object): self._request_counter += 1 return response - def connect(self, host="127.0.0.1", port=58846, username="", password=""): + def connect(self, host='127.0.0.1', port=58846, username='', password=''): """Connect to a daemon process. :param host: str, the hostname of the daemon @@ -145,11 +145,11 @@ class DelugeClient(object): self.transfer.connect((host, port)) # Attempt to fetch local auth info if needed - if not username and host in ("127.0.0.1", "localhost"): + if not username and host in ('127.0.0.1', 'localhost'): username, password = self._get_local_auth() # Authenticate - self.remote_call("daemon.login", username, password).get() + self.remote_call('daemon.login', username, password).get() # Introspect available methods self._introspect() diff --git a/libs/custom/synchronousdeluge/exceptions.py b/libs/custom/synchronousdeluge/exceptions.py index 6da4bdde..dcb23009 100644 --- a/libs/custom/synchronousdeluge/exceptions.py +++ b/libs/custom/synchronousdeluge/exceptions.py @@ -8,4 +8,4 @@ class DelugeRPCError(Exception): self.traceback = traceback def __str__(self): - return "{0}: {1}: {2}".format(self.__class__.__name__, self.name, self.msg) + return '{0}: {1}: {2}'.format(self.__class__.__name__, self.name, self.msg) diff --git a/libs/custom/synchronousdeluge/transfer.py b/libs/custom/synchronousdeluge/transfer.py index 42863b83..cbf49100 100644 --- a/libs/custom/synchronousdeluge/transfer.py +++ b/libs/custom/synchronousdeluge/transfer.py @@ -6,7 +6,7 @@ import zlib import rencode -__all__ = ["DelugeTransfer"] +__all__ = ['DelugeTransfer'] class DelugeTransfer(object): @@ -33,7 +33,7 @@ class DelugeTransfer(object): payload = zlib.compress(rencode.dumps(data)) self.conn.sendall(payload) - buf = b"" + buf = b'' while True: data = self.conn.recv(1024) diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index 5d3e1fec..ee9ef983 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -94,8 +94,8 @@ class UTorrentClient(object): def setprops(self, cur_hash, **kvpairs): params = [('action', 'setprops'), ('hash', cur_hash)] for k, v in iteritems(kvpairs): - params.append(("s", k)) - params.append(("v", v)) + params.append(('s', k)) + params.append(('v', v)) return self._action(params) From c6e35bd2db42fb9e3e9914602fb85da391d56ceb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 14:20:20 -0400 Subject: [PATCH 223/406] Add optional flake8 tests to selective testing Ignore W505 (doc string length) for now --- tox.ini | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fd7a997b..b1074483 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,7 @@ commands = [flake8] max-line-length = 79 +max-doc-length = 79 verbose = 2 statistics = True exclude = @@ -45,7 +46,8 @@ exclude = ignore = ; -- flake8 -- ; E501 line too long - E501 +; W505 doc line too long + E501, W505 ; -- flake8-docstrings -- ; D100 Missing docstring in public module @@ -89,10 +91,18 @@ commands = flake8 ; ** SELECTIVE TESTS ** ; Run flake8 tests (with plugins) for specific optional codes defined below +; -- flake8 -- +; E123 closing bracket does not match indentation of opening bracket’s line +; E226 missing whitespace around arithmetic operator +; E241 multiple spaces after ‘,’ +; E242 tab after ‘,’ +; E704 multiple statements on one line +; W504 line break after binary operator +; W505 doc line too long ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. - flake8 --select=B902,B903 +flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 [coverage:run] omit = From f42cc020ea2a7f1f440a7def925b7a1a0813f095 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 6 Apr 2019 23:19:14 -0400 Subject: [PATCH 224/406] Add flake8-future-import to tox.ini --- tox.ini | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b1074483..cb6b6642 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,8 @@ max-line-length = 79 max-doc-length = 79 verbose = 2 statistics = True +min-version = 2.7 +require-code = True exclude = .github/ .tox/ @@ -65,6 +67,20 @@ ignore = ; D402 First line should not be the function's "signature" D100, D101, D102, D103, D104, D105, D107 +; -- flake8-future-import -- +; x = 1 for missing, 5 for present +; FIx6 nested_scopes 2.2 +; FIx7 generators 2.3 +; FIx2 with_statement 2.6 +; FIx1 absolute_import 3.0 +; FIx0 division 3.0 +; FIx3 print_function 3.0 +; FIx4 unicode_literals 3.0 +; FIx5 generator_stop 3.7 +; ???? annotations 4.0 +; FI90 __future__ import does not exist + FI50, FI51, FI53, FI54 + per-file-ignores = ; F401 imported but unused ; E402 module level import not at top of file @@ -83,6 +99,7 @@ deps = flake8-commas flake8-comprehensions flake8-docstrings + flake8-future-import flake8-quotes skip_install = true commands = @@ -102,7 +119,7 @@ commands = ; -- flake8-bugbear -- ; B902 Invalid first argument used for instance method. ; B903 Data class should be immutable or use __slots__ to save memory. -flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 + flake8 --select=B902,B903,E123,E226,E241,E242,E704,W504,W505 [coverage:run] omit = From 424879e4b6ab60953199b30cd78c8dfad6761dfb Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 7 Apr 2019 17:44:25 -0400 Subject: [PATCH 225/406] Add future imports --- TorrentToMedia.py | 7 +++++++ cleanup.py | 7 ++++++- core/__init__.py | 7 ++++++- core/auto_process/books.py | 7 +++++++ core/auto_process/comics.py | 7 +++++++ core/auto_process/common.py | 7 +++++++ core/auto_process/games.py | 7 +++++++ core/auto_process/movies.py | 7 +++++++ core/auto_process/music.py | 7 +++++++ core/auto_process/tv.py | 7 +++++++ core/configuration.py | 7 +++++++ core/databases.py | 7 +++++++ core/extractor/__init__.py | 7 +++++++ core/forks.py | 7 +++++++ core/github_api.py | 7 +++++++ core/logger.py | 7 +++++++ core/main_db.py | 7 ++++++- core/plugins/downloaders/nzb/configuration.py | 7 +++++++ core/plugins/downloaders/nzb/utils.py | 7 +++++++ core/plugins/downloaders/torrent/configuration.py | 7 +++++++ core/plugins/downloaders/torrent/deluge.py | 7 +++++++ core/plugins/downloaders/torrent/qbittorrent.py | 8 +++++++- core/plugins/downloaders/torrent/transmission.py | 7 +++++++ core/plugins/downloaders/torrent/utils.py | 7 +++++++ core/plugins/downloaders/torrent/utorrent.py | 7 ++++++- core/plugins/plex.py | 7 +++++++ core/plugins/subtitles.py | 7 +++++++ core/scene_exceptions.py | 7 +++++++ core/transcoder.py | 7 +++++++ core/user_scripts.py | 7 +++++++ core/utils/__init__.py | 7 +++++++ core/utils/common.py | 6 ++++++ core/utils/download_info.py | 7 +++++++ core/utils/encoding.py | 7 +++++++ core/utils/files.py | 7 +++++++ core/utils/identification.py | 7 +++++++ core/utils/links.py | 7 +++++++ core/utils/naming.py | 7 +++++++ core/utils/network.py | 7 +++++++ core/utils/parsers.py | 7 +++++++ core/utils/paths.py | 6 ++++++ core/utils/processes.py | 7 +++++++ core/utils/shutil_custom.py | 7 +++++++ core/version_check.py | 7 +++++++ eol.py | 7 +++++++ libs/__init__.py | 7 +++++++ libs/__main__.py | 7 +++++++ libs/autoload.py | 7 +++++++ libs/custom/synchronousdeluge/__init__.py | 7 +++++++ libs/custom/synchronousdeluge/client.py | 8 ++++++++ libs/custom/synchronousdeluge/exceptions.py | 7 +++++++ libs/custom/synchronousdeluge/protocol.py | 7 +++++++ libs/custom/synchronousdeluge/transfer.py | 8 ++++++++ libs/custom/utorrent/client.py | 8 ++++++++ libs/custom/utorrent/upload.py | 7 +++++++ libs/util.py | 7 +++++++ nzbToCouchPotato.py | 7 +++++++ nzbToGamez.py | 7 +++++++ nzbToHeadPhones.py | 7 +++++++ nzbToLazyLibrarian.py | 7 +++++++ nzbToLidarr.py | 7 +++++++ nzbToMedia.py | 7 ++++++- nzbToMylar.py | 7 +++++++ nzbToNzbDrone.py | 7 +++++++ nzbToRadarr.py | 7 +++++++ nzbToSickBeard.py | 7 +++++++ setup.py | 7 ++++++- tests/__init__.py | 7 +++++++ tests/test_initialize.py | 7 ++++++- tests/test_transcoder.py | 7 ++++++- 70 files changed, 483 insertions(+), 9 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index e3665c5e..51a257c8 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -1,6 +1,13 @@ #!/usr/bin/env python # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime import os import sys diff --git a/cleanup.py b/cleanup.py index f0d4e6d1..70c9ce29 100644 --- a/cleanup.py +++ b/cleanup.py @@ -1,6 +1,11 @@ #!/usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import os import subprocess diff --git a/core/__init__.py b/core/__init__.py index 85aeaf27..4621ecfb 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,6 +1,11 @@ # coding=utf-8 -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import itertools import locale diff --git a/core/auto_process/books.py b/core/auto_process/books.py index 8ba74a43..c746d785 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests import core diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 1669d8d8..bb884404 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import requests diff --git a/core/auto_process/common.py b/core/auto_process/common.py index 8c3d9ef2..c083414c 100644 --- a/core/auto_process/common.py +++ b/core/auto_process/common.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from core import logger diff --git a/core/auto_process/games.py b/core/auto_process/games.py index 410b82a3..73907d25 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import shutil diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 4d2a4b2f..2661edd1 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import os import time diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 2f614ee1..e72d9b8c 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import os import time diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 3f8f27c5..57fe2a1e 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import copy import errno import json diff --git a/core/configuration.py b/core/configuration.py index 2d34c8bc..c8e2a465 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import copy import os import shutil diff --git a/core/databases.py b/core/databases.py index 0803b9fc..50e025f3 100644 --- a/core/databases.py +++ b/core/databases.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from core import logger, main_db from core.utils import backup_versioned_file diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index 5d1c51a0..44be3102 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import shutil diff --git a/core/forks.py b/core/forks.py index 400d7e50..c1721aaf 100644 --- a/core/forks.py +++ b/core/forks.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from six import iteritems diff --git a/core/github_api.py b/core/github_api.py index b1a4f566..d61def2f 100644 --- a/core/github_api.py +++ b/core/github_api.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests diff --git a/core/logger.py b/core/logger.py index bdb7e683..98b15432 100644 --- a/core/logger.py +++ b/core/logger.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os import sys diff --git a/core/main_db.py b/core/main_db.py index 14a75d21..4be8aa3c 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -1,6 +1,11 @@ # coding=utf-8 -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import re import sqlite3 diff --git a/core/plugins/downloaders/nzb/configuration.py b/core/plugins/downloaders/nzb/configuration.py index d0883d01..6acd4fc4 100644 --- a/core/plugins/downloaders/nzb/configuration.py +++ b/core/plugins/downloaders/nzb/configuration.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import core diff --git a/core/plugins/downloaders/nzb/utils.py b/core/plugins/downloaders/nzb/utils.py index 23061c96..f5dd29ba 100644 --- a/core/plugins/downloaders/nzb/utils.py +++ b/core/plugins/downloaders/nzb/utils.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import requests diff --git a/core/plugins/downloaders/torrent/configuration.py b/core/plugins/downloaders/torrent/configuration.py index 92001636..dd8e8ffa 100644 --- a/core/plugins/downloaders/torrent/configuration.py +++ b/core/plugins/downloaders/torrent/configuration.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import core from core.plugins.downloaders.torrent.utils import create_torrent_class diff --git a/core/plugins/downloaders/torrent/deluge.py b/core/plugins/downloaders/torrent/deluge.py index 69ae2000..86542a8a 100644 --- a/core/plugins/downloaders/torrent/deluge.py +++ b/core/plugins/downloaders/torrent/deluge.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from synchronousdeluge.client import DelugeClient import core diff --git a/core/plugins/downloaders/torrent/qbittorrent.py b/core/plugins/downloaders/torrent/qbittorrent.py index 61e84a7d..0532b9a7 100644 --- a/core/plugins/downloaders/torrent/qbittorrent.py +++ b/core/plugins/downloaders/torrent/qbittorrent.py @@ -1,4 +1,10 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from qbittorrent import Client as qBittorrentClient diff --git a/core/plugins/downloaders/torrent/transmission.py b/core/plugins/downloaders/torrent/transmission.py index d122aaf0..8c58eae2 100644 --- a/core/plugins/downloaders/torrent/transmission.py +++ b/core/plugins/downloaders/torrent/transmission.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from transmissionrpc.client import Client as TransmissionClient import core diff --git a/core/plugins/downloaders/torrent/utils.py b/core/plugins/downloaders/torrent/utils.py index be2db4e8..0d2896d7 100644 --- a/core/plugins/downloaders/torrent/utils.py +++ b/core/plugins/downloaders/torrent/utils.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import time import core diff --git a/core/plugins/downloaders/torrent/utorrent.py b/core/plugins/downloaders/torrent/utorrent.py index 634e8ca5..e591c46b 100644 --- a/core/plugins/downloaders/torrent/utorrent.py +++ b/core/plugins/downloaders/torrent/utorrent.py @@ -1,4 +1,9 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from utorrent.client import UTorrentClient diff --git a/core/plugins/plex.py b/core/plugins/plex.py index 6d851e85..5c3f0e8b 100644 --- a/core/plugins/plex.py +++ b/core/plugins/plex.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests import core diff --git a/core/plugins/subtitles.py b/core/plugins/subtitles.py index f62f90de..df37e532 100644 --- a/core/plugins/subtitles.py +++ b/core/plugins/subtitles.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from babelfish import Language import subliminal diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 0b70c52c..f00eccd3 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import re diff --git a/core/transcoder.py b/core/transcoder.py index f0047dad..3228c810 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import errno import json import os diff --git a/core/user_scripts.py b/core/user_scripts.py index fbc58614..45adb45b 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os from subprocess import Popen diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 470fe4fd..202e7d19 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import requests from core.utils import shutil_custom diff --git a/core/utils/common.py b/core/utils/common.py index 5ecdf830..2174644c 100644 --- a/core/utils/common.py +++ b/core/utils/common.py @@ -1,3 +1,9 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import os.path diff --git a/core/utils/download_info.py b/core/utils/download_info.py index ce6e6717..2e30d35c 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime from six import text_type diff --git a/core/utils/encoding.py b/core/utils/encoding.py index cbcc3113..bcc4994f 100644 --- a/core/utils/encoding.py +++ b/core/utils/encoding.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os from six import text_type diff --git a/core/utils/files.py b/core/utils/files.py index edb968ae..726d4209 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import re import shutil diff --git a/core/utils/identification.py b/core/utils/identification.py index 9029f7e4..365ea790 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import re diff --git a/core/utils/links.py b/core/utils/links.py index ab558e9b..1daa2441 100644 --- a/core/utils/links.py +++ b/core/utils/links.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import shutil diff --git a/core/utils/naming.py b/core/utils/naming.py index 64cd6d70..4d664036 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import re diff --git a/core/utils/network.py b/core/utils/network.py index 5a7a5758..bfb54381 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import socket import struct import time diff --git a/core/utils/parsers.py b/core/utils/parsers.py index d5f6d933..981de146 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import core diff --git a/core/utils/paths.py b/core/utils/paths.py index a2a96996..ee0a48db 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -1,3 +1,9 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from functools import partial import os diff --git a/core/utils/processes.py b/core/utils/processes.py index 6edcfa44..23bb7a92 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import socket import subprocess diff --git a/core/utils/shutil_custom.py b/core/utils/shutil_custom.py index 5525df1f..a6cb3af8 100644 --- a/core/utils/shutil_custom.py +++ b/core/utils/shutil_custom.py @@ -1,3 +1,10 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from functools import partial import shutil from six import PY2 diff --git a/core/version_check.py b/core/version_check.py index e7dc5578..f1c05ba9 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -2,6 +2,13 @@ # Author: Nic Wolfe # Modified by: echel0n +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform import re diff --git a/eol.py b/eol.py index 5504df7e..7310cfd6 100644 --- a/eol.py +++ b/eol.py @@ -1,5 +1,12 @@ #!/usr/bin/env python +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime import sys import warnings diff --git a/libs/__init__.py b/libs/__init__.py index 4405201d..6c9b28b4 100644 --- a/libs/__init__.py +++ b/libs/__init__.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import site import sys diff --git a/libs/__main__.py b/libs/__main__.py index 767f0d6e..b3d3d8e2 100644 --- a/libs/__main__.py +++ b/libs/__main__.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import shutil import os import time diff --git a/libs/autoload.py b/libs/autoload.py index 23205448..a44ac0fb 100644 --- a/libs/autoload.py +++ b/libs/autoload.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import libs __all__ = ['completed'] diff --git a/libs/custom/synchronousdeluge/__init__.py b/libs/custom/synchronousdeluge/__init__.py index ad90cfe8..fef43298 100644 --- a/libs/custom/synchronousdeluge/__init__.py +++ b/libs/custom/synchronousdeluge/__init__.py @@ -16,6 +16,13 @@ Example usage: download_location = client.core.get_config_value("download_location").get() """ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from .exceptions import DelugeRPCError diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 9194ebe8..191c70f6 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -1,4 +1,12 @@ # coding=utf-8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os import platform from collections import defaultdict diff --git a/libs/custom/synchronousdeluge/exceptions.py b/libs/custom/synchronousdeluge/exceptions.py index dcb23009..1a8693e6 100644 --- a/libs/custom/synchronousdeluge/exceptions.py +++ b/libs/custom/synchronousdeluge/exceptions.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + class DelugeRPCError(Exception): def __init__(self, name, msg, traceback): diff --git a/libs/custom/synchronousdeluge/protocol.py b/libs/custom/synchronousdeluge/protocol.py index 6dcdb2a1..b71dafb4 100644 --- a/libs/custom/synchronousdeluge/protocol.py +++ b/libs/custom/synchronousdeluge/protocol.py @@ -1,5 +1,12 @@ # coding=utf-8 +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + class DelugeRPCRequest(object): def __init__(self, request_id, method, *args, **kwargs): diff --git a/libs/custom/synchronousdeluge/transfer.py b/libs/custom/synchronousdeluge/transfer.py index cbf49100..75228032 100644 --- a/libs/custom/synchronousdeluge/transfer.py +++ b/libs/custom/synchronousdeluge/transfer.py @@ -1,4 +1,12 @@ # coding=utf-8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import socket import ssl import struct diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index ee9ef983..f5eaf93b 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -1,4 +1,12 @@ # coding=utf8 + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import re from six import StringIO, iteritems diff --git a/libs/custom/utorrent/upload.py b/libs/custom/utorrent/upload.py index 1ccbcd37..6aa829a4 100644 --- a/libs/custom/utorrent/upload.py +++ b/libs/custom/utorrent/upload.py @@ -1,6 +1,13 @@ # coding=utf-8 # code copied from http://www.doughellmann.com/PyMOTW/urllib2/ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import itertools import mimetypes from email.generator import _make_boundary as choose_boundary diff --git a/libs/util.py b/libs/util.py index 5ac9f943..354b14cf 100644 --- a/libs/util.py +++ b/libs/util.py @@ -1,4 +1,11 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import subprocess import sys import os diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 9869b0e1..16785b46 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -255,6 +255,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToGamez.py b/nzbToGamez.py index 42d0a2f5..39023421 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -100,6 +100,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index 34fac2e3..a1917f10 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -122,6 +122,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index 0a26013b..a8b88067 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -95,6 +95,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToLidarr.py b/nzbToLidarr.py index 230d8b9d..9681a408 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -237,6 +237,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToMedia.py b/nzbToMedia.py index 1d479d95..1eed58f9 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -655,7 +655,12 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import datetime import os diff --git a/nzbToMylar.py b/nzbToMylar.py index d16b357a..e26cfe6d 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -113,6 +113,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index df551472..fbc3de0c 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -242,6 +242,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToRadarr.py b/nzbToRadarr.py index fa2aa26e..504bbe52 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -247,6 +247,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index d342cf0d..43d23d3d 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -256,6 +256,13 @@ ### NZBGET POST-PROCESSING SCRIPT ### ############################################################################## +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import sys import nzbToMedia diff --git a/setup.py b/setup.py index 36e97020..bac19586 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,11 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import, print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import io import os.path diff --git a/tests/__init__.py b/tests/__init__.py index 1f47cffe..13e499a5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,8 @@ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + __author__ = 'Justin' diff --git a/tests/test_initialize.py b/tests/test_initialize.py index dffcfe76..9e672f4f 100755 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -1,5 +1,10 @@ #! /usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import core diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 148156b0..448ceeaf 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -1,5 +1,10 @@ #! /usr/bin/env python -from __future__ import print_function +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import core from core import transcoder From d3bbcb6b635c686f440b52475345720cad6cb807 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 8 Apr 2019 19:23:21 -0400 Subject: [PATCH 226/406] Remove unnecessary dict factory for database. --- core/main_db.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/core/main_db.py b/core/main_db.py index 4be8aa3c..220b44ca 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -37,10 +37,7 @@ class DBConnection(object): self.filename = filename self.connection = sqlite3.connect(db_filename(filename), 20) - if row_type == 'dict': - self.connection.row_factory = self._dict_factory - else: - self.connection.row_factory = sqlite3.Row + self.connection.row_factory = sqlite3.Row def check_db_version(self): result = None @@ -214,13 +211,6 @@ class DBConnection(object): for column in cursor } - # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query - def _dict_factory(self, cursor, row): - return { - col[0]: row[idx] - for idx, col in enumerate(cursor.description) - } - def sanity_check_database(connection, sanity_check): sanity_check(connection).check() From 455915907b97fd80a4666291134ad10257c45e18 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Mon, 8 Apr 2019 19:22:16 -0400 Subject: [PATCH 227/406] Fix key access for sqlite3.Row on Python 2.7 Fixes #1607 --- core/main_db.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/core/main_db.py b/core/main_db.py index 220b44ca..d9d7c1b9 100644 --- a/core/main_db.py +++ b/core/main_db.py @@ -11,11 +11,37 @@ import re import sqlite3 import time -from six import text_type +from six import text_type, PY2 import core from core import logger +if PY2: + class Row(sqlite3.Row, object): + """ + Row factory that uses Byte Strings for keys. + + The sqlite3.Row in Python 2 does not support unicode keys. + This overrides __getitem__ to attempt to encode the key to bytes first. + """ + + def __getitem__(self, item): + """ + Get an item from the row by index or key. + + :param item: Index or Key of item to return. + :return: An item from the sqlite3.Row. + """ + try: + # sqlite3.Row column names should be Bytes in Python 2 + item = item.encode() + except AttributeError: + pass # assume item is a numeric index + + return super(Row, self).__getitem__(item) +else: + from sqlite3 import Row + def db_filename(filename='nzbtomedia.db', suffix=None): """ @@ -37,7 +63,7 @@ class DBConnection(object): self.filename = filename self.connection = sqlite3.connect(db_filename(filename), 20) - self.connection.row_factory = sqlite3.Row + self.connection.row_factory = Row def check_db_version(self): result = None From 52cae37609e3cf9db2a19d1d9d01147bf57629b6 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 18 Apr 2019 08:40:11 +1200 Subject: [PATCH 228/406] Fix crash of remote_path exception. #1223 --- nzbToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMedia.py b/nzbToMedia.py index 1eed58f9..4435ca04 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -782,7 +782,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d ) except Exception: logger.error('Remote Path {0} is not valid for {1}:{2} Please set this to either 0 to disable or 1 to enable!'.format( - core.get('remote_path'), section_name, input_category)) + cfg.get('remote_path'), section_name, input_category)) input_name, input_directory = convert_to_ascii(input_name, input_directory) From 5375d46c327ee16c90aed4f29ead9c0adad46f92 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 18 Apr 2019 21:46:32 +1200 Subject: [PATCH 229/406] add remote path handling for LazyLibrarian #1223 --- core/auto_process/books.py | 10 ++++++++-- nzbToLazyLibrarian.py | 13 +++++++++++++ nzbToMedia.py | 5 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/auto_process/books.py b/core/auto_process/books.py index c746d785..9c908a94 100644 --- a/core/auto_process/books.py +++ b/core/auto_process/books.py @@ -12,7 +12,12 @@ import requests import core from core import logger from core.auto_process.common import ProcessResult -from core.utils import convert_to_ascii, server_responding +from core.utils import ( + convert_to_ascii, + remote_dir, + server_responding, +) + requests.packages.urllib3.disable_warnings() @@ -28,6 +33,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', ssl = int(cfg.get('ssl', 0)) web_root = cfg.get('web_root', '') protocol = 'https://' if ssl else 'http://' + remote_path = int(cfg.get('remote_path', 0)) url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) if not server_responding(url): @@ -42,7 +48,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', params = { 'apikey': apikey, 'cmd': 'forceProcess', - 'dir': dir_name, + 'dir': remote_dir(dir_name) if remote_path else dir_name, } logger.debug('Opening URL: {0} with params: {1}'.format(url, params), section) diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index a8b88067..dd760709 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -59,6 +59,19 @@ # set this to where your LazyLibrarian completed downloads are. #llwatch_dir= +# LazyLibrarian and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#llremote_path=0 + +## Network + +# Network Mount Points (Needed for remote path above) +# +# Enter Mount points as LocalPath,RemotePath and separate each pair with '|' +# e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ +#mountPoints= + ## Posix # Niceness for external tasks Extractor and Transcoder. diff --git a/nzbToMedia.py b/nzbToMedia.py index 4435ca04..f4fb0759 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -444,6 +444,11 @@ # set this to where your LazyLibrarian completed downloads are. #llwatch_dir= +# LazyLibrarian and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#llremote_path=0 + ## Network # Network Mount Points (Needed for remote path above) From 5ff056844c10c5d8905700727c3c503c7a35f928 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Mon, 20 May 2019 21:17:54 +1200 Subject: [PATCH 230/406] Fix NoExtractFailed usage. Fixes #1618 --- nzbToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMedia.py b/nzbToMedia.py index f4fb0759..03f19a8d 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -791,7 +791,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d input_name, input_directory = convert_to_ascii(input_name, input_directory) - if extract == 1: + if extract == 1 and not (status > 0 and core.NOEXTRACTFAILED): logger.debug('Checking for archives to extract in directory: {0}'.format(input_directory)) extract_files(input_directory) From 8c45e7650774421cbdd9781f01f55edfcf8ee955 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 31 May 2019 14:06:25 +1200 Subject: [PATCH 231/406] Bluray 1 (#1620) * added code to extract bluray images and folder structure. #1588 * add Mounting of iso files as fall-back * add new mkv-bluray default. * clean-up fall-back for ffmpeg not accepting -show error --- autoProcessMedia.cfg.spec | 2 +- core/__init__.py | 13 ++- core/transcoder.py | 222 +++++++++++++++++++++++++++++++------- nzbToCouchPotato.py | 2 +- nzbToLidarr.py | 2 +- nzbToMedia.py | 2 +- nzbToNzbDrone.py | 2 +- nzbToRadarr.py | 2 +- nzbToSickBeard.py | 2 +- 9 files changed, 205 insertions(+), 44 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 30a399a0..6829d6b2 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -418,7 +418,7 @@ generalOptions = # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, leave this blank and set the remaining options below. - # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release + # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release outputDefault = #### Define custom settings below. outputVideoExtension = .mp4 diff --git a/core/__init__.py b/core/__init__.py index 4621ecfb..7fb32140 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -199,6 +199,7 @@ META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] +MOUNTED = None GETSUBS = False TRANSCODE = None CONCAT = None @@ -504,6 +505,7 @@ def configure_containers(): def configure_transcoder(): + global MOUNTED global GETSUBS global TRANSCODE global DUPLICATE @@ -548,6 +550,7 @@ def configure_transcoder(): global ALLOWSUBS global DEFAULTS + MOUNTED = None GETSUBS = int(CFG['Transcoder']['getSubs']) TRANSCODE = int(CFG['Transcoder']['transcode']) DUPLICATE = int(CFG['Transcoder']['duplicate']) @@ -751,7 +754,15 @@ def configure_transcoder(): }, 'mkv': { 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, - 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'SCODEC': 'mov_text' + }, + 'mkv-bluray': { + 'VEXTENSION': '.mkv', 'VCODEC': 'libx265', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'hevc', 'h265', 'libx265', 'h.265', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, diff --git a/core/transcoder.py b/core/transcoder.py index 3228c810..e1850de2 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -9,7 +9,9 @@ from __future__ import ( import errno import json +import sys import os +import time import platform import re import shutil @@ -73,7 +75,10 @@ def is_video_good(videofile, status): def zip_out(file, img, bitbucket): procin = None - cmd = [core.SEVENZIP, '-so', 'e', img, file] + if os.path.isfile(file): + cmd = ['cat', file] + else: + cmd = [core.SEVENZIP, '-so', 'e', img, file] try: procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) except Exception: @@ -104,12 +109,11 @@ def get_video_details(videofile, img=None, bitbucket=None): result = proc.returncode video_details = json.loads(out.decode()) except Exception: - pass - if not video_details: - try: + try: # try this again without -show error in case of ffmpeg limitation command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] + print_cmd(command) if img: - procin = zip_out(file, img) + procin = zip_out(file, img, bitbucket) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) procin.stdout.close() else: @@ -122,6 +126,21 @@ def get_video_details(videofile, img=None, bitbucket=None): return video_details, result +def check_vid_file(video_details, result): + if result != 0: + return False + if video_details.get('error'): + return False + if not video_details.get('streams'): + return False + video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] + audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + if len(video_streams) > 0 and len(audio_streams) > 0: + return True + else: + return False + + def build_commands(file, new_dir, movie_name, bitbucket): if isinstance(file, string_types): input_file = file @@ -139,9 +158,18 @@ def build_commands(file, new_dir, movie_name, bitbucket): name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext' + new_file = file else: img, data = next(iteritems(file)) name = data['name'] + new_file = [] + rem_vid = [] + for vid in data['files']: + video_details, result = get_video_details(vid, img, bitbucket) + if not check_vid_file(video_details, result): #lets not transcode menu or other clips that don't have audio and video. + rem_vid.append(vid) + data['files'] = [ f for f in data['files'] if f not in rem_vid ] + new_file = {img: {'name': data['name'], 'files': data['files']}} video_details, result = get_video_details(data['files'][0], img, bitbucket) input_file = '-' file = '-' @@ -518,7 +546,7 @@ def build_commands(file, new_dir, movie_name, bitbucket): command.append(newfile_path) if platform.system() != 'Windows': command = core.NICENESS + command - return command + return command, new_file def get_subs(file): @@ -597,6 +625,7 @@ def process_list(it, new_dir, bitbucket): new_list = [] combine = [] vts_path = None + mts_path = None success = True for item in it: ext = os.path.splitext(item)[1].lower() @@ -612,6 +641,14 @@ def process_list(it, new_dir, bitbucket): except Exception: vts_path = os.path.split(item)[0] rem_list.append(item) + elif re.match('.+BDMV[/\\]SOURCE[/\\][0-9]+[0-9].[Mm][Tt][Ss]', item) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(item), 'TRANSCODER') + if not mts_path: + try: + mts_path = re.match('(.+BDMV[/\\]SOURCE)', item).groups()[0] + except Exception: + mts_path = os.path.split(item)[0] + rem_list.append(item) elif re.match('.+VIDEO_TS.', item) or re.match('.+VTS_[0-9][0-9]_[0-9].', item): rem_list.append(item) elif core.CONCAT and re.match('.+[cC][dD][0-9].', item): @@ -621,6 +658,8 @@ def process_list(it, new_dir, bitbucket): continue if vts_path: new_list.extend(combine_vts(vts_path)) + if mts_path: + new_list.extend(combine_mts(mts_path)) if combine: new_list.extend(combine_cd(combine)) for file in new_list: @@ -639,17 +678,53 @@ def process_list(it, new_dir, bitbucket): return it, rem_list, new_list, success +def mount_iso(item, new_dir, bitbucket): #Currently only supports Linux Mount when permissions allow. + if platform.system() == 'Windows': + logger.error('No mounting options available under Windows for image file {0}'.format(item), 'TRANSCODER') + return [] + mount_point = os.path.join(os.path.dirname(os.path.abspath(item)),'temp') + make_dir(mount_point) + cmd = ['mount', '-o', 'loop', item, mount_point] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + core.MOUNTED = mount_point # Allows us to verify this has been done and then cleanup. + for root, dirs, files in os.walk(mount_point): + for file in files: + full_path = os.path.join(root, file) + if re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', full_path) and '.vob' not in core.IGNOREEXTENSIONS: + logger.debug('Found VIDEO_TS image file: {0}'.format(full_path), 'TRANSCODER') + try: + vts_path = re.match('(.+VIDEO_TS)', full_path).groups()[0] + except Exception: + vts_path = os.path.split(full_path)[0] + return combine_vts(vts_path) + elif re.match('.+BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]', full_path) and '.mts' not in core.IGNOREEXTENSIONS: + logger.debug('Found MTS image file: {0}'.format(full_path), 'TRANSCODER') + try: + mts_path = re.match('(.+BDMV[/\\]STREAM)', full_path).groups()[0] + except Exception: + mts_path = os.path.split(full_path)[0] + return combine_mts(mts_path) + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file {0}'.format(mount_point), 'TRANSCODER') + return ['failure'] # If we got here, nothing matched our criteria + + def rip_iso(item, new_dir, bitbucket): new_files = [] failure_dir = 'failure' # Mount the ISO in your OS and call combineVTS. if not core.SEVENZIP: - logger.error('No 7zip installed. Can\'t extract image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.debug('No 7zip installed. Attempting to mount image file {0}'.format(item), 'TRANSCODER') + try: + new_files = mount_iso(item, new_dir, bitbucket) # Currently only works for Linux. + except Exception: + logger.error('Failed to mount and extract from image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] return new_files cmd = [core.SEVENZIP, 'l', item] try: - logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') + logger.debug('Attempting to extract .vob or .mts from image file {0}'.format(item), 'TRANSCODER') print_cmd(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) out, err = proc.communicate() @@ -663,31 +738,58 @@ def rip_iso(item, new_dir, bitbucket): if file_match ] combined = [] - for n in range(99): - concat = [] - m = 1 - while True: - vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) - if vts_name in file_list: - concat.append(vts_name) - m += 1 - else: + if file_list: # handle DVD + for n in range(99): + concat = [] + m = 1 + while True: + vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) + if vts_name in file_list: + concat.append(vts_name) + m += 1 + else: + break + if not concat: break - if not concat: - break - if core.CONCAT: - combined.extend(concat) - continue - name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1, + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + ) + new_files.append({item: {'name': name, 'files': concat}}) + else: #check BlueRay for BDMV/STREAM/XXXX.MTS + mts_list_gen = ( + re.match(r'.+(BDMV[/\\]STREAM[/\\][0-9]+[0-9].[Mm]).', line) + for line in out.decode().splitlines() ) - new_files.append({item: {'name': name, 'files': concat}}) - if core.CONCAT: + mts_list = [ + file_match.groups()[0] + for file_match in mts_list_gen + if file_match + ] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) # Sort all .mts files in numerical order + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + n = 0 + for mts_name in mts_list: + concat = [] + n += 1 + concat.append(mts_name) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + ) + new_files.append({item: {'name': name, 'files': concat}}) + if core.CONCAT and combined: name = os.path.splitext(os.path.split(item)[1])[0] new_files.append({item: {'name': name, 'files': combined}}) if not new_files: - logger.error('No VIDEO_TS folder found in image file {0}'.format(item), 'TRANSCODER') - new_files = [failure_dir] + logger.error('No VIDEO_TS or BDMV/SOURCE folder found in image file. Attempting to mount and scan {0}'.format(item), 'TRANSCODER') + new_files = mount_iso(item, new_dir, bitbucket) except Exception: logger.error('Failed to extract from image file {0}'.format(item), 'TRANSCODER') new_files = [failure_dir] @@ -696,25 +798,63 @@ def rip_iso(item, new_dir, bitbucket): def combine_vts(vts_path): new_files = [] - combined = '' + combined = [] + name = re.match(r'(.+)[/\\]VIDEO_TS', vts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) for n in range(99): - concat = '' + concat = [] m = 1 while True: vts_name = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m) if os.path.isfile(os.path.join(vts_path, vts_name)): - concat += '{file}|'.format(file=os.path.join(vts_path, vts_name)) + concat.append(os.path.join(vts_path, vts_name)) m += 1 else: break if not concat: break if core.CONCAT: - combined += '{files}|'.format(files=concat) + combined.extend(concat) continue - new_files.append('concat:{0}'.format(concat[:-1])) + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({vts_path: {'name': name, 'files': concat}}) if core.CONCAT: - new_files.append('concat:{0}'.format(combined[:-1])) + new_files.append({vts_path: {'name': name, 'files': combined}}) + return new_files + + +def combine_mts(mts_path): + new_files = [] + combined = [] + name = re.match(r'(.+)[/\\]BDMV[/\\]STREAM', mts_path).groups()[0] + if os.path.basename(name) == 'temp': + name = os.path.basename(os.path.dirname(name)) + else: + name = os.path.basename(name) + n = 0 + mts_list = [f for f in os.listdir(mts_path) if os.path.isfile(os.path.join(mts_path, f))] + if sys.version_info[0] == 2: # Python2 sorting + mts_list.sort(key=lambda f: int(filter(str.isdigit, f))) + else: # Python3 sorting + mts_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f)))) + for mts_name in mts_list: ### need to sort all files [1 - 998].mts in order + concat = [] + concat.append(os.path.join(mts_path, mts_name)) + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=name, x=n + 1 + ) + new_files.append({mts_path: {'name': name, 'files': concat}}) + n += 1 + if core.CONCAT: + new_files.append({mts_path: {'name': name, 'files': combined}}) return new_files @@ -768,7 +908,7 @@ def transcode_directory(dir_name): for file in file_list: if isinstance(file, string_types) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: continue - command = build_commands(file, new_dir, movie_name, bitbucket) + command, file = build_commands(file, new_dir, movie_name, bitbucket) newfile_path = command[-1] # transcoding files may remove the original file, so make sure to extract subtitles first @@ -795,6 +935,7 @@ def transcode_directory(dir_name): for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: + logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() proc.communicate() @@ -826,6 +967,15 @@ def transcode_directory(dir_name): logger.error('Transcoding of video to {0} failed with result {1}'.format(newfile_path, result)) # this will be 0 (successful) it all are successful, else will return a positive integer for failure. final_result = final_result + result + if core.MOUNTED: # In case we mounted an .iso file, unmount here. + time.sleep(5) # play it safe and avoid failing to unmount. + cmd = ['umount', '-l', core.MOUNTED] + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + time.sleep(5) + os.rmdir(core.MOUNTED) + core.MOUNTED = None if final_result == 0 and not core.DUPLICATE: for file in rem_list: try: diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 16785b46..862e7f78 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -202,7 +202,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToLidarr.py b/nzbToLidarr.py index 9681a408..c38f5975 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -187,7 +187,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToMedia.py b/nzbToMedia.py index 03f19a8d..12a72ae6 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -558,7 +558,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index fbc3de0c..680074cd 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -192,7 +192,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToRadarr.py b/nzbToRadarr.py index 504bbe52..f75ca350 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -197,7 +197,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir = -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 43d23d3d..00f18077 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -203,7 +203,7 @@ # externalSubDir. set the directory where subs should be saved (if not the same directory as the video) #externalSubDir= -# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mp4-scene-release, MKV-SD). +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). # # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, set None and set the remaining options below. From fd1149aea1e0b1e524d821900d586c05bc9ce639 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 6 Jun 2019 21:46:56 +1200 Subject: [PATCH 232/406] add additional options to pass into ffmpeg. #1619 --- autoProcessMedia.cfg.spec | 4 +++- core/__init__.py | 7 +++++++ core/transcoder.py | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 6829d6b2..7969831d 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -414,8 +414,10 @@ externalSubDir = # hwAccel. 1 will set ffmpeg to enable hardware acceleration (this requires a recent ffmpeg) hwAccel = 0 - # generalOptions. Enter your additional ffmpeg options here with commas to separate each option/value (i.e replace spaces with commas). + # generalOptions. Enter your additional ffmpeg options (these insert before the '-i' input files) here with commas to separate each option/value (i.e replace spaces with commas). generalOptions = + # otherOptions. Enter your additional ffmpeg options (these insert after the '-i' input files and before the output file) here with commas to separate each option/value (i.e replace spaces with commas). + otherOptions = # outputDefault. Loads default configs for the selected device. The remaining options below are ignored. # If you want to use your own profile, leave this blank and set the remaining options below. # outputDefault profiles allowed: iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release diff --git a/core/__init__.py b/core/__init__.py index 7fb32140..b3c643fb 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -211,6 +211,7 @@ VEXTENSION = None OUTPUTVIDEOPATH = None PROCESSOUTPUT = False GENERALOPTS = [] +OTHEROPTS = [] ALANGUAGE = None AINCLUDE = False SLANGUAGES = [] @@ -513,6 +514,7 @@ def configure_transcoder(): global IGNOREEXTENSIONS global OUTPUTFASTSTART global GENERALOPTS + global OTHEROPTS global OUTPUTQUALITYPERCENT global OUTPUTVIDEOPATH global PROCESSOUTPUT @@ -568,6 +570,11 @@ def configure_transcoder(): GENERALOPTS.append('-fflags') if '+genpts' not in GENERALOPTS: GENERALOPTS.append('+genpts') + OTHEROPTS = (CFG['Transcoder']['otherOptions']) + if isinstance(OTHEROPTS, str): + OTHEROPTS = OTHEROPTS.split(',') + if OTHEROPTS == ['']: + OTHEROPTS = [] try: OUTPUTQUALITYPERCENT = int(CFG['Transcoder']['outputQualityPercent']) except Exception: diff --git a/core/transcoder.py b/core/transcoder.py index e1850de2..fcbeb43f 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -493,6 +493,8 @@ def build_commands(file, new_dir, movie_name, bitbucket): if core.OUTPUTFASTSTART: other_cmd.extend(['-movflags', '+faststart']) + if core.OTHEROPTS: + other_cmd.extend(core.OTHEROPTS) command = [core.FFMPEG, '-loglevel', 'warning'] From 323733677572579bcb2e2122d8f5672e78b9516d Mon Sep 17 00:00:00 2001 From: TheHolyRoger <39387497+TheHolyRoger@users.noreply.github.com> Date: Sat, 8 Jun 2019 00:20:12 +0100 Subject: [PATCH 233/406] Don't replace apostrophes in qBittorrent input_name Don't replace apostrophes in qBittorrent input_name - only trim if found at beginning/end of string. This stops nzbtomedia processing the entire download folder when asked to process a folder with apostrophes in the title --- core/utils/parsers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/utils/parsers.py b/core/utils/parsers.py index 981de146..fd88ec63 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -127,7 +127,11 @@ def parse_qbittorrent(args): except Exception: input_directory = '' try: - input_name = cur_input[1].replace('\'', '') + input_name = cur_input[1] + if input_name[0] == '\'': + input_name = input_name[1:] + if input_name[-1] == '\'': + input_name = input_name[:-1] except Exception: input_name = '' try: From d39a7dd23497943dd853d585e7976cdb46fc7877 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 18 Jun 2019 20:52:19 +1200 Subject: [PATCH 234/406] fix to make deluge client py 2 and 3 compatible. Fixes #1626 --- libs/custom/synchronousdeluge/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 191c70f6..0418e806 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -82,13 +82,13 @@ class DelugeClient(object): def func(obj, *args, **kwargs): return self.remote_call(fullname, *args, **kwargs) - func.__name__ = method + func.__name__ = str(method) return func def _introspect(self): def splitter(value): - return value.split('.') + return value.split(b'.') self.modules = [] @@ -100,8 +100,8 @@ class DelugeClient(object): for module, methods in methodmap.items(): clsname = 'DelugeModule{0}'.format(module.capitalize()) - cls = type(clsname, (), methods) - setattr(self, module, cls()) + cls = type(str(clsname), (), methods) + setattr(self, str(module), cls()) self.modules.append(module) def remote_call(self, method, *args, **kwargs): From f1dc6720564ab7c4d255fdf54c87a164e7eb6dcf Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 19 Jun 2019 22:50:36 +1200 Subject: [PATCH 235/406] fix deluge client for python3. Fixes #1626 --- libs/custom/synchronousdeluge/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/custom/synchronousdeluge/client.py b/libs/custom/synchronousdeluge/client.py index 0418e806..f446cede 100644 --- a/libs/custom/synchronousdeluge/client.py +++ b/libs/custom/synchronousdeluge/client.py @@ -88,11 +88,12 @@ class DelugeClient(object): def _introspect(self): def splitter(value): - return value.split(b'.') + return value.split('.') self.modules = [] methods = self.remote_call('daemon.get_method_list').get() + methods = (x.decode() for x in methods) methodmap = defaultdict(dict) for module, method in imap(splitter, methods): From ce50a1c27da5ebe3b38adfdc363eda5f7254941f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 19 Jun 2019 21:37:42 +0000 Subject: [PATCH 236/406] Fix allready running handling for Python3. #1626 --- core/utils/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/processes.py b/core/utils/processes.py index 23bb7a92..56882757 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -54,7 +54,7 @@ class PosixProcess(object): self.lasterror = False return self.lasterror except socket.error as e: - if 'Address already in use' in e: + if 'Address already in use' in str(e): self.lasterror = True return self.lasterror except AttributeError: From 9f6c068cdec1a31943a8f1d785f84d1476ebb749 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 20 Jun 2019 12:56:02 +1200 Subject: [PATCH 237/406] Transcode patch 1 (#1627) * Add Piping of stderr to capture transcoding failures. #1619 * Allow passing absolute nice command. #1619 * Change .cfg description for niceness * Fix errors due to VM packages out of date (ffmpeg) * Fix Sqlite import error on tests * Fix Azure issues https://developercommunity.visualstudio.com/content/problem/598264/known-issue-azure-pipelines-images-missing-sqlite3.html --- autoProcessMedia.cfg.spec | 4 +++- azure-pipelines.yml | 44 +++++++++++++++++++++++++++++++++++---- core/__init__.py | 5 ++++- core/transcoder.py | 10 +++++---- nzbToCouchPotato.py | 6 ++++-- nzbToGamez.py | 6 ++++-- nzbToHeadPhones.py | 6 ++++-- nzbToLazyLibrarian.py | 6 ++++-- nzbToLidarr.py | 6 ++++-- nzbToMedia.py | 6 ++++-- nzbToMylar.py | 6 ++++-- nzbToNzbDrone.py | 6 ++++-- nzbToRadarr.py | 6 ++++-- nzbToSickBeard.py | 6 ++++-- 14 files changed, 93 insertions(+), 30 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 7969831d..e3d11cb6 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -36,7 +36,9 @@ [Posix] ### Process priority setting for External commands (Extractor and Transcoder) on Posix (Unix/Linux/OSX) systems. # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). - niceness = 0 + # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). + # If entering a comma separated list e.g. 'niceness = nice,4' this will be passed as 'nice 4' (Safer). + niceness = nice,-n0 # Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. ionice_class = 0 # Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 96321a5b..bc64c2d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,42 @@ jobs: maxParallel: 4 steps: + - script: | + # Make sure all packages are pulled from latest + sudo apt-get update + + # Fail out if any setups fail + set -e + + # Delete old Pythons + rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + + # Download new Pythons + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + + # Install new Pythons + original_directory=$PWD + setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + for setup in $setups; do + chmod +x $setup; + cd $(dirname $setup); + ./$(basename $setup); + cd $original_directory; + done; + displayName: 'Workaround: update apt and roll back Python versions' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' @@ -41,10 +77,10 @@ jobs: displayName: 'pytest' - script: | - rm -rf .git - python cleanup.py - python TorrentToMedia.py - python nzbToMedia.py + rm -rf .git + python cleanup.py + python TorrentToMedia.py + python nzbToMedia.py displayName: 'Test source install cleanup' - task: PublishTestResults@2 diff --git a/core/__init__.py b/core/__init__.py index b3c643fb..a7a4ce24 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -457,7 +457,10 @@ def configure_niceness(): with open(os.devnull, 'w') as devnull: try: subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) + if len(CFG['Posix']['niceness'].split(',')) > 1: #Allow passing of absolute command, not just value. + NICENESS.extend(CFG['Posix']['niceness'].split(',')) + else: + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) except Exception: pass try: diff --git a/core/transcoder.py b/core/transcoder.py index fcbeb43f..f0da8dd4 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -607,7 +607,7 @@ def extract_subs(file, newfile_path, bitbucket): result = 1 # set result to failed in case call fails. try: proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - proc.communicate() + out, err = proc.communicate() result = proc.returncode except Exception: logger.error('Extracting subtitle has failed') @@ -930,17 +930,19 @@ def transcode_directory(dir_name): result = 1 # set result to failed in case call fails. try: if isinstance(file, string_types): - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE) else: img, data = next(iteritems(file)) - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE) for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() - proc.communicate() + out, err = proc.communicate() + if err: + logger.error('Transcoder returned:{0} has failed'.format(err)) result = proc.returncode except Exception: logger.error('Transcoding of video {0} has failed'.format(newfile_path)) diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 862e7f78..60638200 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -112,8 +112,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToGamez.py b/nzbToGamez.py index 39023421..92237f07 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -68,8 +68,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index a1917f10..d875d151 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -82,8 +82,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index dd760709..ad8d2534 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -76,8 +76,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLidarr.py b/nzbToLidarr.py index c38f5975..da89ab46 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -97,8 +97,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMedia.py b/nzbToMedia.py index 12a72ae6..f2a5a57f 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -468,8 +468,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMylar.py b/nzbToMylar.py index e26cfe6d..cd408dcb 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -73,8 +73,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index 680074cd..b99ed190 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -102,8 +102,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToRadarr.py b/nzbToRadarr.py index f75ca350..a7e02f93 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -107,8 +107,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 00f18077..b6d8b42e 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -113,8 +113,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # From 95e4c70d9a85b4f1737bd5b272756d7efce2df64 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 9 Jul 2019 15:05:33 +1200 Subject: [PATCH 238/406] Set theme jekyll-theme-cayman --- _config.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 _config.yml diff --git a/_config.yml b/_config.yml new file mode 100644 index 00000000..c4192631 --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-cayman \ No newline at end of file From 9a958afac8053c0e579d9094b9c469681245bb5b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 12 Jul 2019 19:39:55 +1200 Subject: [PATCH 239/406] don't crash when no optionalParameters. Fixes #1630 (#1632) --- core/forks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/forks.py b/core/forks.py index c1721aaf..b5e8c840 100644 --- a/core/forks.py +++ b/core/forks.py @@ -114,11 +114,14 @@ def auto_fork(section, input_category): else: json_data = json_data.get('data', json_data) - optional_parameters = json_data['optionalParameters'].keys() - # Find excess parameters - excess_parameters = set(params).difference(optional_parameters) - logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) - rem_params.extend(excess_parameters) + try: + optional_parameters = json_data['optionalParameters'].keys() + # Find excess parameters + excess_parameters = set(params).difference(optional_parameters) + logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) + rem_params.extend(excess_parameters) + except: + logger.error('Failed to identify optionalParameters') else: # Find excess parameters rem_params.extend( From f21e18b1bfd07dc2a4e20ff66fb020034340572d Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 16 Jul 2019 14:36:00 +1200 Subject: [PATCH 240/406] Fix2 (#1636) add chnaged api handling for SickGear. Fixes #1630 --- core/forks.py | 63 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/core/forks.py b/core/forks.py index b5e8c840..97be31da 100644 --- a/core/forks.py +++ b/core/forks.py @@ -13,6 +13,34 @@ from six import iteritems import core from core import logger +def api_check(r, params, rem_params): + try: + json_data = r.json() + except ValueError: + logger.error('Failed to get JSON data from response') + logger.debug('Response received') + raise + + try: + json_data = json_data['data'] + except KeyError: + logger.error('Failed to get data from JSON') + logger.debug('Response received: {}'.format(json_data)) + raise + else: + json_data = json_data.get('data', json_data) + + try: + optional_parameters = json_data['optionalParameters'].keys() + # Find excess parameters + excess_parameters = set(params).difference(optional_parameters) + logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) + rem_params.extend(excess_parameters) + return rem_params, True + except: + logger.error('Failed to identify optionalParameters') + return rem_params, False + def auto_fork(section, input_category): # auto-detect correct section @@ -98,30 +126,17 @@ def auto_fork(section, input_category): r = [] if r and r.ok: if apikey: - try: - json_data = r.json() - except ValueError: - logger.error('Failed to get JSON data from response') - logger.debug('Response received') - raise - - try: - json_data = json_data['data'] - except KeyError: - logger.error('Failed to get data from JSON') - logger.debug('Response received: {}'.format(json_data)) - raise - else: - json_data = json_data.get('data', json_data) - - try: - optional_parameters = json_data['optionalParameters'].keys() - # Find excess parameters - excess_parameters = set(params).difference(optional_parameters) - logger.debug('Removing excess parameters: {}'.format(sorted(excess_parameters))) - rem_params.extend(excess_parameters) - except: - logger.error('Failed to identify optionalParameters') + rem_params, found = api_check(r, params, rem_params) + if not found: # try different api set for SickGear. + url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=postprocess&help=1'.format( + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey, + ) + try: + r = s.get(url, auth=(username, password), verify=False) + except requests.ConnectionError: + logger.info('Could not connect to {section}:{category} to perform auto-fork detection!'.format + (section=section, category=input_category)) + rem_params, found = api_check(r, params, rem_params) else: # Find excess parameters rem_params.extend( From 8ba8caf02155e311dff6137129fe46a48209f64a Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 22 Jul 2019 12:35:01 +1200 Subject: [PATCH 241/406] Fix3 (#1637) * add singular fork detection for multiple runs. Fixes #1637 * Add newly identified fork variants #1630 #1637 * remove encoding of paths. #1637 #1582 --- TorrentToMedia.py | 24 ------------------------ core/__init__.py | 5 +++++ core/forks.py | 5 +++++ core/utils/identification.py | 9 --------- core/utils/naming.py | 4 ---- nzbToMedia.py | 8 -------- 6 files changed, 10 insertions(+), 45 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 51a257c8..42707b1a 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -70,14 +70,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp input_category = 'UNCAT' usercat = input_category - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass - # try: - # input_directory = input_directory.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format (input_directory, input_name, input_category)) @@ -135,10 +127,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp else: output_destination = os.path.normpath( core.os.path.join(core.OUTPUT_DIRECTORY, input_category)) - # try: - # output_destination = output_destination.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass if output_destination in input_directory: output_destination = input_directory @@ -180,10 +168,6 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) logger.debug('Setting outputDestination to {0} to preserve folder structure'.format (os.path.dirname(target_file))) - # try: - # target_file = target_file.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass if root == 1: if not found_file: logger.debug('Looking for {0} in: {1}'.format(input_name, inputFile)) @@ -362,15 +346,7 @@ def main(args): if client_agent.lower() not in core.TORRENT_CLIENTS: continue - # try: - # dir_name = dir_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass input_name = os.path.basename(dir_name) - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except UnicodeError: - # pass results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, client_agent) diff --git a/core/__init__.py b/core/__init__.py index a7a4ce24..e2436182 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -99,10 +99,12 @@ FORK_FAILED = 'failed' FORK_FAILED_TORRENT = 'failed-torrent' FORK_SICKRAGE = 'SickRage' FORK_SICKCHILL = 'SickChill' +FORK_SICKCHILL_API = 'SickChill-api' FORK_SICKBEARD_API = 'SickBeard-api' FORK_MEDUSA = 'Medusa' FORK_MEDUSA_API = 'Medusa-api' FORK_SICKGEAR = 'SickGear' +FORK_SICKGEAR_API = 'SickGear-api' FORK_STHENO = 'Stheno' FORKS = { @@ -111,10 +113,12 @@ FORKS = { FORK_FAILED_TORRENT: {'dir': None, 'failed': None, 'process_method': None}, FORK_SICKRAGE: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None}, FORK_SICKCHILL: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None}, + FORK_SICKCHILL_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'is_priority': None}, FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, + FORK_SICKGEAR_API: {'path': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'is priority': None}, FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, } ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} @@ -198,6 +202,7 @@ META_CONTAINER = [] SECTIONS = [] CATEGORIES = [] +FORK_SET = [] MOUNTED = None GETSUBS = False diff --git a/core/forks.py b/core/forks.py index 97be31da..f49ed5a9 100644 --- a/core/forks.py +++ b/core/forks.py @@ -45,6 +45,10 @@ def api_check(r, params, rem_params): def auto_fork(section, input_category): # auto-detect correct section # config settings + if core.FORK_SET: # keep using determined fork for multiple (manual) post-processing + logger.info('{section}:{category} fork already set to {fork}'.format + (section=section, category=input_category, fork=core.FORK_SET[0])) + return core.FORK_SET[0], core.FORK_SET[1] cfg = dict(core.CFG[section][input_category]) @@ -168,4 +172,5 @@ def auto_fork(section, input_category): logger.info('{section}:{category} fork set to {fork}'.format (section=section, category=input_category, fork=fork[0])) + core.FORK_SET = fork return fork[0], fork[1] diff --git a/core/utils/identification.py b/core/utils/identification.py index 365ea790..7ea2fdc1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -96,15 +96,6 @@ def find_imdbid(dir_name, input_name, omdb_api_key): def category_search(input_directory, input_name, input_category, root, categories): tordir = False - # try: - # input_name = input_name.encode(core.SYS_ENCODING) - # except Exception: - # pass - # try: - # input_directory = input_directory.encode(core.SYS_ENCODING) - # except Exception: - # pass - if input_directory is None: # =Nothing to process here. return input_directory, input_name, input_category, root diff --git a/core/utils/naming.py b/core/utils/naming.py index 4d664036..dfabec3f 100644 --- a/core/utils/naming.py +++ b/core/utils/naming.py @@ -26,10 +26,6 @@ def sanitize_name(name): # remove leading/trailing periods and spaces name = name.strip(' .') - # try: - # name = name.encode(core.SYS_ENCODING) - # except Exception: - # pass return name diff --git a/nzbToMedia.py b/nzbToMedia.py index f2a5a57f..a6139fe3 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -973,15 +973,7 @@ def main(args, section=None): if client_agent and client_agent.lower() not in core.NZB_CLIENTS: continue - try: - dir_name = dir_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass input_name = os.path.basename(dir_name) - try: - input_name = input_name.encode(core.SYS_ENCODING) - except UnicodeError: - pass results = process(dir_name, input_name, 0, client_agent=client_agent, download_id=download_id or None, input_category=subsection) From 5a18ee9a27ba9fabfab44a90e3320071c54d06f3 Mon Sep 17 00:00:00 2001 From: currently-off-my-rocker Date: Mon, 22 Jul 2019 13:07:09 +0200 Subject: [PATCH 242/406] identify imdb ids with 8 digits --- core/utils/identification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 7ea2fdc1..621907a1 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -23,14 +23,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): # find imdbid in dirName logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7})', dir_name + input_name) + m = re.search(r'(tt\d{7,8})', dir_name + input_name) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}]'.format(imdbid)) return imdbid if os.path.isdir(dir_name): for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7})', file) + m = re.search(r'(tt\d{7,8})', file) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}] via file name'.format(imdbid)) From d7eab5d2d39575d006d23b06f6f12977b9388cca Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 23 Jul 2019 14:24:31 +1200 Subject: [PATCH 243/406] Add word boundary to imdb match. #1639 Prevents matching (and truncating) longer ids. Thanks @currently-off-my-rocker --- core/utils/identification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/identification.py b/core/utils/identification.py index 621907a1..84fa834a 100644 --- a/core/utils/identification.py +++ b/core/utils/identification.py @@ -23,14 +23,14 @@ def find_imdbid(dir_name, input_name, omdb_api_key): # find imdbid in dirName logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7,8})', dir_name + input_name) + m = re.search(r'\b(tt\d{7,8})\b', dir_name + input_name) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}]'.format(imdbid)) return imdbid if os.path.isdir(dir_name): for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7,8})', file) + m = re.search(r'\b(tt\d{7,8})\b', file) if m: imdbid = m.group(1) logger.info('Found imdbID [{0}] via file name'.format(imdbid)) From 5714540949b183372b038d8ebf05911f2db93932 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 2 Aug 2019 13:02:46 +1200 Subject: [PATCH 244/406] Fix uTorrent with Python3 (#1644) * Remove temp workaround for Microsoft Azure python issues. --- azure-pipelines.yml | 50 +++++++++++++++++----------------- libs/custom/utorrent/client.py | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc64c2d4..db2d4f2b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,42 +24,42 @@ jobs: maxParallel: 4 steps: - - script: | + #- script: | # Make sure all packages are pulled from latest - sudo apt-get update + #sudo apt-get update # Fail out if any setups fail - set -e + #set -e # Delete old Pythons - rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 - rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 - rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 # Download new Pythons - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ - --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ - --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 - azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ - --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + #azcopy --recursive \ + #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + #--destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 # Install new Pythons - original_directory=$PWD - setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) - for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; - done; - displayName: 'Workaround: update apt and roll back Python versions' - + #original_directory=$PWD + #setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + #for setup in $setups; do + #chmod +x $setup; + #cd $(dirname $setup); + #./$(basename $setup); + #cd $original_directory; + #done; + #displayName: 'Workaround: update apt and roll back Python versions' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' diff --git a/libs/custom/utorrent/client.py b/libs/custom/utorrent/client.py index f5eaf93b..ac1a9beb 100644 --- a/libs/custom/utorrent/client.py +++ b/libs/custom/utorrent/client.py @@ -59,7 +59,7 @@ class UTorrentClient(object): url = urljoin(self.base_url, 'token.html') response = self.opener.open(url) token_re = "" - match = re.search(token_re, response.read()) + match = re.search(token_re, str(response.read())) return match.group(1) def list(self, **kwargs): From bde5a15f660c82eb2b827710042b02158078fae6 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 6 Aug 2019 09:04:45 +1200 Subject: [PATCH 245/406] Fixes for user_script categories (#1645) Fixes for user_script categories. #1643 --- TorrentToMedia.py | 25 ++++++++++++++----------- core/user_scripts.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 42707b1a..39b08329 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -76,16 +76,19 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # auto-detect section section = core.CFG.findsection(input_category).isenabled() - if section is None: - section = core.CFG.findsection('ALL').isenabled() - if section is None: - logger.error('Category:[{0}] is not defined or is not enabled. ' - 'Please rename it or ensure it is enabled for the appropriate section ' - 'in your autoProcessMedia.cfg and try again.'.format - (input_category)) - return [-1, ''] + if section is None: #Check for user_scripts for 'ALL' and 'UNCAT' + if usercat in core.CATEGORIES: + section = core.CFG.findsection('ALL').isenabled() + usercat = 'ALL' else: - usercat = 'ALL' + section = core.CFG.findsection('UNCAT').isenabled() + usercat = 'UNCAT' + if section is None: # We haven't found any categories to process. + logger.error('Category:[{0}] is not defined or is not enabled. ' + 'Please rename it or ensure it is enabled for the appropriate section ' + 'in your autoProcessMedia.cfg and try again.'.format + (input_category)) + return [-1, ''] if len(section) > 1: logger.error('Category:[{0}] is not unique, {1} are using it. ' @@ -108,7 +111,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp torrent_no_link = int(section.get('Torrent_NoLink', 0)) keep_archive = int(section.get('keep_archive', 0)) extract = int(section.get('extract', 0)) - extensions = section.get('user_script_mediaExtensions', '').lower().split(',') + extensions = section.get('user_script_mediaExtensions', '') unique_path = int(section.get('unique_path', 1)) if client_agent != 'manual': @@ -278,7 +281,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp replace_links(os.path.join(dirpath, file)) core.remove_torrent(client_agent, input_hash, input_id, input_name) - if not section_name == 'UserScript': + if section_name != 'UserScript': # for user script, we assume this is cleaned by the script or option USER_SCRIPT_CLEAN # cleanup our processing folders of any misc unwanted files and empty directories core.clean_dir(output_destination, section_name, input_category) diff --git a/core/user_scripts.py b/core/user_scripts.py index 45adb45b..eaeb1b0f 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -14,33 +14,46 @@ import core from core import logger, transcoder from core.plugins.subtitles import import_subs from core.utils import list_media_files, remove_dir +from core.auto_process.common import ( + ProcessResult, +) + def external_script(output_destination, torrent_name, torrent_label, settings): final_result = 0 # start at 0. num_files = 0 + core.USER_SCRIPT_MEDIAEXTENSIONS = settings.get('user_script_mediaExtensions', '') try: - core.USER_SCRIPT_MEDIAEXTENSIONS = settings['user_script_mediaExtensions'].lower() if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): - core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',') + core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.lower().split(',') except Exception: + logger.error('user_script_mediaExtensions could not be set', 'USERSCRIPT') core.USER_SCRIPT_MEDIAEXTENSIONS = [] - core.USER_SCRIPT = settings.get('user_script_path') + core.USER_SCRIPT = settings.get('user_script_path', '') - if not core.USER_SCRIPT or core.USER_SCRIPT == 'None': # do nothing and return success. - return [0, ''] + if not core.USER_SCRIPT or core.USER_SCRIPT == 'None': + # do nothing and return success. This allows the user an option to Link files only and not run a script. + return ProcessResult( + status_code=0, + message='No user script defined', + ) + + core.USER_SCRIPT_PARAM = settings.get('user_script_param', '') try: - core.USER_SCRIPT_PARAM = settings['user_script_param'] if isinstance(core.USER_SCRIPT_PARAM, str): core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',') except Exception: + logger.error('user_script_params could not be set', 'USERSCRIPT') core.USER_SCRIPT_PARAM = [] + + core.USER_SCRIPT_SUCCESSCODES = settings.get('user_script_successCodes', 0) try: - core.USER_SCRIPT_SUCCESSCODES = settings['user_script_successCodes'] if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',') except Exception: + logger.error('user_script_successCodes could not be set', 'USERSCRIPT') core.USER_SCRIPT_SUCCESSCODES = 0 core.USER_SCRIPT_CLEAN = int(settings.get('user_script_clean', 1)) @@ -59,6 +72,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): file_path = core.os.path.join(dirpath, file) file_name, file_extension = os.path.splitext(file) + logger.debug('Checking file {0} to see if this should be processed.'.format(file), 'USERSCRIPT') if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or 'all' in core.USER_SCRIPT_MEDIAEXTENSIONS: num_files += 1 @@ -122,4 +136,7 @@ def external_script(output_destination, torrent_name, torrent_label, settings): elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0: logger.info('{0} files were processed, but {1} still remain. outputDirectory will not be cleaned.'.format( num_files, num_files_new)) - return [final_result, ''] + return ProcessResult( + status_code=final_result, + message='User Script Completed', + ) From dc5d43b028e28aa025b57a2d63bc3009d4c6d0a4 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 6 Aug 2019 13:16:25 +1200 Subject: [PATCH 246/406] update to version 12.1.00 --- .bumpversion.cfg | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 31885a91..cc1a1057 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.10 +current_version = 12.1.00 commit = True tag = False diff --git a/core/__init__.py b/core/__init__.py index e2436182..55b8acba 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -83,7 +83,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.0.10' +__version__ = '12.1.00' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index bac19586..a115dfa2 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.10', + version='12.1.00', license='GPLv3', description='Efficient on demand post processing', long_description=""" From e738727c523838ac5dc91d1f5cacdd35b450674c Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sat, 10 Aug 2019 19:35:50 +1200 Subject: [PATCH 247/406] Force status from SABnzbd to be integer. #1646 #1647 (#1648) --- nzbToMedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nzbToMedia.py b/nzbToMedia.py index a6139fe3..2ab16d59 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -923,7 +923,7 @@ def main(args, section=None): # 7 Status of post processing. 0 = OK, 1=failed verification, 2=failed unpack, 3=1+2 client_agent = 'sabnzbd' logger.info('Script triggered from SABnzbd') - result = process(args[1], input_name=args[2], status=args[7], input_category=args[5], client_agent=client_agent, + result = process(args[1], input_name=args[2], status=int(args[7]), input_category=args[5], client_agent=client_agent, download_id='') # SABnzbd 0.7.17+ elif len(args) >= core.SABNZB_0717_NO_OF_ARGUMENTS: @@ -938,7 +938,7 @@ def main(args, section=None): # 8 Failure URL client_agent = 'sabnzbd' logger.info('Script triggered from SABnzbd 0.7.17+') - result = process(args[1], input_name=args[2], status=args[7], input_category=args[5], client_agent=client_agent, + result = process(args[1], input_name=args[2], status=int(args[7]), input_category=args[5], client_agent=client_agent, download_id='', failure_link=''.join(args[8:])) # Generic program elif len(args) > 5 and args[5] == 'generic': From 77f34261fac7d0501fcedb61c8301512c4fb1efd Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 13 Aug 2019 18:38:05 +1200 Subject: [PATCH 248/406] update to v12.1.01 --- .bumpversion.cfg | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cc1a1057..1edaabcf 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.1.00 +current_version = 12.1.01 commit = True tag = False diff --git a/core/__init__.py b/core/__init__.py index 55b8acba..76798031 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -83,7 +83,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.1.00' +__version__ = '12.1.01' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index a115dfa2..5e79ab2c 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.1.00', + version='12.1.01', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 80ef0d094e1c316ff9e688d34489b4a1846c6895 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 19 Sep 2019 20:47:13 +1200 Subject: [PATCH 249/406] Fix autofork fallback. #163 --- core/forks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/forks.py b/core/forks.py index f49ed5a9..ac73c547 100644 --- a/core/forks.py +++ b/core/forks.py @@ -168,7 +168,8 @@ def auto_fork(section, input_category): else: logger.info('{section}:{category} fork auto-detection failed'.format (section=section, category=input_category)) - fork = core.FORKS.items()[core.FORKS.keys().index(core.FORK_DEFAULT)] + fork = list(core.FORKS.items())[list(core.FORKS.keys()).index(core.FORK_DEFAULT)] + logger.info('{section}:{category} fork set to {fork}'.format (section=section, category=input_category, fork=fork[0])) From 1814bd5ae1a40885217e7a4427768b689b93542b Mon Sep 17 00:00:00 2001 From: Sergio Cambra Date: Mon, 4 Nov 2019 00:05:00 +0100 Subject: [PATCH 250/406] add watcher3 integration (#1665) --- TorrentToMedia.py | 8 ++--- _config.yml | 1 + autoProcessMedia.cfg.spec | 60 +++++++++++++++++++++++++++---------- core/auto_process/movies.py | 41 ++++++++++++++++++++++++- nzbToMedia.py | 4 +-- 5 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 _config.yml diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 39b08329..28f09d76 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -79,7 +79,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if section is None: #Check for user_scripts for 'ALL' and 'UNCAT' if usercat in core.CATEGORIES: section = core.CFG.findsection('ALL').isenabled() - usercat = 'ALL' + usercat = 'ALL' else: section = core.CFG.findsection('UNCAT').isenabled() usercat = 'UNCAT' @@ -213,7 +213,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp core.flatten(output_destination) # Now check if video files exist in destination: - if section_name in ['SickBeard', 'NzbDrone', 'Sonarr', 'CouchPotato', 'Radarr']: + if section_name in ['SickBeard', 'NzbDrone', 'Sonarr', 'CouchPotato', 'Radarr', 'Watcher3']: num_videos = len( core.list_media_files(output_destination, media=True, audio=False, meta=False, archives=False)) if num_videos > 0: @@ -227,7 +227,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp # Only these sections can handling failed downloads # so make sure everything else gets through without the check for failed - if section_name not in ['CouchPotato', 'Radarr', 'SickBeard', 'NzbDrone', 'Sonarr']: + if section_name not in ['CouchPotato', 'Radarr', 'SickBeard', 'NzbDrone', 'Sonarr', 'Watcher3']: status = 0 logger.info('Calling {0}:{1} to post-process:{2}'.format(section_name, usercat, input_name)) @@ -241,7 +241,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp ) if section_name == 'UserScript': result = external_script(output_destination, input_name, input_category, section) - elif section_name in ['CouchPotato', 'Radarr']: + elif section_name in ['CouchPotato', 'Radarr', 'Watcher3']: result = movies.process(section_name, output_destination, input_name, status, client_agent, input_hash, input_category) elif section_name in ['SickBeard', 'NzbDrone', 'Sonarr']: if input_hash: diff --git a/_config.yml b/_config.yml new file mode 100644 index 00000000..c4192631 --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-cayman \ No newline at end of file diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index e3d11cb6..7af48e01 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -12,7 +12,7 @@ git_user = # GitHUB branch for repo git_branch = - # Enable/Disable forceful cleaning of leftover files following postprocess + # Enable/Disable forceful cleaning of leftover files following postprocess force_clean = 0 # Enable/Disable logging debug messages to nzbtomedia.log log_debug = 0 @@ -36,7 +36,7 @@ [Posix] ### Process priority setting for External commands (Extractor and Transcoder) on Posix (Unix/Linux/OSX) systems. # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). - # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). + # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). # If entering a comma separated list e.g. 'niceness = nice,4' this will be passed as 'nice 4' (Safer). niceness = nice,-n0 # Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. @@ -111,6 +111,36 @@ ##### Set to define import behavior Move or Copy importMode = Copy +[Watcher3] + #### autoProcessing for Movies + #### movie - category that gets called for post-processing with CPS + [[movie]] + enabled = 0 + apikey = + host = localhost + port = 9090 + ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### + ssl = 0 + web_root = + # api key for www.omdbapi.com (used as alternative to imdb) + omdbapikey = + # Enable/Disable linking for Torrents + Torrent_NoLink = 0 + keep_archive = 1 + delete_failed = 0 + wait_for = 0 + extract = 1 + # Set this to minimum required size to consider a media file valid (in MB) + minSize = 0 + # Enable/Disable deleting ignored files (samples and invalid media files) + delete_ignored = 0 + ##### Enable if Watcher3 is on a remote server for this category + remote_path = 0 + ##### Set to path where download client places completed downloads locally for this category + watch_dir = + ##### Set the recursive directory permissions to the following (0 to disable) + chmodDirectory = 0 + [SickBeard] #### autoProcessing for TV Series #### tv - category that gets called for post-processing with SB @@ -266,7 +296,7 @@ apikey = host = localhost port = 8085 - ###### + ###### library = Set to path where you want the processed games to be moved to. ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### ssl = 0 @@ -312,7 +342,7 @@ [Network] # Enter Mount points as LocalPath,RemotePath and separate each pair with '|' # e.g. MountPoints = /volume1/Public/,E:\|/volume2/share/,\\NAS\ - mount_points = + mount_points = [Nzb] ###### clientAgent - Supported clients: sabnzbd, nzbget @@ -331,7 +361,7 @@ useLink = hard ###### outputDirectory - Default output directory (categories will be appended as sub directory to outputDirectory) outputDirectory = /abs/path/to/complete/ - ###### Enter the default path to your default download directory (non-category downloads). this directory is protected by safe_mode. + ###### Enter the default path to your default download directory (non-category downloads). this directory is protected by safe_mode. default_downloadDirectory = ###### Other categories/labels defined for your downloader. Does not include CouchPotato, SickBeard, HeadPhones, Mylar categories. categories = music_videos,pictures,software,manual @@ -374,15 +404,15 @@ plex_host = localhost plex_port = 32400 plex_token = - plex_ssl = 0 + plex_ssl = 0 # Enter Plex category to section mapping as Category,section and separate each pair with '|' # e.g. plex_sections = movie,3|tv,4 - plex_sections = + plex_sections = [Transcoder] # getsubs. enable to download subtitles. getSubs = 0 - # subLanguages. create a list of languages in the order you want them in your subtitles. + # subLanguages. create a list of languages in the order you want them in your subtitles. subLanguages = eng,spa,fra # transcode. enable to use transcoder transcode = 0 @@ -397,7 +427,7 @@ # outputQualityPercent. used as -q:a value. 0 will disable this from being used. outputQualityPercent = 0 # outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. - outputVideoPath = + outputVideoPath = # processOutput. 1 will send the outputVideoPath to SickBeard/CouchPotato. 0 will send original files. processOutput = 0 # audioLanguage. set the 3 letter language code you want as your primary audio track. @@ -427,7 +457,7 @@ #### Define custom settings below. outputVideoExtension = .mp4 outputVideoCodec = libx264 - VideoCodecAllow = + VideoCodecAllow = outputVideoPreset = medium outputVideoResolution = 1920:1080 outputVideoFramerate = 24 @@ -435,15 +465,15 @@ outputVideoCRF = 19 outputVideoLevel = 3.1 outputAudioCodec = ac3 - AudioCodecAllow = + AudioCodecAllow = outputAudioChannels = 6 outputAudioBitrate = 640k outputAudioTrack2Codec = libfaac - AudioCodec2Allow = - outputAudioTrack2Channels = 2 + AudioCodec2Allow = + outputAudioTrack2Channels = 2 outputAudioTrack2Bitrate = 128000 outputAudioOtherCodec = libmp3lame - AudioOtherCodecAllow = + AudioOtherCodecAllow = outputAudioOtherChannels = outputAudioOtherBitrate = 128000 outputSubtitleCodec = @@ -500,4 +530,4 @@ # enter a list (comma separated) of Group Tags you want removed from filenames to help with subtitle matching. # e.g remove_group = [rarbag],-NZBgeek # be careful if your "group" is a common "real" word. Please report if you have any group replacements that would fall in this category. - remove_group = + remove_group = diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 2661edd1..e7439aa0 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -72,6 +72,8 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', base_url = '{0}{1}:{2}{3}/api/command'.format(protocol, host, port, web_root) url2 = '{0}{1}:{2}{3}/api/config/downloadClient'.format(protocol, host, port, web_root) headers = {'X-Api-Key': apikey} + if section == 'Watcher3': + base_url = '{0}{1}:{2}{3}/postprocessing'.format(protocol, host, port, web_root) if not apikey: logger.info('No CouchPotato or Radarr apikey entered. Performing transcoder functions only') release = None @@ -178,7 +180,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', os.rename(video, video2) if not apikey: # If only using Transcoder functions, exit here. - logger.info('No CouchPotato or Radarr apikey entered. Processing completed.') + logger.info('No CouchPotato or Radarr or Watcher3 apikey entered. Processing completed.') return ProcessResult( message='{0}: Successfully post-processed {1}'.format(section, input_name), status_code=0, @@ -210,9 +212,20 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', logger.debug('Opening URL: {0} with PARAMS: {1}'.format(base_url, payload), section) logger.postprocess('Starting DownloadedMoviesScan scan for {0}'.format(input_name), section) + if section == 'Watcher3': + if input_name and os.path.isfile(os.path.join(dir_name, input_name)): + params['media_folder'] = os.path.join(params['media_folder'], input_name) + payload = {'apikey': apikey, 'path': params['media_folder'], 'guid': download_id, 'mode': 'complete'} + if not download_id: + payload.pop('guid') + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(base_url, payload), section) + logger.postprocess('Starting postprocessing scan for {0}'.format(input_name), section) + try: if section == 'CouchPotato': r = requests.get(url, params=params, verify=False, timeout=(30, 1800)) + elif section == 'Watcher3': + r = requests.post(base_url, data=payload, verify=False, timeout=(30, 1800)) else: r = requests.post(base_url, data=json.dumps(payload), headers=headers, stream=True, verify=False, timeout=(30, 1800)) except requests.ConnectionError: @@ -245,6 +258,18 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', except Exception as e: logger.warning('No scan id was returned due to: {0}'.format(e), section) scan_id = None + elif section == 'Watcher3' and result['status'] == 'finished': + logger.postprocess('Watcher3 updated status to {0}'.format(result['tasks']['update_movie_status'])) + if result['tasks']['update_movie_status'] == 'Finished': + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=status, + ) + else: + return ProcessResult( + message='{0}: Failed to post-process - changed status to {1}'.format(section, result['tasks']['update_movie_status']), + status_code=1, + ) else: logger.error('FAILED: {0} scan was unable to finish for folder {1}. exiting!'.format(method, dir_name), section) @@ -264,6 +289,20 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', message='{0}: Sending failed download back to {0}'.format(section), status_code=1, # Return as failed to flag this in the downloader. ) # Return failed flag, but log the event as successful. + elif section == 'Watcher3': + logger.postprocess('Sending failed download to {0} for CDH processing'.format(section), section) + path = remote_dir(dir_name) if remote_path else dir_name + if input_name and os.path.isfile(os.path.join(dir_name, input_name)): + path = os.path.join(path, input_name) + payload = {'apikey': apikey, 'path': path, 'guid': download_id, 'mode': 'failed'} + r = requests.post(base_url, data=payload, verify=False, timeout=(30, 1800)) + result = r.json() + logger.postprocess('Watcher3 response: {0}'.format(result)) + if result['status'] == 'finished': + return ProcessResult( + message='{0}: Sending failed download back to {0}'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) # Return failed flag, but log the event as successful. if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) diff --git a/nzbToMedia.py b/nzbToMedia.py index 2ab16d59..1c04cd7b 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -5,7 +5,7 @@ ### NZBGET POST-PROCESSING SCRIPT ### # Post-Process to CouchPotato, SickBeard, Sonarr, Mylar, Gamez, HeadPhones, -# LazyLibrarian, Radarr, Lidarr +# LazyLibrarian, Radarr, Lidarr, Watcher3 # # This script sends the download to your automated media management servers. # @@ -799,7 +799,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d logger.info('Calling {0}:{1} to post-process:{2}'.format(section_name, input_category, input_name)) - if section_name in ['CouchPotato', 'Radarr']: + if section_name in ['CouchPotato', 'Radarr', 'Watcher3']: result = movies.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link) elif section_name in ['SickBeard', 'NzbDrone', 'Sonarr']: result = tv.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link) From c92588c3bea0b614a4b70cc77d036139c5775f80 Mon Sep 17 00:00:00 2001 From: Sergio Cambra Date: Mon, 4 Nov 2019 00:10:20 +0100 Subject: [PATCH 251/406] fix downloading subtitles, no provider was registered (#1664) --- core/plugins/subtitles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/plugins/subtitles.py b/core/plugins/subtitles.py index df37e532..181975e5 100644 --- a/core/plugins/subtitles.py +++ b/core/plugins/subtitles.py @@ -11,6 +11,10 @@ import subliminal import core from core import logger +for provider in subliminal.provider_manager.internal_extensions: + if provider not in [str(x) for x in subliminal.provider_manager.list_entry_points()]: + subliminal.provider_manager.register(str(provider)) + def import_subs(filename): if not core.GETSUBS: From fde87148627ff66650201087dab840dfd6fb5d70 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Nov 2019 12:28:35 +1300 Subject: [PATCH 252/406] Update all qBittorrent WebAPI paths for client v4.1.0+ (#1666) --- libs/common/qbittorrent/client.py | 234 +++++++++++------------------- 1 file changed, 83 insertions(+), 151 deletions(-) diff --git a/libs/common/qbittorrent/client.py b/libs/common/qbittorrent/client.py index 73d8d753..08df1e63 100644 --- a/libs/common/qbittorrent/client.py +++ b/libs/common/qbittorrent/client.py @@ -1,7 +1,6 @@ import requests import json - class LoginRequired(Exception): def __str__(self): return 'Please login first.' @@ -15,7 +14,7 @@ class Client(object): self.url = url session = requests.Session() - check_prefs = session.get(url+'query/preferences') + check_prefs = session.get(url+'api/v2/app/preferences') if check_prefs.status_code == 200: self._is_authenticated = True @@ -24,9 +23,9 @@ class Client(object): elif check_prefs.status_code == 404: self._is_authenticated = False raise RuntimeError(""" - This wrapper only supports qBittorrent applications - with version higher than 3.1.x. - Please use the latest qBittorrent release. + This wrapper only supports qBittorrent applications with + version higher than 4.1.0 (which implemented Web API v2.0). + Please use the latest qBittorrent release. """) else: @@ -35,10 +34,8 @@ class Client(object): def _get(self, endpoint, **kwargs): """ Method to perform GET request on the API. - :param endpoint: Endpoint of the API. :param kwargs: Other keyword arguments for requests. - :return: Response of the GET request. """ return self._request(endpoint, 'get', **kwargs) @@ -46,11 +43,9 @@ class Client(object): def _post(self, endpoint, data, **kwargs): """ Method to perform POST request on the API. - :param endpoint: Endpoint of the API. :param data: POST DATA for the request. :param kwargs: Other keyword arguments for requests. - :return: Response of the POST request. """ return self._request(endpoint, 'post', data, **kwargs) @@ -58,12 +53,10 @@ class Client(object): def _request(self, endpoint, method, data=None, **kwargs): """ Method to hanle both GET and POST requests. - :param endpoint: Endpoint of the API. :param method: Method of HTTP request. :param data: POST DATA for the request. :param kwargs: Other keyword arguments. - :return: Response for the request. """ final_url = self.url + endpoint @@ -93,18 +86,15 @@ class Client(object): def login(self, username='admin', password='admin'): """ Method to authenticate the qBittorrent Client. - Declares a class attribute named ``session`` which stores the authenticated session if the login is correct. Else, shows the login error. - :param username: Username. :param password: Password. - :return: Response to login request to the API. """ self.session = requests.Session() - login = self.session.post(self.url+'login', + login = self.session.post(self.url+'api/v2/auth/login', data={'username': username, 'password': password}) if login.text == 'Ok.': @@ -116,7 +106,7 @@ class Client(object): """ Logout the current session. """ - response = self._get('logout') + response = self._get('api/v2/auth/logout') self._is_authenticated = False return response @@ -125,39 +115,31 @@ class Client(object): """ Get qBittorrent version. """ - return self._get('version/qbittorrent') + return self._get('api/v2/app/version') @property def api_version(self): """ Get WEB API version. """ - return self._get('version/api') - - @property - def api_min_version(self): - """ - Get minimum WEB API version. - """ - return self._get('version/api_min') + return self._get('api/v2/app/webapiVersion') def shutdown(self): """ Shutdown qBittorrent. """ - return self._get('command/shutdown') + return self._get('api/v2/app/shutdown') def torrents(self, **filters): """ Returns a list of torrents matching the supplied filters. - :param filter: Current status of the torrents. :param category: Fetch all torrents with the supplied label. :param sort: Sort torrents by. :param reverse: Enable reverse sorting. :param limit: Limit the number of torrents returned. :param offset: Set offset (if less than 0, offset from end). - + :param hashes: Filter by hashes. Can contain multiple hashes separated by |. :return: list() of torrent with matching filter. """ params = {} @@ -166,46 +148,42 @@ class Client(object): name = 'filter' if name == 'status' else name params[name] = value - return self._get('query/torrents', params=params) + return self._get('api/v2/torrents/info', params=params) def get_torrent(self, infohash): """ Get details of the torrent. - :param infohash: INFO HASH of the torrent. """ - return self._get('query/propertiesGeneral/' + infohash.lower()) + return self._get('api/v2/torrents/properties', params={'hash': infohash.lower()}) def get_torrent_trackers(self, infohash): """ Get trackers for the torrent. - :param infohash: INFO HASH of the torrent. """ - return self._get('query/propertiesTrackers/' + infohash.lower()) + return self._get('api/v2/torrents/trackers', params={'hash': infohash.lower()}) def get_torrent_webseeds(self, infohash): """ Get webseeds for the torrent. - :param infohash: INFO HASH of the torrent. """ - return self._get('query/propertiesWebSeeds/' + infohash.lower()) + return self._get('api/v2/torrents/webseeds', params={'hash': infohash.lower()}) def get_torrent_files(self, infohash): """ Get list of files for the torrent. - :param infohash: INFO HASH of the torrent. """ - return self._get('query/propertiesFiles/' + infohash.lower()) + return self._get('api/v2/torrents/files', params={'hash': infohash.lower()}) @property def global_transfer_info(self): """ Get JSON data of the global transfer info of qBittorrent. """ - return self._get('query/transferInfo') + return self._get('api/v2/transfer/info') @property def preferences(self): @@ -214,39 +192,27 @@ class Client(object): Can also be used to assign individual preferences. For setting multiple preferences at once, see ``set_preferences`` method. - Note: Even if this is a ``property``, to fetch the current preferences dict, you are required to call it like a bound method. - Wrong:: - qb.preferences - Right:: - qb.preferences() - """ - prefs = self._get('query/preferences') + prefs = self._get('api/v2/app/preferences') class Proxy(Client): """ Proxy class to to allow assignment of individual preferences. this class overrides some methods to ease things. - Because of this, settings can be assigned like:: - In [5]: prefs = qb.preferences() - In [6]: prefs['autorun_enabled'] Out[6]: True - In [7]: prefs['autorun_enabled'] = False - In [8]: prefs['autorun_enabled'] Out[8]: False - """ def __init__(self, url, prefs, auth, session): @@ -270,78 +236,74 @@ class Client(object): def sync(self, rid=0): """ Sync the torrents by supplied LAST RESPONSE ID. - Read more @ http://git.io/vEgXr - + Read more @ https://git.io/fxgB8 :param rid: Response ID of last request. """ - return self._get('sync/maindata', params={'rid': rid}) + return self._get('api/v2/sync/maindata', params={'rid': rid}) def download_from_link(self, link, **kwargs): """ Download torrent using a link. - :param link: URL Link or list of. :param savepath: Path to download the torrent. :param category: Label or Category of the torrent(s). - :return: Empty JSON data. """ - # old:new format - old_arg_map = {'save_path': 'savepath'} # , 'label': 'category'} - - # convert old option names to new option names - options = kwargs.copy() - for old_arg, new_arg in old_arg_map.items(): - if options.get(old_arg) and not options.get(new_arg): - options[new_arg] = options[old_arg] - - options['urls'] = link - - # workaround to send multipart/formdata request - # http://stackoverflow.com/a/23131823/4726598 - dummy_file = {'_dummy': (None, '_dummy')} - - return self._post('command/download', data=options, files=dummy_file) + # qBittorrent requires adds to be done with multipath/form-data + # POST requests for both URLs and .torrent files. Info on this + # can be found here, and here: + # http://docs.python-requests.org/en/master/user/quickstart/#post-a-multipart-encoded-file + # http://docs.python-requests.org/en/master/user/advanced/#post-multiple-multipart-encoded-files + if isinstance(link, list): + links = '\n'.join(link) + else: + links = link + torrent_data = {} + torrent_data['urls'] = (None, links) + for k, v in kwargs.iteritems(): + torrent_data[k] = (None, v) + return self._post('api/v2/torrents/add', data=None, files=torrent_data) def download_from_file(self, file_buffer, **kwargs): """ Download torrent using a file. - :param file_buffer: Single file() buffer or list of. :param save_path: Path to download the torrent. :param label: Label of the torrent(s). - :return: Empty JSON data. """ + # qBittorrent requires adds to be done with multipath/form-data + # POST requests for both URLs and .torrent files. Info on this + # can be found here, and here: + # http://docs.python-requests.org/en/master/user/quickstart/#post-a-multipart-encoded-file + # http://docs.python-requests.org/en/master/user/advanced/#post-multiple-multipart-encoded-files if isinstance(file_buffer, list): - torrent_files = {} - for i, f in enumerate(file_buffer): - torrent_files.update({'torrents%s' % i: f}) + torrent_data = [] + for f in file_buffer: + fname = f.name + torrent_data.append(('torrents', (fname, f))) else: - torrent_files = {'torrents': file_buffer} + fname = file_buffer.name + torrent_data = [('torrents', (fname, file_buffer))] + for k, v in kwargs.iteritems(): + torrent_data.append((k, (None, v))) - data = kwargs.copy() - - if data.get('save_path'): - data.update({'savepath': data['save_path']}) - return self._post('command/upload', data=data, files=torrent_files) + return self._post('api/v2/torrents/add', data=None, files=torrent_data) def add_trackers(self, infohash, trackers): """ Add trackers to a torrent. - :param infohash: INFO HASH of torrent. :param trackers: Trackers. """ data = {'hash': infohash.lower(), 'urls': trackers} - return self._post('command/addTrackers', data=data) + return self._post('api/v2/torrents/addTrackers', data=data) @staticmethod def _process_infohash_list(infohash_list): """ Method to convert the infohash_list to qBittorrent API friendly values. - :param infohash_list: List of infohash. """ if isinstance(infohash_list, list): @@ -353,142 +315,122 @@ class Client(object): def pause(self, infohash): """ Pause a torrent. - :param infohash: INFO HASH of torrent. """ - return self._post('command/pause', data={'hash': infohash.lower()}) + return self._post('api/v2/torrents/pause', data={'hashes': infohash.lower()}) def pause_all(self): """ Pause all torrents. """ - return self._get('command/pauseAll') + return self._post('api/v2/torrents/pause', data={'hashes': 'all'}) def pause_multiple(self, infohash_list): """ Pause multiple torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/pauseAll', data=data) - - def set_label(self, infohash_list, label): - """ - Set the label on multiple torrents. - IMPORTANT: OLD API method, kept as it is to avoid breaking stuffs. - - :param infohash_list: Single or list() of infohashes. - """ - data = self._process_infohash_list(infohash_list) - data['label'] = label - return self._post('command/setLabel', data=data) + return self._post('api/v2/torrents/pause', data=data) def set_category(self, infohash_list, category): """ Set the category on multiple torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) data['category'] = category - return self._post('command/setCategory', data=data) + return self._post('api/v2/torrents/setCategory', data=data) def resume(self, infohash): """ Resume a paused torrent. - :param infohash: INFO HASH of torrent. """ - return self._post('command/resume', data={'hash': infohash.lower()}) + return self._post('api/v2/torrents/resume', data={'hashes': infohash.lower()}) def resume_all(self): """ Resume all torrents. """ - return self._get('command/resumeAll') + return self._get('api/v2/torrents/resume', data={'hashes': 'all'}) def resume_multiple(self, infohash_list): """ Resume multiple paused torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/resumeAll', data=data) + return self._post('api/v2/torrents/resume', data=data) def delete(self, infohash_list): """ Delete torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/delete', data=data) + data['deleteFiles'] = 'false' + return self._post('api/v2/torrents/delete', data=data) def delete_permanently(self, infohash_list): """ Permanently delete torrents. - + *** WARNING : This will instruct qBittorrent to delete files + *** from your hard disk. Use with caution. :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/deletePerm', data=data) + data['deleteFiles'] = 'true' + return self._post('api/v2/torrents/delete', data=data) def recheck(self, infohash_list): """ Recheck torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/recheck', data=data) + return self._post('api/v2/torrents/recheck', data=data) def increase_priority(self, infohash_list): """ Increase priority of torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/increasePrio', data=data) + return self._post('api/v2/torrents/increasePrio', data=data) def decrease_priority(self, infohash_list): """ Decrease priority of torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/decreasePrio', data=data) + return self._post('api/v2/torrents/decreasePrio', data=data) def set_max_priority(self, infohash_list): """ Set torrents to maximum priority level. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/topPrio', data=data) + return self._post('api/v2/torrents/topPrio', data=data) def set_min_priority(self, infohash_list): """ Set torrents to minimum priority level. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/bottomPrio', data=data) + return self._post('api/v2/torrents/bottomPrio', data=data) def set_file_priority(self, infohash, file_id, priority): """ Set file of a torrent to a supplied priority level. - :param infohash: INFO HASH of torrent. :param file_id: ID of the file to set priority. :param priority: Priority level of the file. """ - if priority not in [0, 1, 2, 7]: + if priority not in [0, 1, 6, 7]: raise ValueError("Invalid priority, refer WEB-UI docs for info.") elif not isinstance(file_id, int): raise TypeError("File ID must be an int") @@ -497,7 +439,7 @@ class Client(object): 'id': file_id, 'priority': priority} - return self._post('command/setFilePrio', data=data) + return self._post('api/v2/torrents/filePrio', data=data) # Get-set global download and upload speed limits. @@ -505,15 +447,14 @@ class Client(object): """ Get global download speed limit. """ - return self._get('command/getGlobalDlLimit') + return self._get('api/v2/transfer/downloadLimit') def set_global_download_limit(self, limit): """ Set global download speed limit. - :param limit: Speed limit in bytes. """ - return self._post('command/setGlobalDlLimit', data={'limit': limit}) + return self._post('api/v2/transfer/setDownloadLimit', data={'limit': limit}) global_download_limit = property(get_global_download_limit, set_global_download_limit) @@ -522,15 +463,14 @@ class Client(object): """ Get global upload speed limit. """ - return self._get('command/getGlobalUpLimit') + return self._get('api/v2/transfer/uploadLimit') def set_global_upload_limit(self, limit): """ Set global upload speed limit. - :param limit: Speed limit in bytes. """ - return self._post('command/setGlobalUpLimit', data={'limit': limit}) + return self._post('api/v2/transfer/setUploadLimit', data={'limit': limit}) global_upload_limit = property(get_global_upload_limit, set_global_upload_limit) @@ -539,61 +479,56 @@ class Client(object): def get_torrent_download_limit(self, infohash_list): """ Get download speed limit of the supplied torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/getTorrentsDlLimit', data=data) + return self._post('api/v2/torrents/downloadLimit', data=data) def set_torrent_download_limit(self, infohash_list, limit): """ Set download speed limit of the supplied torrents. - :param infohash_list: Single or list() of infohashes. :param limit: Speed limit in bytes. """ data = self._process_infohash_list(infohash_list) data.update({'limit': limit}) - return self._post('command/setTorrentsDlLimit', data=data) + return self._post('api/v2/torrents/setDownloadLimit', data=data) def get_torrent_upload_limit(self, infohash_list): """ Get upoload speed limit of the supplied torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/getTorrentsUpLimit', data=data) + return self._post('api/v2/torrents/uploadLimit', data=data) def set_torrent_upload_limit(self, infohash_list, limit): """ Set upload speed limit of the supplied torrents. - :param infohash_list: Single or list() of infohashes. :param limit: Speed limit in bytes. """ data = self._process_infohash_list(infohash_list) data.update({'limit': limit}) - return self._post('command/setTorrentsUpLimit', data=data) + return self._post('api/v2/torrents/setUploadLimit', data=data) # setting preferences def set_preferences(self, **kwargs): """ Set preferences of qBittorrent. - Read all possible preferences @ http://git.io/vEgDQ - + Read all possible preferences @ https://git.io/fx2Y9 :param kwargs: set preferences in kwargs form. """ json_data = "json={}".format(json.dumps(kwargs)) headers = {'content-type': 'application/x-www-form-urlencoded'} - return self._post('command/setPreferences', data=json_data, + return self._post('api/v2/app/setPreferences', data=json_data, headers=headers) def get_alternative_speed_status(self): """ Get Alternative speed limits. (1/0) """ - return self._get('command/alternativeSpeedLimitsEnabled') + return self._get('api/v2/transfer/speedLimitsMode') alternative_speed_status = property(get_alternative_speed_status) @@ -601,33 +536,30 @@ class Client(object): """ Toggle alternative speed limits. """ - return self._get('command/toggleAlternativeSpeedLimits') + return self._get('api/v2/transfer/toggleSpeedLimitsMode') def toggle_sequential_download(self, infohash_list): """ Toggle sequential download in supplied torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/toggleSequentialDownload', data=data) + return self._post('api/v2/torrents/toggleSequentialDownload', data=data) def toggle_first_last_piece_priority(self, infohash_list): """ Toggle first/last piece priority of supplied torrents. - :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - return self._post('command/toggleFirstLastPiecePrio', data=data) + return self._post('api/v2/torrents/toggleFirstLastPiecePrio', data=data) def force_start(self, infohash_list, value=True): """ Force start selected torrents. - :param infohash_list: Single or list() of infohashes. :param value: Force start value (bool) """ data = self._process_infohash_list(infohash_list) data.update({'value': json.dumps(value)}) - return self._post('command/setForceStart', data=data) + return self._post('api/v2/torrents/setForceStart', data=data) From 70ab7d3d611475d0085082aade49d7163e6eb0d1 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Nov 2019 13:17:38 +1300 Subject: [PATCH 253/406] Add Watcher3 Config (#1667) * Set NZBGet config #1665 --- core/configuration.py | 37 +++++- nzbToMedia.py | 48 ++++++++ nzbToWatcher3.py | 268 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 nzbToWatcher3.py diff --git a/core/configuration.py b/core/configuration.py index c8e2a465..13dc8f72 100644 --- a/core/configuration.py +++ b/core/configuration.py @@ -150,7 +150,7 @@ class ConfigObj(configobj.ConfigObj, Section): if CFG_OLD[section].sections: subsections.update({section: CFG_OLD[section].sections}) for option, value in CFG_OLD[section].items(): - if option in ['category', 'cpsCategory', 'sbCategory', 'hpCategory', 'mlCategory', 'gzCategory', 'raCategory', 'ndCategory']: + if option in ['category', 'cpsCategory', 'sbCategory', 'hpCategory', 'mlCategory', 'gzCategory', 'raCategory', 'ndCategory', 'W3Category']: if not isinstance(value, list): value = [value] @@ -271,6 +271,16 @@ class ConfigObj(configobj.ConfigObj, Section): logger.warning('{x} category is set for CouchPotato and Radarr. ' 'Please check your config in NZBGet'.format (x=os.environ['NZBPO_RACATEGORY'])) + if 'NZBPO_RACATEGORY' in os.environ and 'NZBPO_W3CATEGORY' in os.environ: + if os.environ['NZBPO_RACATEGORY'] == os.environ['NZBPO_W3CATEGORY']: + logger.warning('{x} category is set for Watcher3 and Radarr. ' + 'Please check your config in NZBGet'.format + (x=os.environ['NZBPO_RACATEGORY'])) + if 'NZBPO_W3CATEGORY' in os.environ and 'NZBPO_CPSCATEGORY' in os.environ: + if os.environ['NZBPO_W3CATEGORY'] == os.environ['NZBPO_CPSCATEGORY']: + logger.warning('{x} category is set for CouchPotato and Watcher3. ' + 'Please check your config in NZBGet'.format + (x=os.environ['NZBPO_W3CATEGORY'])) if 'NZBPO_LICATEGORY' in os.environ and 'NZBPO_HPCATEGORY' in os.environ: if os.environ['NZBPO_LICATEGORY'] == os.environ['NZBPO_HPCATEGORY']: logger.warning('{x} category is set for HeadPhones and Lidarr. ' @@ -321,6 +331,29 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['Radarr'].sections: cfg_new['Radarr'][env_cat_key]['enabled'] = 0 + if os.environ[env_cat_key] in cfg_new['Watcher3'].sections: + cfg_new['Watcher3'][env_cat_key]['enabled'] = 0 + + section = 'Watcher3' + env_cat_key = 'NZBPO_W3CATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', + 'WAIT_FOR', 'WATCH_DIR', 'OMDBAPIKEY'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', + 'wait_for', 'watch_dir', 'omdbapikey'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_W3{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['Radarr'].sections: + cfg_new['Radarr'][env_cat_key]['enabled'] = 0 + if os.environ[env_cat_key] in cfg_new['CouchPotato'].sections: + cfg_new['CouchPotato'][env_cat_key]['enabled'] = 0 section = 'SickBeard' env_cat_key = 'NZBPO_SBCATEGORY' @@ -444,6 +477,8 @@ class ConfigObj(configobj.ConfigObj, Section): cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 if os.environ[env_cat_key] in cfg_new['CouchPotato'].sections: cfg_new['CouchPotato'][env_cat_key]['enabled'] = 0 + if os.environ[env_cat_key] in cfg_new['Wacther3'].sections: + cfg_new['Watcher3'][env_cat_key]['enabled'] = 0 section = 'Lidarr' env_cat_key = 'NZBPO_LICATEGORY' diff --git a/nzbToMedia.py b/nzbToMedia.py index 1c04cd7b..073cb6f1 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -142,6 +142,54 @@ # Enable to replace local path with the path as per the mountPoints below. #raremote_path=0 +## Watcher3 + +# Wather3 script category. +# +# category that gets called for post-processing with Watcher3. +#W3Category=movie + +# Watcher3 api key. +#W3apikey= + +# Watcher3 host. +# +# The ipaddress for your Watcher3 server. e.g For the Same system use localhost or 127.0.0.1 +#W3host=localhost + +# Watcher3 port. +#W3port=9090 + +# Watcher3 uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#W3ssl=0 + +# Watcher3 URL_Base +# +# set this if using a reverse proxy. +#W3web_root= + +# OMDB API Key. +# +# api key for www.omdbapi.com (used as alternative to imdb to assist with movie identification). +#W3omdbapikey= + +# Wacther3 Delete Failed Downloads (0, 1). +# +# set to 1 to delete failed, or 0 to leave files in place. +#W3delete_failed=0 + +# Wacther3 wait_for +# +# Set the number of minutes to wait after calling the renamer, to check the movie has changed status. +#W3wait_for=2 + +# Watcher3 and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#W3remote_path=0 + ## SickBeard # SickBeard script category. diff --git a/nzbToWatcher3.py b/nzbToWatcher3.py new file mode 100644 index 00000000..6fafcba2 --- /dev/null +++ b/nzbToWatcher3.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python +# coding=utf-8 +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Post-Process to Watcher3 +# +# This script sends the download to your automated media management servers. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### + +## General + +# Auto Update nzbToMedia (0, 1). +# +# Set to 1 if you want nzbToMedia to automatically check for and update to the latest version +#auto_update=0 + +# Check Media for corruption (0, 1). +# +# Enable/Disable media file checking using ffprobe. +#check_media=1 + +# Safe Mode protection of DestDir (0, 1). +# +# Enable/Disable a safety check to ensure we don't process all downloads in the default_downloadDirectory by mistake. +#safe_mode=1 + +# Disable additional extraction checks for failed (0, 1). +# +# Turn this on to disable additional extraction attempts for failed downloads. Default = 0 this will attempt to extract and verify if media is present. +#no_extract_failed = 0 + +## Watcher3 + +# Watcher3 script category. +# +# category that gets called for post-processing with Watcher3. +#W3Category=movie + +# Watcher3 api key. +#W3apikey= + +# Watcher3 host. +# +# The ipaddress for your Watcher3 server. e.g For the Same system use localhost or 127.0.0.1 +#W3host=localhost + +# Watcher3 port. +#W3port=5050 + +# Watcher3 uses ssl (0, 1). +# +# Set to 1 if using ssl, else set to 0. +#W3ssl=0 + +# Watcher3 URL_Base +# +# set this if using a reverse proxy. +#W3web_root= + +# Watcher3 watch directory. +# +# set this to where your Watcher3 completed downloads are. +#W3watch_dir= + +# OMDB API Key. +# +# api key for www.omdbapi.com (used as alternative to imdb to assist with movie identification). +#W3omdbapikey= + +# Watcher3 Delete Failed Downloads (0, 1). +# +# set to 1 to delete failed, or 0 to leave files in place. +#W3delete_failed=0 + +# Watcher3 wait_for +# +# Set the number of minutes to wait after calling the renamer, to check the movie has changed status. +#W3wait_for=2 + +# Watcher3 and NZBGet are a different system (0, 1). +# +# Enable to replace local path with the path as per the mountPoints below. +#W3remote_path=0 + +## Network + +# Network Mount Points (Needed for remote path above) +# +# Enter Mount points as LocalPath,RemotePath and separate each pair with '|' +# e.g. mountPoints=/volume1/Public/,E:\|/volume2/share/,\\NAS\ +#mountPoints= + +## Extensions + +# Media Extensions +# +# This is a list of media extensions that are used to verify that the download does contain valid media. +#mediaExtensions=.mkv,.avi,.divx,.xvid,.mov,.wmv,.mp4,.mpg,.mpeg,.vob,.iso,.ts + +## Posix + +# Niceness for external tasks Extractor and Transcoder. +# +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 + +# ionice scheduling class (0, 1, 2, 3). +# +# Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. +#ionice_class=2 + +# ionice scheduling class data. +# +# Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. +#ionice_classdata=4 + +## Transcoder + +# getSubs (0, 1). +# +# set to 1 to download subtitles. +#getSubs=0 + +# subLanguages. +# +# subLanguages. create a list of languages in the order you want them in your subtitles. +#subLanguages=eng,spa,fra + +# Transcode (0, 1). +# +# set to 1 to transcode, otherwise set to 0. +#transcode=0 + +# create a duplicate, or replace the original (0, 1). +# +# set to 1 to cretae a new file or 0 to replace the original +#duplicate=1 + +# ignore extensions. +# +# list of extensions that won't be transcoded. +#ignoreExtensions=.avi,.mkv + +# outputFastStart (0,1). +# +# outputFastStart. 1 will use -movflags + faststart. 0 will disable this from being used. +#outputFastStart=0 + +# outputVideoPath. +# +# outputVideoPath. Set path you want transcoded videos moved to. Leave blank to disable. +#outputVideoPath= + +# processOutput (0,1). +# +# processOutput. 1 will send the outputVideoPath to SickBeard/CouchPotato. 0 will send original files. +#processOutput=0 + +# audioLanguage. +# +# audioLanguage. set the 3 letter language code you want as your primary audio track. +#audioLanguage=eng + +# allAudioLanguages (0,1). +# +# allAudioLanguages. 1 will keep all audio tracks (uses AudioCodec3) where available. +#allAudioLanguages=0 + +# allSubLanguages (0,1). +# +# allSubLanguages. 1 will keep all exisiting sub languages. 0 will discare those not in your list above. +#allSubLanguages=0 + +# embedSubs (0,1). +# +# embedSubs. 1 will embded external sub/srt subs into your video if this is supported. +#embedSubs=1 + +# burnInSubtitle (0,1). +# +# burnInSubtitle. burns the default sub language into your video (needed for players that don't support subs) +#burnInSubtitle=0 + +# extractSubs (0,1). +# +# extractSubs. 1 will extract subs from the video file and save these as external srt files. +#extractSubs=0 + +# externalSubDir. +# +# externalSubDir. set the directory where subs should be saved (if not the same directory as the video) +#externalSubDir= + +# outputDefault (None, iPad, iPad-1080p, iPad-720p, Apple-TV2, iPod, iPhone, PS3, xbox, Roku-1080p, Roku-720p, Roku-480p, mkv, mkv-bluray, mp4-scene-release, MKV-SD). +# +# outputDefault. Loads default configs for the selected device. The remaining options below are ignored. +# If you want to use your own profile, set None and set the remaining options below. +#outputDefault=None + +# hwAccel (0,1). +# +# hwAccel. 1 will set ffmpeg to enable hardware acceleration (this requires a recent ffmpeg). +#hwAccel=0 + +# ffmpeg output settings. +#outputVideoExtension=.mp4 +#outputVideoCodec=libx264 +#VideoCodecAllow= +#outputVideoResolution=720:-1 +#outputVideoPreset=medium +#outputVideoFramerate=24 +#outputVideoBitrate=800k +#outputAudioCodec=ac3 +#AudioCodecAllow= +#outputAudioChannels=6 +#outputAudioBitrate=640k +#outputQualityPercent= +#outputAudioTrack2Codec=libfaac +#AudioCodec2Allow= +#outputAudioTrack2Channels=2 +#outputAudioTrack2Bitrate=160k +#outputAudioOtherCodec=libmp3lame +#AudioOtherCodecAllow= +#outputAudioOtherChannels=2 +#outputAudioOtherBitrate=128k +#outputSubtitleCodec= + +## WakeOnLan + +# use WOL (0, 1). +# +# set to 1 to send WOL broadcast to the mac and test the server (e.g. xbmc) on the host and port specified. +#wolwake=0 + +# WOL MAC +# +# enter the mac address of the system to be woken. +#wolmac=00:01:2e:2D:64:e1 + +# Set the Host and Port of a server to verify system has woken. +#wolhost=192.168.1.37 +#wolport=80 + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +import sys + +import nzbToMedia + +section = 'Watcher3' +result = nzbToMedia.main(sys.argv, section) +sys.exit(result) From 5cd449632f1f46eb7ad840cd2b39d4206242ef9f Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 8 Nov 2019 14:13:07 +1300 Subject: [PATCH 254/406] Py3.8 (#1659) * Add Python3.8 and CI Tests * Force testing of video in case ffmpeg not working --- azure-pipelines.yml | 46 ++++++---------------------------------- eol.py | 1 + tests/test_transcoder.py | 2 +- 3 files changed, 8 insertions(+), 41 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index db2d4f2b..948047c9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,45 +21,14 @@ jobs: python.version: '3.6' Python37: python.version: '3.7' - maxParallel: 4 + Python38: + python.version: '3.8' + maxParallel: 5 steps: - #- script: | - # Make sure all packages are pulled from latest - #sudo apt-get update - - # Fail out if any setups fail - #set -e - - # Delete old Pythons - #rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 - #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 - #rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 - - # Download new Pythons - #azcopy --recursive \ - #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ - #--destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 - - #azcopy --recursive \ - #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ - #--destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 - - #azcopy --recursive \ - #--source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ - #--destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 - - # Install new Pythons - #original_directory=$PWD - #setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) - #for setup in $setups; do - #chmod +x $setup; - #cd $(dirname $setup); - #./$(basename $setup); - #cd $original_directory; - #done; - #displayName: 'Workaround: update apt and roll back Python versions' - + - script: sudo apt-get install ffmpeg + displayName: 'Install ffmpeg' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' @@ -68,9 +37,6 @@ jobs: - script: python -m pip install --upgrade pip displayName: 'Install dependencies' - - script: sudo apt-get install ffmpeg - displayName: 'Install ffmpeg' - - script: | pip install pytest pytest tests --doctest-modules --junitxml=junit/test-results.xml diff --git a/eol.py b/eol.py index 7310cfd6..fa328fdd 100644 --- a/eol.py +++ b/eol.py @@ -28,6 +28,7 @@ def date(string, fmt='%Y-%m-%d'): # https://devguide.python.org/ # https://devguide.python.org/devcycle/#devcycle PYTHON_EOL = { + (3, 8): date('2024-10-14'), (3, 7): date('2023-06-27'), (3, 6): date('2021-12-23'), (3, 5): date('2020-09-13'), diff --git a/tests/test_transcoder.py b/tests/test_transcoder.py index 448ceeaf..5c5b73ac 100755 --- a/tests/test_transcoder.py +++ b/tests/test_transcoder.py @@ -11,4 +11,4 @@ from core import transcoder def test_transcoder_check(): - assert transcoder.is_video_good(core.TEST_FILE, 0) is True + assert transcoder.is_video_good(core.TEST_FILE, 1) is True From fdaa00775674a887bb4002e82b89ba6de3fa9083 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 10 Nov 2019 09:38:48 +1300 Subject: [PATCH 255/406] Don't write byte code (#1669) --- cleanup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cleanup.py b/cleanup.py index 70c9ce29..3724844b 100644 --- a/cleanup.py +++ b/cleanup.py @@ -12,6 +12,8 @@ import subprocess import sys import shutil +sys.dont_write_bytecode = True + FOLDER_STRUCTURE = { 'libs': [ 'common', From 0d7c59f1f035d51a7409449bc4e335928a6b29bd Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Wed, 13 Nov 2019 18:32:03 +1300 Subject: [PATCH 256/406] Remove Encode of directory #1671 (#1672) --- core/utils/files.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/utils/files.py b/core/utils/files.py index 726d4209..f29efa6b 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -53,10 +53,11 @@ def move_file(mediafile, path, link): title = os.path.splitext(os.path.basename(mediafile))[0] new_path = os.path.join(path, sanitize_name(title)) - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass + # Removed as encoding of directory no-longer required + #try: + # new_path = new_path.encode(core.SYS_ENCODING) + #except Exception: + # pass # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). if os.path.isfile(new_path): From d95e4e56c8569bc41a085336a797044ca2c4a1d9 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 8 Dec 2019 12:31:46 +1300 Subject: [PATCH 257/406] remove redundant json.loads #1671 (#1681) --- core/auto_process/movies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index e7439aa0..6535cddd 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -252,8 +252,7 @@ def process(section, dir_name, input_name=None, status=0, client_agent='manual', elif section == 'Radarr': logger.postprocess('Radarr response: {0}'.format(result['state'])) try: - res = json.loads(r.content) - scan_id = int(res['id']) + scan_id = int(result['id']) logger.debug('Scan started with id: {0}'.format(scan_id), section) except Exception as e: logger.warning('No scan id was returned due to: {0}'.format(e), section) From 75ecbd48629f31c0456c99b726eff99a650e1530 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 8 Dec 2019 14:35:15 +1300 Subject: [PATCH 258/406] Add Submodule checks (#1682) --- core/version_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version_check.py b/core/version_check.py index f1c05ba9..a6dde88d 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -53,7 +53,7 @@ class CheckVersion(object): 'source': running from source without git """ # check if we're a windows build - if os.path.isdir(os.path.join(core.APP_ROOT, u'.git')): + if os.path.exists(os.path.join(core.APP_ROOT, u'.git')): install_type = 'git' else: install_type = 'source' From feb4e36c4c1dab5d6b4d467a1355076c50d28de6 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Sun, 8 Dec 2019 14:42:59 +1300 Subject: [PATCH 259/406] update to v12.1.02 --- .bumpversion.cfg | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1edaabcf..92813093 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.1.01 +current_version = 12.1.02 commit = True tag = False diff --git a/core/__init__.py b/core/__init__.py index 76798031..3dff1f4c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -83,7 +83,7 @@ from core.utils import ( wake_up, ) -__version__ = '12.1.01' +__version__ = '12.1.02' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 5e79ab2c..abe29733 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.1.01', + version='12.1.02', license='GPLv3', description='Efficient on demand post processing', long_description=""" From aeb3e0fd6d6e01987fdfc33fc17a52631dd3706b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Tue, 10 Dec 2019 12:55:13 +1300 Subject: [PATCH 260/406] Deluge update to V2 (#1683) Fixes #1680 --- core/plugins/downloaders/torrent/deluge.py | 6 +- core/utils/parsers.py | 2 +- libs/custom/deluge_client/__init__.py | 0 libs/custom/deluge_client/client.py | 275 ++++++++++++ libs/custom/deluge_client/rencode.py | 474 ++++++++++++++++++++ libs/custom/deluge_client/tests.py | 65 +++ libs/custom/synchronousdeluge/__init__.py | 31 -- libs/custom/synchronousdeluge/client.py | 172 ------- libs/custom/synchronousdeluge/exceptions.py | 18 - libs/custom/synchronousdeluge/protocol.py | 46 -- libs/custom/synchronousdeluge/transfer.py | 64 --- libs/requirements-custom.txt | 2 +- tox.ini | 2 +- 13 files changed, 820 insertions(+), 337 deletions(-) create mode 100644 libs/custom/deluge_client/__init__.py create mode 100644 libs/custom/deluge_client/client.py create mode 100644 libs/custom/deluge_client/rencode.py create mode 100644 libs/custom/deluge_client/tests.py delete mode 100644 libs/custom/synchronousdeluge/__init__.py delete mode 100644 libs/custom/synchronousdeluge/client.py delete mode 100644 libs/custom/synchronousdeluge/exceptions.py delete mode 100644 libs/custom/synchronousdeluge/protocol.py delete mode 100644 libs/custom/synchronousdeluge/transfer.py diff --git a/core/plugins/downloaders/torrent/deluge.py b/core/plugins/downloaders/torrent/deluge.py index 86542a8a..0c4c07c8 100644 --- a/core/plugins/downloaders/torrent/deluge.py +++ b/core/plugins/downloaders/torrent/deluge.py @@ -5,7 +5,7 @@ from __future__ import ( unicode_literals, ) -from synchronousdeluge.client import DelugeClient +from deluge_client.client import DelugeRPCClient import core from core import logger @@ -19,9 +19,9 @@ def configure_client(): password = core.DELUGE_PASSWORD logger.debug('Connecting to {0}: http://{1}:{2}'.format(agent, host, port)) - client = DelugeClient() + client = DelugeRPCClient(host, port, user, password) try: - client.connect(host, port, user, password) + client.connect() except Exception: logger.error('Failed to connect to Deluge') else: diff --git a/core/utils/parsers.py b/core/utils/parsers.py index fd88ec63..eff4b3e9 100644 --- a/core/utils/parsers.py +++ b/core/utils/parsers.py @@ -65,7 +65,7 @@ def parse_deluge(args): input_hash = args[1] input_id = args[1] try: - input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] + input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get(b'label').decode() except Exception: input_category = '' return input_directory, input_name, input_category, input_hash, input_id diff --git a/libs/custom/deluge_client/__init__.py b/libs/custom/deluge_client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/custom/deluge_client/client.py b/libs/custom/deluge_client/client.py new file mode 100644 index 00000000..827fcd7c --- /dev/null +++ b/libs/custom/deluge_client/client.py @@ -0,0 +1,275 @@ +import logging +import socket +import ssl +import struct +import warnings +import zlib + +from .rencode import dumps, loads + +RPC_RESPONSE = 1 +RPC_ERROR = 2 +RPC_EVENT = 3 + +MESSAGE_HEADER_SIZE = 5 +READ_SIZE = 10 + +logger = logging.getLogger(__name__) + + +class DelugeClientException(Exception): + """Base exception for all deluge client exceptions""" + + +class ConnectionLostException(DelugeClientException): + pass + + +class CallTimeoutException(DelugeClientException): + pass + + +class InvalidHeaderException(DelugeClientException): + pass + + +class FailedToReconnectException(DelugeClientException): + pass + + +class RemoteException(DelugeClientException): + pass + + +class DelugeRPCClient(object): + timeout = 20 + + def __init__(self, host, port, username, password, decode_utf8=False, automatic_reconnect=True): + self.host = host + self.port = port + self.username = username + self.password = password + self.deluge_version = None + # This is only applicable if deluge_version is 2 + self.deluge_protocol_version = None + + self.decode_utf8 = decode_utf8 + if not self.decode_utf8: + warnings.warn('Using `decode_utf8=False` is deprecated, please set it to True.' + 'The argument will be removed in a future release where it will be always True', DeprecationWarning) + + self.automatic_reconnect = automatic_reconnect + + self.request_id = 1 + self.connected = False + self._create_socket() + + def _create_socket(self, ssl_version=None): + if ssl_version is not None: + self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM), ssl_version=ssl_version) + else: + self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + self._socket.settimeout(self.timeout) + + def connect(self): + """ + Connects to the Deluge instance + """ + self._connect() + logger.debug('Connected to Deluge, detecting daemon version') + self._detect_deluge_version() + logger.debug('Daemon version {} detected, logging in'.format(self.deluge_version)) + if self.deluge_version == 2: + result = self.call('daemon.login', self.username, self.password, client_version='deluge-client') + else: + result = self.call('daemon.login', self.username, self.password) + logger.debug('Logged in with value %r' % result) + self.connected = True + + def _connect(self): + logger.info('Connecting to %s:%s' % (self.host, self.port)) + try: + self._socket.connect((self.host, self.port)) + except ssl.SSLError as e: + # Note: have not verified that we actually get errno 258 for this error + if (hasattr(ssl, 'PROTOCOL_SSLv3') and + (getattr(e, 'reason', None) == 'UNSUPPORTED_PROTOCOL' or e.errno == 258)): + logger.warning('Was unable to ssl handshake, trying to force SSLv3 (insecure)') + self._create_socket(ssl_version=ssl.PROTOCOL_SSLv3) + self._socket.connect((self.host, self.port)) + else: + raise + + def disconnect(self): + """ + Disconnect from deluge + """ + if self.connected: + self._socket.close() + self._socket = None + self.connected = False + + def _detect_deluge_version(self): + if self.deluge_version is not None: + return + + self._send_call(1, None, 'daemon.info') + self._send_call(2, None, 'daemon.info') + self._send_call(2, 1, 'daemon.info') + result = self._socket.recv(1) + if result[:1] == b'D': + # This is a protocol deluge 2.0 was using before release + self.deluge_version = 2 + self.deluge_protocol_version = None + # If we need the specific version of deluge 2, this is it. + daemon_version = self._receive_response(2, None, partial_data=result) + elif ord(result[:1]) == 1: + self.deluge_version = 2 + self.deluge_protocol_version = 1 + # If we need the specific version of deluge 2, this is it. + daemon_version = self._receive_response(2, 1, partial_data=result) + else: + self.deluge_version = 1 + # Deluge 1 doesn't recover well from the bad request. Re-connect the socket. + self._socket.close() + self._create_socket() + self._connect() + + def _send_call(self, deluge_version, protocol_version, method, *args, **kwargs): + self.request_id += 1 + if method == 'daemon.login': + debug_args = list(args) + if len(debug_args) >= 2: + debug_args[1] = '