From c2eaa72a2c76b47b39ae3c6306b3b1bbd35a057a Mon Sep 17 00:00:00 2001 From: p0ps Date: Wed, 17 Feb 2021 08:31:08 +0100 Subject: [PATCH] Fix other sickbeard forks errorring. (#1814) * Update SickBeard section with is_priority param for medusa. * Add param type to medusa-apiv2 fork. * Extract param only when not a fork_obj * Directly return process_result from api_call() * Implemented classes for PymedusaApiV1 and PymedusaApiv2. * improve linting --- autoProcessMedia.cfg.spec | 3 + core/__init__.py | 2 +- core/auto_process/managers/pymedusa.py | 151 +++++++++++++++++++++--- core/auto_process/managers/sickbeard.py | 95 +++++++++++++-- core/auto_process/tv.py | 105 ++++++++-------- 5 files changed, 274 insertions(+), 82 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 30845362..d7ee9be7 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -164,6 +164,9 @@ process_method = # force processing of already processed content when running a manual scan. force = 0 + # Additionally to force, handle the download as a priority downlaod. + # The processed files will always replace existing qualities, also if this is a lower quality. + is_priority = 0 # tell SickRage/Medusa to delete all source files after processing. delete_on = 0 # tell Medusa to ignore check for associated subtitle check when postponing release diff --git a/core/__init__.py b/core/__init__.py index 98bc39bf..ec72281c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -116,7 +116,7 @@ FORKS = { FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'cmd': 'postprocess'}, FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None, 'cmd': 'postprocess'}, - FORK_MEDUSA_APIV2: {'proc_dir': None, 'resource': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'is_priority': None}, + FORK_MEDUSA_APIV2: {'proc_dir': None, 'resource': None, 'failed': None, 'process_method': None, 'force': None, 'type': None, 'delete_on': None, 'is_priority': None}, FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None}, FORK_SICKGEAR_API: {'path': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'is priority': None, 'failed': None, 'cmd': 'sg.postprocess'}, FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None}, diff --git a/core/auto_process/managers/pymedusa.py b/core/auto_process/managers/pymedusa.py index e08b6173..83a9e478 100644 --- a/core/auto_process/managers/pymedusa.py +++ b/core/auto_process/managers/pymedusa.py @@ -1,8 +1,10 @@ -import requests +import time from core import logger +from core.auto_process.common import ProcessResult +from core.auto_process.managers.sickbeard import SickBeard -from .sickbeard import SickBeard +import requests class PyMedusa(SickBeard): @@ -10,29 +12,140 @@ class PyMedusa(SickBeard): def __init__(self, sb_init): super(PyMedusa, self).__init__(sb_init) - self.cfg = self.sb_init.config # in case we need something that's not already directly on self.sb_init. - - def _configure(): - """Configure pymedusa with config options.""" def _create_url(self): - if self.sb_init.apikey: - return '{0}{1}:{2}{3}/api/{4}/'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root, self.sb_init.apikey) return '{0}{1}:{2}{3}/home/postprocess/processEpisode'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root) - def api_call(self): - """Perform the api call with PyMedusa.""" - s = requests.Session() +class PyMedusaApiV1(SickBeard): + """PyMedusa apiv1 class.""" + + def __init__(self, sb_init): + super(PyMedusaApiV1, self).__init__(sb_init) + + def _create_url(self): + return '{0}{1}:{2}{3}/api/{4}/'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root, self.sb_init.apikey) + + def api_call(self): self._process_fork_prarams() url = self._create_url() logger.debug('Opening URL: {0} with params: {1}'.format(url, self.sb_init.fork_params), self.sb_init.section) - if not self.sb_init.apikey and self.sb_init.username and self.sb_init.password: - login = '{0}{1}:{2}{3}/login'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root) - login_params = {'username': self.sb_init.username, 'password': self.sb_init.password} - r = s.get(login, verify=False, timeout=(30, 60)) - if r.status_code in [401, 403] and r.cookies.get('_xsrf'): - login_params['_xsrf'] = r.cookies.get('_xsrf') - s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60)) - return s.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800)) + try: + response = self.session.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url), self.sb_init.section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(self.sb_init.section), + status_code=1, + ) + + if response.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(response.status_code), self.sb_init.section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(self.sb_init.section, response.status_code), + status_code=1, + ) + + if response.json()['result'] == 'success': + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name), + status_code=0, + ) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section), + status_code=1, # We did not receive Success confirmation. + ) + + +class PyMedusaApiV2(SickBeard): + """PyMedusa apiv2 class.""" + + def __init__(self, sb_init): + super(PyMedusaApiV2, self).__init__(sb_init) + + def _create_url(self): + return '{0}{1}:{2}{3}/api/v2/postprocess'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root) + + def _get_identifier_status(self, url): + # Loop through requesting medusa for the status on the queueitem. + try: + response = self.session.get(url, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to get postprocess identifier status', self.sb_init.section) + return False + + try: + jdata = response.json() + except ValueError: + return False + + return jdata + + def api_call(self): + self._process_fork_prarams() + url = self._create_url() + + logger.debug('Opening URL: {0}'.format(url), self.sb_init.section) + payload = self.sb_init.fork_params + payload['resource'] = self.sb_init.fork_params['nzbName'] + del payload['nzbName'] + + # Update the session with the x-api-key + self.session.headers.update({ + 'x-api-key': self.sb_init.apikey, + 'Content-type': 'application/json' + }) + + # Send postprocess request + try: + response = self.session.post(url, json=payload, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to send postprocess request', self.sb_init.section) + return ProcessResult( + message='{0}: Unable to send postprocess request to PyMedusa', + status_code=1, + ) + + # Get UUID + if response: + try: + jdata = response.json() + except ValueError: + logger.debug('No data returned from provider') + return False + + if not jdata.get('status') or not jdata['status'] == 'success': + return False + + queueitem_identifier = jdata['queueItem']['identifier'] + + wait_for = self.sb_init.config.get('wait_for', 2) + n = 0 + response = {} + url = '{0}/{1}'.format(url, queueitem_identifier) + while n < 12: # set up wait_for minutes to see if command completes.. + time.sleep(5 * wait_for) + response = self._get_identifier_status(url) + if response and response.get('success'): + break + if 'error' in response: + break + n += 1 + + # Log Medusa's PP logs here. + if response.get('output'): + for line in response['output']: + logger.postprocess('{0}'.format(line), self.sb_init.section) + + # For now this will most likely always be True. But in the future we could return an exit state + # for when the PP in medusa didn't yield an expected result. + if response.get('success'): + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name), + status_code=0, + ) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section), + status_code=1, # We did not receive Success confirmation. + ) diff --git a/core/auto_process/managers/sickbeard.py b/core/auto_process/managers/sickbeard.py index 74b44ee8..b1cbb2b4 100644 --- a/core/auto_process/managers/sickbeard.py +++ b/core/auto_process/managers/sickbeard.py @@ -7,19 +7,14 @@ from __future__ import ( unicode_literals, ) - import copy import core from core import logger -from core.utils import ( - convert_to_ascii, - flatten, - list_media_files, - remote_dir, - remove_dir, - server_responding, +from core.auto_process.common import ( + ProcessResult, ) +from core.utils import remote_dir from oauthlib.oauth2 import LegacyApplicationClient @@ -88,13 +83,15 @@ class InitSickBeard(object): replace = { 'medusa': 'Medusa', 'medusa-api': 'Medusa-api', + 'medusa-apiv1': 'Medusa-api', + 'medusa-apiv2': 'Medusa-apiv2', 'sickbeard-api': 'SickBeard-api', 'sickgear': 'SickGear', 'sickchill': 'SickChill', 'stheno': 'Stheno', } _val = cfg.get('fork', 'auto') - f1 = replace.get(_val, _val) + f1 = replace.get(_val.lower(), _val) try: fork = f1, core.FORKS[f1] except KeyError: @@ -303,9 +300,13 @@ class InitSickBeard(object): return fork def _init_fork(self): - from .pymedusa import PyMedusa + # These need to be imported here, to prevent a circular import. + from .pymedusa import PyMedusa, PyMedusaApiV1, PyMedusaApiV2 + mapped_forks = { - 'Medusa': PyMedusa + 'Medusa': PyMedusa, + 'Medusa-api': PyMedusaApiV1, + 'Medusa-apiv2': PyMedusaApiV2 } logger.debug('Create object for fork {fork}'.format(fork=self.fork)) if self.fork and mapped_forks.get(self.fork): @@ -323,6 +324,8 @@ class SickBeard(object): def __init__(self, sb_init): """SB constructor.""" self.sb_init = sb_init + self.session = requests.Session() + self.failed = None self.status = None self.input_name = None @@ -336,10 +339,14 @@ class SickBeard(object): self.force = int(self.sb_init.config.get('force', 0)) self.delete_on = int(self.sb_init.config.get('delete_on', 0)) self.ignore_subs = int(self.sb_init.config.get('ignore_subs', 0)) + self.is_priority = int(self.sb_init.config.get('is_priority', 0)) # get importmode, default to 'Move' for consistency with legacy self.import_mode = self.sb_init.config.get('importMode', 'Move') + # Keep track of result state + self.success = False + def initialize(self, dir_name, input_name=None, failed=False, client_agent='manual'): """We need to call this explicitely because we need some variables. @@ -419,8 +426,74 @@ class SickBeard(object): else: del fork_params[param] + if param == 'is_priority': + if self.is_priority: + fork_params[param] = self.is_priority + else: + del fork_params[param] + if param == 'force_next': fork_params[param] = 1 # delete any unused params so we don't pass them to SB by mistake [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None] + + def api_call(self): + """Perform a base sickbeard api call.""" + self._process_fork_prarams() + url = self._create_url() + + logger.debug('Opening URL: {0} with params: {1}'.format(url, self.sb_init.fork_params), self.sb_init.section) + try: + if not self.sb_init.apikey and self.sb_init.username and self.sb_init.password: + # If not using the api, we need to login using user/pass first. + login = '{0}{1}:{2}{3}/login'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root) + login_params = {'username': self.sb_init.username, 'password': self.sb_init.password} + r = self.session.get(login, verify=False, timeout=(30, 60)) + if r.status_code in [401, 403] and r.cookies.get('_xsrf'): + login_params['_xsrf'] = r.cookies.get('_xsrf') + self.session.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60)) + response = self.session.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url), self.sb_init.section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(self.sb_init.section), + status_code=1, + ) + + if response.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(response.status_code), self.sb_init.section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(self.sb_init.section, response.status_code), + status_code=1, + ) + + return self.process_response(response) + + def process_response(self, response): + """Iterate over the lines returned, and log. + + :param response: Streamed Requests response object. + This method will need to be overwritten in the forks, for alternative response handling. + """ + for line in response.iter_lines(): + if line: + line = line.decode('utf-8') + logger.postprocess('{0}'.format(line), self.sb_init.section) + # if 'Moving file from' in line: + # input_name = os.path.split(line)[1] + # if 'added to the queue' in line: + # queued = True + # For the refactoring i'm only considering vanilla sickbeard, as for the base class. + if 'Processing succeeded' in line or 'Successfully processed' in line: + self.success = True + + if self.success: + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name), + status_code=0, + ) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section), + status_code=1, # We did not receive Success confirmation. + ) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 336fb81c..179e51b5 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -196,66 +196,69 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu init_sickbeard.fork_obj.initialize(dir_name, input_name, failed, client_agent='manual') # configure SB params to pass - fork_params['quiet'] = 1 - fork_params['proc_type'] = 'manual' - if input_name is not None: - fork_params['nzbName'] = input_name + # We don't want to remove params, for the Forks that have been refactored. + # As we don't want to duplicate this part of the code. + if not init_sickbeard.fork_obj: + fork_params['quiet'] = 1 + fork_params['proc_type'] = 'manual' + if input_name is not None: + fork_params['nzbName'] = input_name - for param in copy.copy(fork_params): - if param == 'failed': - if failed > 1: - failed = 1 - fork_params[param] = failed - if 'proc_type' in fork_params: - del fork_params['proc_type'] - if 'type' in fork_params: - del fork_params['type'] + for param in copy.copy(fork_params): + if param == 'failed': + if failed > 1: + failed = 1 + fork_params[param] = failed + if 'proc_type' in fork_params: + del fork_params['proc_type'] + if 'type' in fork_params: + del fork_params['type'] - if param == 'return_data': - fork_params[param] = 0 - if 'quiet' in fork_params: - del fork_params['quiet'] + if param == 'return_data': + fork_params[param] = 0 + if 'quiet' in fork_params: + del fork_params['quiet'] - if param == 'type': - if 'type' in fork_params: # only set if we haven't already deleted for 'failed' above. - fork_params[param] = 'manual' - if 'proc_type' in fork_params: - del fork_params['proc_type'] + if param == 'type': + if 'type' in fork_params: # only set if we haven't already deleted for 'failed' above. + fork_params[param] = 'manual' + if 'proc_type' in fork_params: + del fork_params['proc_type'] - if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']: - fork_params[param] = dir_name - if remote_path: - fork_params[param] = remote_dir(dir_name) + if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']: + fork_params[param] = dir_name + if remote_path: + fork_params[param] = remote_dir(dir_name) - if param == 'process_method': - if process_method: - fork_params[param] = process_method - else: - del fork_params[param] + if param == 'process_method': + if process_method: + fork_params[param] = process_method + else: + del fork_params[param] - if param in ['force', 'force_replace']: - if force: - fork_params[param] = force - else: - del fork_params[param] + if param in ['force', 'force_replace']: + if force: + fork_params[param] = force + else: + del fork_params[param] - if param in ['delete_on', 'delete']: - if delete_on: - fork_params[param] = delete_on - else: - del fork_params[param] + if param in ['delete_on', 'delete']: + if delete_on: + fork_params[param] = delete_on + else: + del fork_params[param] - if param == 'ignore_subs': - if ignore_subs: - fork_params[param] = ignore_subs - else: - del fork_params[param] + if param == 'ignore_subs': + if ignore_subs: + fork_params[param] = ignore_subs + else: + del fork_params[param] - if param == 'force_next': - fork_params[param] = 1 + if param == 'force_next': + fork_params[param] = 1 - # delete any unused params so we don't pass them to SB by mistake - [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None] + # delete any unused params so we don't pass them to SB by mistake + [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None] if status == 0: if section == 'NzbDrone' and not apikey: @@ -323,7 +326,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu try: if section == 'SickBeard': if init_sickbeard.fork_obj: - r = init_sickbeard.fork_obj.api_call() + return init_sickbeard.fork_obj.api_call() else: s = requests.Session()