Fix other sickbeard forks errorring. (#1814)

* Update SickBeard section with is_priority param for medusa.

* Add param type to medusa-apiv2 fork.

* Extract param only when not a fork_obj
* Directly return process_result from api_call()

* Implemented classes for PymedusaApiV1 and PymedusaApiv2.

* improve linting

autoProcessMedia.cfg.spec

@@ -164,6 +164,9 @@
         process_method =
         # force processing of already processed content when running a manual scan.
         force = 0
+        # Additionally to force, handle the download as a priority downlaod.
+        # The processed files will always replace existing qualities, also if this is a lower quality.
+        is_priority = 0
         # tell SickRage/Medusa to delete all source files after processing.
         delete_on = 0
         # tell Medusa to ignore check for associated subtitle check when postponing release

core/__init__.py

@@ -116,7 +116,7 @@ FORKS = {
     FORK_SICKBEARD_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None, 'cmd': 'postprocess'},
     FORK_MEDUSA: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None},
     FORK_MEDUSA_API: {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete_files': None, 'is_priority': None, 'cmd': 'postprocess'},
-    FORK_MEDUSA_APIV2: {'proc_dir': None, 'resource': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'is_priority': None},
+    FORK_MEDUSA_APIV2: {'proc_dir': None, 'resource': None, 'failed': None, 'process_method': None, 'force': None, 'type': None, 'delete_on': None, 'is_priority': None},
     FORK_SICKGEAR: {'dir': None, 'failed': None, 'process_method': None, 'force': None},
     FORK_SICKGEAR_API: {'path': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'is priority': None, 'failed': None, 'cmd': 'sg.postprocess'},
     FORK_STHENO: {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None},

core/auto_process/managers/pymedusa.py

@@ -1,8 +1,10 @@
-import requests
+import time
 
 from core import logger
+from core.auto_process.common import ProcessResult
+from core.auto_process.managers.sickbeard import SickBeard
 
-from .sickbeard import SickBeard
+import requests
 
 
 class PyMedusa(SickBeard):
@@ -10,29 +12,140 @@ class PyMedusa(SickBeard):
 
     def __init__(self, sb_init):
         super(PyMedusa, self).__init__(sb_init)
-        self.cfg = self.sb_init.config  # in case we need something that's not already directly on self.sb_init.
-
-    def _configure():
-        """Configure pymedusa with config options."""
 
     def _create_url(self):
-        if self.sb_init.apikey:
-            return '{0}{1}:{2}{3}/api/{4}/'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root, self.sb_init.apikey)
         return '{0}{1}:{2}{3}/home/postprocess/processEpisode'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root)
 
-    def api_call(self):
-        """Perform the api call with PyMedusa."""
-        s = requests.Session()
 
+class PyMedusaApiV1(SickBeard):
+    """PyMedusa apiv1 class."""
+
+    def __init__(self, sb_init):
+        super(PyMedusaApiV1, self).__init__(sb_init)
+
+    def _create_url(self):
+        return '{0}{1}:{2}{3}/api/{4}/'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root, self.sb_init.apikey)
+
+    def api_call(self):
         self._process_fork_prarams()
         url = self._create_url()
 
         logger.debug('Opening URL: {0} with params: {1}'.format(url, self.sb_init.fork_params), self.sb_init.section)
-        if not self.sb_init.apikey and self.sb_init.username and self.sb_init.password:
+        try:
-            login = '{0}{1}:{2}{3}/login'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root)
+            response = self.session.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800))
-            login_params = {'username': self.sb_init.username, 'password': self.sb_init.password}
+        except requests.ConnectionError:
-            r = s.get(login, verify=False, timeout=(30, 60))
+            logger.error('Unable to open URL: {0}'.format(url), self.sb_init.section)
-            if r.status_code in [401, 403] and r.cookies.get('_xsrf'):
+            return ProcessResult(
-                login_params['_xsrf'] = r.cookies.get('_xsrf')
+                message='{0}: Failed to post-process - Unable to connect to {0}'.format(self.sb_init.section),
-            s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60))
+                status_code=1,
-        return s.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800))
+            )
+
+        if response.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
+            logger.error('Server returned status {0}'.format(response.status_code), self.sb_init.section)
+            return ProcessResult(
+                message='{0}: Failed to post-process - Server returned status {1}'.format(self.sb_init.section, response.status_code),
+                status_code=1,
+            )
+
+        if response.json()['result'] == 'success':
+            return ProcessResult(
+                message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name),
+                status_code=0,
+            )
+        return ProcessResult(
+            message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section),
+            status_code=1,  # We did not receive Success confirmation.
+        )
+
+
+class PyMedusaApiV2(SickBeard):
+    """PyMedusa apiv2 class."""
+
+    def __init__(self, sb_init):
+        super(PyMedusaApiV2, self).__init__(sb_init)
+
+    def _create_url(self):
+        return '{0}{1}:{2}{3}/api/v2/postprocess'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root)
+
+    def _get_identifier_status(self, url):
+        # Loop through requesting medusa for the status on the queueitem.
+        try:
+            response = self.session.get(url, verify=False, timeout=(30, 1800))
+        except requests.ConnectionError:
+            logger.error('Unable to get postprocess identifier status', self.sb_init.section)
+            return False
+
+        try:
+            jdata = response.json()
+        except ValueError:
+            return False
+
+        return jdata
+
+    def api_call(self):
+        self._process_fork_prarams()
+        url = self._create_url()
+
+        logger.debug('Opening URL: {0}'.format(url), self.sb_init.section)
+        payload = self.sb_init.fork_params
+        payload['resource'] = self.sb_init.fork_params['nzbName']
+        del payload['nzbName']
+
+        # Update the session with the x-api-key
+        self.session.headers.update({
+            'x-api-key': self.sb_init.apikey,
+            'Content-type': 'application/json'
+        })
+
+        # Send postprocess request
+        try:
+            response = self.session.post(url, json=payload, verify=False, timeout=(30, 1800))
+        except requests.ConnectionError:
+            logger.error('Unable to send postprocess request', self.sb_init.section)
+            return ProcessResult(
+                message='{0}: Unable to send postprocess request to PyMedusa',
+                status_code=1,
+            )
+
+        # Get UUID
+        if response:
+            try:
+                jdata = response.json()
+            except ValueError:
+                logger.debug('No data returned from provider')
+                return False
+
+        if not jdata.get('status') or not jdata['status'] == 'success':
+            return False
+
+        queueitem_identifier = jdata['queueItem']['identifier']
+
+        wait_for = self.sb_init.config.get('wait_for', 2)
+        n = 0
+        response = {}
+        url = '{0}/{1}'.format(url, queueitem_identifier)
+        while n < 12:  # set up wait_for minutes to see if command completes..
+            time.sleep(5 * wait_for)
+            response = self._get_identifier_status(url)
+            if response and response.get('success'):
+                break
+            if 'error' in response:
+                break
+            n += 1
+
+        # Log Medusa's PP logs here.
+        if response.get('output'):
+            for line in response['output']:
+                logger.postprocess('{0}'.format(line), self.sb_init.section)
+
+        # For now this will most likely always be True. But in the future we could return an exit state
+        # for when the PP in medusa didn't yield an expected result.
+        if response.get('success'):
+            return ProcessResult(
+                message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name),
+                status_code=0,
+            )
+        return ProcessResult(
+            message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section),
+            status_code=1,  # We did not receive Success confirmation.
+        )

core/auto_process/managers/sickbeard.py

@@ -7,19 +7,14 @@ from __future__ import (
     unicode_literals,
 )
 
-
 import copy
 
 import core
 from core import logger
-from core.utils import (
+from core.auto_process.common import (
-    convert_to_ascii,
+    ProcessResult,
-    flatten,
-    list_media_files,
-    remote_dir,
-    remove_dir,
-    server_responding,
 )
+from core.utils import remote_dir
 
 from oauthlib.oauth2 import LegacyApplicationClient
 
@@ -88,13 +83,15 @@ class InitSickBeard(object):
         replace = {
             'medusa': 'Medusa',
             'medusa-api': 'Medusa-api',
+            'medusa-apiv1': 'Medusa-api',
+            'medusa-apiv2': 'Medusa-apiv2',
             'sickbeard-api': 'SickBeard-api',
             'sickgear': 'SickGear',
             'sickchill': 'SickChill',
             'stheno': 'Stheno',
         }
         _val = cfg.get('fork', 'auto')
-        f1 = replace.get(_val, _val)
+        f1 = replace.get(_val.lower(), _val)
         try:
             fork = f1, core.FORKS[f1]
         except KeyError:
@@ -303,9 +300,13 @@ class InitSickBeard(object):
         return fork
 
     def _init_fork(self):
-        from .pymedusa import PyMedusa
+        # These need to be imported here, to prevent a circular import.
+        from .pymedusa import PyMedusa, PyMedusaApiV1, PyMedusaApiV2
+
         mapped_forks = {
-            'Medusa': PyMedusa
+            'Medusa': PyMedusa,
+            'Medusa-api': PyMedusaApiV1,
+            'Medusa-apiv2': PyMedusaApiV2
         }
         logger.debug('Create object for fork {fork}'.format(fork=self.fork))
         if self.fork and mapped_forks.get(self.fork):
@@ -323,6 +324,8 @@ class SickBeard(object):
     def __init__(self, sb_init):
         """SB constructor."""
         self.sb_init = sb_init
+        self.session = requests.Session()
+
         self.failed = None
         self.status = None
         self.input_name = None
@@ -336,10 +339,14 @@ class SickBeard(object):
         self.force = int(self.sb_init.config.get('force', 0))
         self.delete_on = int(self.sb_init.config.get('delete_on', 0))
         self.ignore_subs = int(self.sb_init.config.get('ignore_subs', 0))
+        self.is_priority = int(self.sb_init.config.get('is_priority', 0))
 
         # get importmode, default to 'Move' for consistency with legacy
         self.import_mode = self.sb_init.config.get('importMode', 'Move')
 
+        # Keep track of result state
+        self.success = False
+
     def initialize(self, dir_name, input_name=None, failed=False, client_agent='manual'):
         """We need to call this explicitely because we need some variables.
 
@@ -419,8 +426,74 @@ class SickBeard(object):
                 else:
                     del fork_params[param]
 
+            if param == 'is_priority':
+                if self.is_priority:
+                    fork_params[param] = self.is_priority
+                else:
+                    del fork_params[param]
+
             if param == 'force_next':
                 fork_params[param] = 1
 
         # delete any unused params so we don't pass them to SB by mistake
         [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None]
+
+    def api_call(self):
+        """Perform a base sickbeard api call."""
+        self._process_fork_prarams()
+        url = self._create_url()
+
+        logger.debug('Opening URL: {0} with params: {1}'.format(url, self.sb_init.fork_params), self.sb_init.section)
+        try:
+            if not self.sb_init.apikey and self.sb_init.username and self.sb_init.password:
+                # If not using the api, we need to login using user/pass first.
+                login = '{0}{1}:{2}{3}/login'.format(self.sb_init.protocol, self.sb_init.host, self.sb_init.port, self.sb_init.web_root)
+                login_params = {'username': self.sb_init.username, 'password': self.sb_init.password}
+                r = self.session.get(login, verify=False, timeout=(30, 60))
+                if r.status_code in [401, 403] and r.cookies.get('_xsrf'):
+                    login_params['_xsrf'] = r.cookies.get('_xsrf')
+                self.session.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60))
+            response = self.session.get(url, auth=(self.sb_init.username, self.sb_init.password), params=self.sb_init.fork_params, stream=True, verify=False, timeout=(30, 1800))
+        except requests.ConnectionError:
+            logger.error('Unable to open URL: {0}'.format(url), self.sb_init.section)
+            return ProcessResult(
+                message='{0}: Failed to post-process - Unable to connect to {0}'.format(self.sb_init.section),
+                status_code=1,
+            )
+
+        if response.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
+            logger.error('Server returned status {0}'.format(response.status_code), self.sb_init.section)
+            return ProcessResult(
+                message='{0}: Failed to post-process - Server returned status {1}'.format(self.sb_init.section, response.status_code),
+                status_code=1,
+            )
+
+        return self.process_response(response)
+
+    def process_response(self, response):
+        """Iterate over the lines returned, and log.
+
+        :param response: Streamed Requests response object.
+        This method will need to be overwritten in the forks, for alternative response handling.
+        """
+        for line in response.iter_lines():
+            if line:
+                line = line.decode('utf-8')
+                logger.postprocess('{0}'.format(line), self.sb_init.section)
+                # if 'Moving file from' in line:
+                #     input_name = os.path.split(line)[1]
+                # if 'added to the queue' in line:
+                #     queued = True
+                # For the refactoring i'm only considering vanilla sickbeard, as for the base class.
+                if 'Processing succeeded' in line or 'Successfully processed' in line:
+                    self.success = True
+
+        if self.success:
+            return ProcessResult(
+                message='{0}: Successfully post-processed {1}'.format(self.sb_init.section, self.input_name),
+                status_code=0,
+            )
+        return ProcessResult(
+            message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(self.sb_init.section),
+            status_code=1,  # We did not receive Success confirmation.
+        )

core/auto_process/tv.py

@@ -196,66 +196,69 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu
         init_sickbeard.fork_obj.initialize(dir_name, input_name, failed, client_agent='manual')
 
     # configure SB params to pass
-    fork_params['quiet'] = 1
+    # We don't want to remove params, for the Forks that have been refactored.
-    fork_params['proc_type'] = 'manual'
+    # As we don't want to duplicate this part of the code.
-    if input_name is not None:
+    if not init_sickbeard.fork_obj:
-        fork_params['nzbName'] = input_name
+        fork_params['quiet'] = 1
+        fork_params['proc_type'] = 'manual'
+        if input_name is not None:
+            fork_params['nzbName'] = input_name
 
-    for param in copy.copy(fork_params):
+        for param in copy.copy(fork_params):
-        if param == 'failed':
+            if param == 'failed':
-            if failed > 1:
+                if failed > 1:
-                failed = 1
+                    failed = 1
-            fork_params[param] = failed
+                fork_params[param] = failed
-            if 'proc_type' in fork_params:
+                if 'proc_type' in fork_params:
-                del fork_params['proc_type']
+                    del fork_params['proc_type']
-            if 'type' in fork_params:
+                if 'type' in fork_params:
-                del fork_params['type']
+                    del fork_params['type']
 
-        if param == 'return_data':
+            if param == 'return_data':
-            fork_params[param] = 0
+                fork_params[param] = 0
-            if 'quiet' in fork_params:
+                if 'quiet' in fork_params:
-                del fork_params['quiet']
+                    del fork_params['quiet']
 
-        if param == 'type':
+            if param == 'type':
-            if 'type' in fork_params:  # only set if we haven't already deleted for 'failed' above.
+                if 'type' in fork_params:  # only set if we haven't already deleted for 'failed' above.
-                fork_params[param] = 'manual'
+                    fork_params[param] = 'manual'
-            if 'proc_type' in fork_params:
+                if 'proc_type' in fork_params:
-                del fork_params['proc_type']
+                    del fork_params['proc_type']
 
-        if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']:
+            if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']:
-            fork_params[param] = dir_name
+                fork_params[param] = dir_name
-            if remote_path:
+                if remote_path:
-                fork_params[param] = remote_dir(dir_name)
+                    fork_params[param] = remote_dir(dir_name)
 
-        if param == 'process_method':
+            if param == 'process_method':
-            if process_method:
+                if process_method:
-                fork_params[param] = process_method
+                    fork_params[param] = process_method
-            else:
+                else:
-                del fork_params[param]
+                    del fork_params[param]
 
-        if param in ['force', 'force_replace']:
+            if param in ['force', 'force_replace']:
-            if force:
+                if force:
-                fork_params[param] = force
+                    fork_params[param] = force
-            else:
+                else:
-                del fork_params[param]
+                    del fork_params[param]
 
-        if param in ['delete_on', 'delete']:
+            if param in ['delete_on', 'delete']:
-            if delete_on:
+                if delete_on:
-                fork_params[param] = delete_on
+                    fork_params[param] = delete_on
-            else:
+                else:
-                del fork_params[param]
+                    del fork_params[param]
 
-        if param == 'ignore_subs':
+            if param == 'ignore_subs':
-            if ignore_subs:
+                if ignore_subs:
-                fork_params[param] = ignore_subs
+                    fork_params[param] = ignore_subs
-            else:
+                else:
-                del fork_params[param]
+                    del fork_params[param]
 
-        if param == 'force_next':
+            if param == 'force_next':
-            fork_params[param] = 1
+                fork_params[param] = 1
 
-    # delete any unused params so we don't pass them to SB by mistake
+        # delete any unused params so we don't pass them to SB by mistake
-    [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None]
+        [fork_params.pop(k) for k, v in list(fork_params.items()) if v is None]
 
     if status == 0:
         if section == 'NzbDrone' and not apikey:
@@ -323,7 +326,7 @@ def process(section, dir_name, input_name=None, failed=False, client_agent='manu
     try:
         if section == 'SickBeard':
             if init_sickbeard.fork_obj:
-                r = init_sickbeard.fork_obj.api_call()
+                return init_sickbeard.fork_obj.api_call()
             else:
                 s = requests.Session()