Merge pull request #1463 from clinton-hall/refactor/standardize

Remove superfluous classes and work towards standardizing processing
This commit is contained in:
Labrys of Knossos 2018-12-27 11:35:26 -05:00 committed by GitHub
commit fbf98457c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 1086 additions and 1062 deletions

View file

@ -10,6 +10,8 @@ import sys
import core
from core import logger, main_db
from core.auto_process import comics, games, movies, music, tv
from core.auto_process.common import ProcessResult
from core.user_scripts import external_script
from core.utils import char_replace, convert_to_ascii, plex_update, replace_links
from six import text_type
@ -232,31 +234,28 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp
if core.TORRENT_CHMOD_DIRECTORY:
core.rchmod(output_destination, core.TORRENT_CHMOD_DIRECTORY)
result = [0, ""]
result = ProcessResult(
message="",
status_code=0,
)
if section_name == 'UserScript':
result = external_script(output_destination, input_name, input_category, section)
elif section_name in ['CouchPotato', 'Radarr']:
result = core.Movie().process(section_name, output_destination, input_name,
status, client_agent, input_hash, input_category)
result = movies.process(section_name, output_destination, input_name, status, client_agent, input_hash, input_category)
elif section_name in ['SickBeard', 'NzbDrone', 'Sonarr']:
if input_hash:
input_hash = input_hash.upper()
result = core.TV().process(section_name, output_destination, input_name,
status, client_agent, input_hash, input_category)
result = tv.process(section_name, output_destination, input_name, status, client_agent, input_hash, input_category)
elif section_name in ['HeadPhones', 'Lidarr']:
result = core.Music().process(section_name, output_destination, input_name,
status, client_agent, input_category)
result = music.process(section_name, output_destination, input_name, status, client_agent, input_category)
elif section_name == 'Mylar':
result = core.Comic().process(section_name, output_destination, input_name,
status, client_agent, input_category)
result = comics.process(section_name, output_destination, input_name, status, client_agent, input_category)
elif section_name == 'Gamez':
result = core.Game().process(section_name, output_destination, input_name,
status, client_agent, input_category)
result = games.process(section_name, output_destination, input_name, status, client_agent, input_category)
plex_update(input_category)
if result[0] != 0:
if result.status_code != 0:
if not core.TORRENT_RESUME_ON_FAILURE:
logger.error("A problem was reported in the autoProcess* script. "
"Torrent won't resume seeding (settings)")
@ -302,7 +301,10 @@ def main(args):
logger.debug("Options passed into TorrentToMedia: {0}".format(args))
# Post-Processing Result
result = [0, ""]
result = ProcessResult(
message="",
status_code=0,
)
try:
input_directory, input_name, input_category, input_hash, input_id = core.parse_args(client_agent, args)
@ -361,12 +363,12 @@ def main(args):
(section, subsection))
result = results
if result[0] == 0:
if result.status_code == 0:
logger.info("The {0} script completed successfully.".format(args[0]))
else:
logger.error("A problem was reported in the {0} script.".format(args[0]))
del core.MYAPP
return result[0]
return result.status_code
if __name__ == "__main__":

View file

@ -6,13 +6,13 @@ import requests
import core
from core import logger
from core.auto_process.common import ProcessResult
from core.utils import convert_to_ascii, remote_dir, server_responding
requests.packages.urllib3.disable_warnings()
class Comic(object):
def process(self, section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None):
def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None):
apc_version = "2.04"
comicrn_version = "1.01"
@ -29,7 +29,10 @@ class Comic(object):
url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root)
if not server_responding(url):
logger.error("Server did not respond. Exiting", section)
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - {0} did not respond.".format(section),
status_code=1,
)
input_name, dir_name = convert_to_ascii(input_name, dir_name)
clean_name, ext = os.path.splitext(input_name)
@ -55,10 +58,16 @@ class Comic(object):
r = requests.post(url, params=params, stream=True, verify=False, timeout=(30, 300))
except requests.ConnectionError:
logger.error("Unable to open URL", section)
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - Unable to connect to {0}".format(section),
status_code=1
)
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
return ProcessResult(
message="{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code),
status_code=1,
)
result = r.content
if not type(result) == list:
@ -71,7 +80,13 @@ class Comic(object):
if success:
logger.postprocess("SUCCESS: This issue has been processed successfully", section)
return [0, "{0}: Successfully post-processed {1}".format(section, input_name)]
return ProcessResult(
message="{0}: Successfully post-processed {1}".format(section, input_name),
status_code=0,
)
else:
logger.warning("The issue does not appear to have successfully processed. Please check your Logs", section)
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - Returned log from {0} was not as expected.".format(section),
status_code=1,
)

View file

@ -0,0 +1,62 @@
import requests
from core import logger
class ProcessResult(object):
def __init__(self, message, status_code):
self.message = message
self.status_code = status_code
def __iter__(self):
return self.status_code, self.message
def __bool__(self):
return not bool(self.status_code)
def __str__(self):
return 'Processing {0}: {1}'.format(
'succeeded' if bool(self) else 'failed',
self.message
)
def __repr__(self):
return '<ProcessResult {0}: {1}>'.format(
self.status_code,
self.message,
)
def command_complete(url, params, headers, section):
try:
r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url), section)
return None
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return None
else:
try:
return r.json()['state']
except (ValueError, KeyError):
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("{0} did not return expected json data.".format(section), section)
return None
def completed_download_handling(url2, headers, section="MAIN"):
try:
r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url2), section)
return False
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return False
else:
try:
return r.json().get("enableCompletedDownloadHandling", False)
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
return False

View file

@ -7,13 +7,13 @@ import requests
import core
from core import logger
from core.auto_process.common import ProcessResult
from core.utils import convert_to_ascii, server_responding
requests.packages.urllib3.disable_warnings()
class Game(object):
def process(self, section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None):
def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None):
status = int(status)
cfg = dict(core.CFG[section][input_category])
@ -29,7 +29,10 @@ class Game(object):
url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root)
if not server_responding(url):
logger.error("Server did not respond. Exiting", section)
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - {0} did not respond.".format(section),
status_code=1,
)
input_name, dir_name = convert_to_ascii(input_name, dir_name)
@ -52,7 +55,10 @@ class Game(object):
r = requests.get(url, params=params, verify=False, timeout=(30, 300))
except requests.ConnectionError:
logger.error("Unable to open URL")
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - Unable to connect to {1}".format(section, section),
status_code=1,
)
result = r.json()
logger.postprocess("{0}".format(result), section)
@ -62,17 +68,32 @@ class Game(object):
shutil.move(dir_name, os.path.join(library, input_name))
except Exception:
logger.error("Unable to move {0} to {1}".format(dir_name, os.path.join(library, input_name)), section)
return [1, "{0}: Failed to post-process - Unable to move files".format(section)]
return ProcessResult(
message="{0}: Failed to post-process - Unable to move files".format(section),
status_code=1,
)
else:
logger.error("No library specified to move files to. Please edit your configuration.", section)
return [1, "{0}: Failed to post-process - No library defined in {1}".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - No library defined in {0}".format(section),
status_code=1,
)
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
return ProcessResult(
message="{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code),
status_code=1,
)
elif result['success']:
logger.postprocess("SUCCESS: Status for {0} has been set to {1} in Gamez".format(gamez_id, download_status), section)
return [0, "{0}: Successfully post-processed {1}".format(section, input_name)]
return ProcessResult(
message="{0}: Successfully post-processed {1}".format(section, input_name),
status_code=0,
)
else:
logger.error("FAILED: Status for {0} has NOT been updated in Gamez".format(gamez_id), section)
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)]
return ProcessResult(
message="{0}: Failed to post-process - Returned log from {0} was not as expected.".format(section),
status_code=1,
)

View file

@ -8,14 +8,14 @@ import requests
import core
from core import logger, transcoder
from core.auto_process.common import command_complete, completed_download_handling
from core.scene_exceptions import process_all_exceptions
from core.utils import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding
requests.packages.urllib3.disable_warnings()
class Movie(object):
def process(self, section, dir_name, input_name=None, status=0, client_agent="manual", download_id="", input_category=None, failure_link=None):
def process(section, dir_name, input_name=None, status=0, client_agent="manual", download_id="", input_category=None, failure_link=None):
cfg = dict(core.CFG[section][input_category])
@ -56,7 +56,7 @@ class Movie(object):
release = None
elif server_responding(base_url):
if section == "CouchPotato":
release = self.get_release(base_url, imdbid, download_id)
release = get_release(base_url, imdbid, download_id)
else:
release = None
else:
@ -282,7 +282,7 @@ class Movie(object):
while time.time() < timeout: # only wait 2 (default) minutes, then return.
logger.postprocess("Checking for status change, please stand by ...", section)
if section == "CouchPotato":
release = self.get_release(base_url, imdbid, download_id, release_id)
release = get_release(base_url, imdbid, download_id, release_id)
scan_id = None
else:
release = None
@ -304,7 +304,7 @@ class Movie(object):
pass
elif scan_id:
url = "{0}/{1}".format(base_url, scan_id)
command_status = self.command_complete(url, params, headers, section)
command_status = command_complete(url, params, headers, section)
if command_status:
logger.debug("The Scan command return status: {0}".format(command_status), section)
if command_status in ['completed']:
@ -328,7 +328,7 @@ class Movie(object):
time.sleep(10 * wait_for)
# The status hasn't changed. we have waited wait_for minutes which is more than enough. uTorrent can resume seeding now.
if section == "Radarr" and self.completed_download_handling(url2, headers, section=section):
if section == "Radarr" and completed_download_handling(url2, headers, section=section):
logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.".format(section), section)
return [status, "{0}: Complete DownLoad Handling is enabled. Passing back to {1}".format(section, section)]
logger.warning(
@ -336,40 +336,8 @@ class Movie(object):
section)
return [1, "{0}: Failed to post-process - No change in status".format(section)]
def command_complete(self, url, params, headers, section):
try:
r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url), section)
return None
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return None
else:
try:
return r.json()['state']
except (ValueError, KeyError):
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("{0} did not return expected json data.".format(section), section)
return None
def completed_download_handling(self, url2, headers, section="MAIN"):
try:
r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url2), section)
return False
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return False
else:
try:
return r.json().get("enableCompletedDownloadHandling", False)
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
return False
def get_release(self, base_url, imdb_id=None, download_id=None, release_id=None):
def get_release(base_url, imdb_id=None, download_id=None, release_id=None):
results = {}
params = {}

View file

@ -8,14 +8,14 @@ import requests
import core
from core import logger
from core.auto_process.common import command_complete
from core.scene_exceptions import process_all_exceptions
from core.utils import convert_to_ascii, list_media_files, remote_dir, remove_dir, server_responding
requests.packages.urllib3.disable_warnings()
class Music(object):
def process(self, section, dir_name, input_name=None, status=0, client_agent="manual", input_category=None):
def process(section, dir_name, input_name=None, status=0, client_agent="manual", input_category=None):
status = int(status)
cfg = dict(core.CFG[section][input_category])
@ -73,7 +73,7 @@ class Music(object):
'dir': remote_dir(dir_name) if remote_path else dir_name
}
res = self.force_process(params, url, apikey, input_name, dir_name, section, wait_for)
res = force_process(params, url, apikey, input_name, dir_name, section, wait_for)
if res[0] in [0, 1]:
return res
@ -83,7 +83,7 @@ class Music(object):
'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0]
}
res = self.force_process(params, url, apikey, input_name, dir_name, section, wait_for)
res = force_process(params, url, apikey, input_name, dir_name, section, wait_for)
if res[0] in [0, 1]:
return res
@ -127,7 +127,7 @@ class Music(object):
url = "{0}/{1}".format(url, scan_id)
while n < 6: # set up wait_for minutes to see if command completes..
time.sleep(10 * wait_for)
command_status = self.command_complete(url, params, headers, section)
command_status = command_complete(url, params, headers, section)
if command_status and command_status in ['completed', 'failed']:
break
n += 1
@ -157,24 +157,8 @@ class Music(object):
remove_dir(dir_name)
return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader.
def command_complete(self, url, params, headers, section):
try:
r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url), section)
return None
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return None
else:
try:
return r.json()['state']
except (ValueError, KeyError):
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("{0} did not return expected json data.".format(section), section)
return None
def get_status(self, url, apikey, dir_name):
def get_status(url, apikey, dir_name):
logger.debug("Attempting to get current status for release:{0}".format(os.path.basename(dir_name)))
params = {
@ -200,8 +184,9 @@ class Music(object):
if os.path.basename(dir_name) == album['FolderName']:
return album["Status"].lower()
def force_process(self, params, url, apikey, input_name, dir_name, section, wait_for):
release_status = self.get_status(url, apikey, dir_name)
def force_process(params, url, apikey, input_name, dir_name, section, wait_for):
release_status = get_status(url, apikey, dir_name)
if not release_status:
logger.error("Could not find a status for {0}, is it in the wanted list ?".format(input_name), section)
@ -227,7 +212,7 @@ class Music(object):
# we will now wait for this album to be processed before returning to TorrentToMedia and unpausing.
timeout = time.time() + 60 * wait_for
while time.time() < timeout:
current_status = self.get_status(url, apikey, dir_name)
current_status = get_status(url, apikey, dir_name)
if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie.
logger.postprocess("SUCCESS: This release is now marked as status [{0}]".format(current_status), section)
return [0, "{0}: Successfully post-processed {1}".format(section, input_name)]

View file

@ -10,6 +10,7 @@ import requests
import core
from core import logger, transcoder
from core.auto_process.common import command_complete, completed_download_handling
from core.forks import auto_fork
from core.scene_exceptions import process_all_exceptions
from core.utils import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding
@ -17,8 +18,7 @@ from core.utils import convert_to_ascii, flatten, import_subs, list_media_files,
requests.packages.urllib3.disable_warnings()
class TV(object):
def process(self, section, dir_name, input_name=None, failed=False, client_agent="manual", download_id=None, input_category=None, failure_link=None):
def process(section, dir_name, input_name=None, failed=False, client_agent="manual", download_id=None, input_category=None, failure_link=None):
cfg = dict(core.CFG[section][input_category])
@ -316,7 +316,7 @@ class TV(object):
url = "{0}/{1}".format(url, scan_id)
while n < 6: # set up wait_for minutes to see if command completes..
time.sleep(10 * wait_for)
command_status = self.command_complete(url, params, headers, section)
command_status = command_complete(url, params, headers, section)
if command_status and command_status in ['completed', 'failed']:
break
n += 1
@ -331,7 +331,7 @@ class TV(object):
elif command_status and command_status in ['failed']:
logger.debug("The Scan command has failed. Renaming was not successful.", section)
# return [1, "%s: Failed to post-process %s" % (section, input_name) ]
if self.completed_download_handling(url2, headers, section=section):
if completed_download_handling(url2, headers, section=section):
logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.".format(section), section)
return [status, "{0}: Complete DownLoad Handling is enabled. Passing back to {1}".format(section, section)]
else:
@ -339,36 +339,3 @@ class TV(object):
return [1, "{0}: Failed to post-process {1}".format(section, input_name)]
else:
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] # We did not receive Success confirmation.
def command_complete(self, url, params, headers, section):
try:
r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url), section)
return None
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return None
else:
try:
return r.json()['state']
except (ValueError, KeyError):
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("{0} did not return expected json data.".format(section), section)
return None
def completed_download_handling(self, url2, headers, section="MAIN"):
try:
r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: {0}".format(url2), section)
return False
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return False
else:
try:
return r.json().get("enableCompletedDownloadHandling", False)
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
return False

View file

@ -12,7 +12,8 @@ import sys
import core
from core import logger, main_db
from core.auto_process import Comic, Game, Movie, Music, TV
from core.auto_process import comics, games, movies, music, tv
from core.auto_process.common import ProcessResult
from core.user_scripts import external_script
from core.utils import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status
@ -109,26 +110,26 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d
logger.info("Calling {0}:{1} to post-process:{2}".format(section_name, input_category, input_name))
if section_name in ["CouchPotato", "Radarr"]:
result = Movie().process(section_name, input_directory, input_name, status, client_agent, download_id,
input_category, failure_link)
result = movies.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link)
elif section_name in ["SickBeard", "NzbDrone", "Sonarr"]:
result = TV().process(section_name, input_directory, input_name, status, client_agent,
download_id, input_category, failure_link)
result = tv.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link)
elif section_name in ["HeadPhones", "Lidarr"]:
result = Music().process(section_name, input_directory, input_name, status, client_agent, input_category)
result = music.process(section_name, input_directory, input_name, status, client_agent, input_category)
elif section_name == "Mylar":
result = Comic().process(section_name, input_directory, input_name, status, client_agent,
input_category)
result = comics.process(section_name, input_directory, input_name, status, client_agent, input_category)
elif section_name == "Gamez":
result = Game().process(section_name, input_directory, input_name, status, client_agent, input_category)
result = games.process(section_name, input_directory, input_name, status, client_agent, input_category)
elif section_name == 'UserScript':
result = external_script(input_directory, input_name, input_category, section[usercat])
else:
result = [-1, ""]
result = ProcessResult(
message="",
status_code=-1,
)
plex_update(input_category)
if result[0] == 0:
if result.staus_code == 0:
if client_agent != 'manual':
# update download status in our DB
update_download_info_status(input_name, 1)
@ -151,7 +152,10 @@ def main(args, section=None):
logger.debug("Options passed into nzbToMedia: {0}".format(args))
# Post-Processing Result
result = [0, ""]
result = ProcessResult(
message="",
status_code=0,
)
status = 0
# NZBGet
@ -289,27 +293,27 @@ def main(args, section=None):
results = process(dir_name, input_name, 0, client_agent=client_agent,
download_id=download_id or None, input_category=subsection)
if results[0] != 0:
if results.status_code != 0:
logger.error("A problem was reported when trying to perform a manual run for {0}:{1}.".format
(section, subsection))
result = results
if result[0] == 0:
if result.status_code == 0:
logger.info("The {0} script completed successfully.".format(args[0]))
if result[1]:
print(result[1] + "!")
if result.message:
print(result.message + "!")
if 'NZBOP_SCRIPTDIR' in os.environ: # return code for nzbget v11
del core.MYAPP
return core.NZBGET_POSTPROCESS_SUCCESS
else:
logger.error("A problem was reported in the {0} script.".format(args[0]))
if result[1]:
print(result[1] + "!")
if result.message:
print(result.message + "!")
if 'NZBOP_SCRIPTDIR' in os.environ: # return code for nzbget v11
del core.MYAPP
return core.NZBGET_POSTPROCESS_ERROR
del core.MYAPP
return result[0]
return result.status_code
if __name__ == '__main__':