From 3b670b895bb0ccfe67764719cb25eae42fe5d284 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 13:44:25 -0500 Subject: [PATCH 01/38] Refactor utils module to package --- cleanup.py | 1 + core/{utils.py => utils/__init__.py} | 0 2 files changed, 1 insertion(+) rename core/{utils.py => utils/__init__.py} (100%) diff --git a/cleanup.py b/cleanup.py index 8c2d54d2..02bdf648 100644 --- a/cleanup.py +++ b/cleanup.py @@ -17,6 +17,7 @@ FOLDER_STRUCTURE = { 'core': [ 'auto_process', 'extractor', + 'utils', ], } diff --git a/core/utils.py b/core/utils/__init__.py similarity index 100% rename from core/utils.py rename to core/utils/__init__.py From 21fa4e38965764ebc91e6da4cf1a5e2873e0e2f3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 13:57:46 -0500 Subject: [PATCH 02/38] Refactor utils.*Process -> utils.processes.*Process --- core/utils/__init__.py | 92 +---------------------------------------- core/utils/processes.py | 92 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 91 deletions(-) create mode 100644 core/utils/processes.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 9c8d81cc..c1df352e 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -26,14 +26,7 @@ from utorrent.client import UTorrentClient import core from core import extractor, logger, main_db - -try: - from win32event import CreateMutex - from win32api import CloseHandle, GetLastError - from winerror import ERROR_ALREADY_EXISTS -except ImportError: - if os.name == 'nt': - raise +from core.utils.processes import RunningProcess try: import jaraco @@ -1321,86 +1314,3 @@ def get_download_info(input_name, status): [text_type(input_name), status]) return sql_results - - -class WindowsProcess(object): - def __init__(self): - self.mutex = None - self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' - self.CreateMutex = CreateMutex - self.CloseHandle = CloseHandle - self.GetLastError = GetLastError - self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS - - def alreadyrunning(self): - self.mutex = self.CreateMutex(None, 0, self.mutexname) - self.lasterror = self.GetLastError() - if self.lasterror == self.ERROR_ALREADY_EXISTS: - self.CloseHandle(self.mutex) - return True - else: - return False - - def __del__(self): - if self.mutex: - self.CloseHandle(self.mutex) - - -class PosixProcess(object): - def __init__(self): - self.pidpath = core.PID_FILE - self.lock_socket = None - - def alreadyrunning(self): - try: - self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) - self.lasterror = False - return self.lasterror - except socket.error as e: - if 'Address already in use' in e: - self.lasterror = True - return self.lasterror - except AttributeError: - pass - if os.path.exists(self.pidpath): - # Make sure it is not a 'stale' pidFile - try: - pid = int(open(self.pidpath, 'r').read().strip()) - except Exception: - pid = None - # Check list of running pids, if not running it is stale so overwrite - if isinstance(pid, int): - try: - os.kill(pid, 0) - self.lasterror = True - except OSError: - self.lasterror = False - else: - self.lasterror = False - else: - self.lasterror = False - - if not self.lasterror: - # Write my pid into pidFile to keep multiple copies of program from running - try: - fp = open(self.pidpath, 'w') - fp.write(str(os.getpid())) - fp.close() - except Exception: - pass - - return self.lasterror - - def __del__(self): - if not self.lasterror: - if self.lock_socket: - self.lock_socket.close() - if os.path.isfile(self.pidpath): - os.unlink(self.pidpath) - - -if os.name == 'nt': - RunningProcess = WindowsProcess -else: - RunningProcess = PosixProcess diff --git a/core/utils/processes.py b/core/utils/processes.py new file mode 100644 index 00000000..6513a12c --- /dev/null +++ b/core/utils/processes.py @@ -0,0 +1,92 @@ +import os +import socket + +import core + +if os.name == 'nt': + from win32event import CreateMutex + from win32api import CloseHandle, GetLastError + from winerror import ERROR_ALREADY_EXISTS + + +class WindowsProcess(object): + def __init__(self): + self.mutex = None + self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' + self.CreateMutex = CreateMutex + self.CloseHandle = CloseHandle + self.GetLastError = GetLastError + self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS + + def alreadyrunning(self): + self.mutex = self.CreateMutex(None, 0, self.mutexname) + self.lasterror = self.GetLastError() + if self.lasterror == self.ERROR_ALREADY_EXISTS: + self.CloseHandle(self.mutex) + return True + else: + return False + + def __del__(self): + if self.mutex: + self.CloseHandle(self.mutex) + + +class PosixProcess(object): + def __init__(self): + self.pidpath = core.PID_FILE + self.lock_socket = None + + def alreadyrunning(self): + try: + self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) + self.lasterror = False + return self.lasterror + except socket.error as e: + if 'Address already in use' in e: + self.lasterror = True + return self.lasterror + except AttributeError: + pass + if os.path.exists(self.pidpath): + # Make sure it is not a 'stale' pidFile + try: + pid = int(open(self.pidpath, 'r').read().strip()) + except Exception: + pid = None + # Check list of running pids, if not running it is stale so overwrite + if isinstance(pid, int): + try: + os.kill(pid, 0) + self.lasterror = True + except OSError: + self.lasterror = False + else: + self.lasterror = False + else: + self.lasterror = False + + if not self.lasterror: + # Write my pid into pidFile to keep multiple copies of program from running + try: + fp = open(self.pidpath, 'w') + fp.write(str(os.getpid())) + fp.close() + except Exception: + pass + + return self.lasterror + + def __del__(self): + if not self.lasterror: + if self.lock_socket: + self.lock_socket.close() + if os.path.isfile(self.pidpath): + os.unlink(self.pidpath) + + +if os.name == 'nt': + RunningProcess = WindowsProcess +else: + RunningProcess = PosixProcess From 4143aa77f8f0c6d849ad5ec88e8bbb58e33ef152 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:09:46 -0500 Subject: [PATCH 03/38] Refactor torrents from utils to utils.torrents --- core/utils/__init__.py | 100 +-------------------------------------- core/utils/torrents.py | 104 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 99 deletions(-) create mode 100644 core/utils/torrents.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index c1df352e..500f1498 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -16,17 +16,14 @@ from babelfish import Language import beets import guessit import linktastic -from qbittorrent import Client as qBittorrentClient import requests from six import text_type import subliminal -from synchronousdeluge.client import DelugeClient -from transmissionrpc.client import Client as TransmissionClient -from utorrent.client import UTorrentClient import core from core import extractor, logger, main_db from core.utils.processes import RunningProcess +from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent try: import jaraco @@ -812,101 +809,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def create_torrent_class(client_agent): - # Hardlink solution for Torrents - tc = None - - if client_agent == 'utorrent': - try: - logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) - tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) - except Exception: - logger.error('Failed to connect to uTorrent') - - if client_agent == 'transmission': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format( - client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) - tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, - core.TRANSMISSIONUSR, - core.TRANSMISSIONPWD) - except Exception: - logger.error('Failed to connect to Transmission') - - if client_agent == 'deluge': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) - tc = DelugeClient() - tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, - password=core.DELUGEPWD) - except Exception: - logger.error('Failed to connect to Deluge') - - if client_agent == 'qbittorrent': - try: - logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) - except Exception: - logger.error('Failed to connect to qBittorrent') - - return tc - - -def pause_torrent(client_agent, input_hash, input_id, input_name): - logger.debug('Stopping torrent {0} in {1} while processing'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.stop(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.stop_torrent(input_id) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.pause_torrent([input_id]) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.pause(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to stop torrent {0} in {1}'.format(input_name, client_agent)) - - -def resume_torrent(client_agent, input_hash, input_id, input_name): - if not core.TORRENT_RESUME == 1: - return - logger.debug('Starting torrent {0} in {1}'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.start(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.start_torrent(input_id) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.resume_torrent([input_id]) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.resume(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to start torrent {0} in {1}'.format(input_name, client_agent)) - - -def remove_torrent(client_agent, input_hash, input_id, input_name): - if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': - logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) - try: - if client_agent == 'utorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.removedata(input_hash) - core.TORRENT_CLASS.remove(input_hash) - if client_agent == 'transmission' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.remove_torrent(input_id, True) - if client_agent == 'deluge' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.core.remove_torrent(input_id, True) - if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': - core.TORRENT_CLASS.delete_permanently(input_hash) - time.sleep(5) - except Exception: - logger.warning('Failed to delete torrent {0} in {1}'.format(input_name, client_agent)) - else: - resume_torrent(client_agent, input_hash, input_id, input_name) - - def find_download(client_agent, download_id): logger.debug('Searching for Download on {0} ...'.format(client_agent)) if client_agent == 'utorrent': diff --git a/core/utils/torrents.py b/core/utils/torrents.py new file mode 100644 index 00000000..e9786da9 --- /dev/null +++ b/core/utils/torrents.py @@ -0,0 +1,104 @@ +import time + +from qbittorrent import Client as qBittorrentClient +from synchronousdeluge.client import DelugeClient +from transmissionrpc.client import Client as TransmissionClient +from utorrent.client import UTorrentClient + +import core +from core import logger + + +def create_torrent_class(client_agent): + # Hardlink solution for Torrents + tc = None + + if client_agent == 'utorrent': + try: + logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) + tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) + except Exception: + logger.error('Failed to connect to uTorrent') + + if client_agent == 'transmission': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format( + client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) + tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, + core.TRANSMISSIONUSR, + core.TRANSMISSIONPWD) + except Exception: + logger.error('Failed to connect to Transmission') + + if client_agent == 'deluge': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) + tc = DelugeClient() + tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, + password=core.DELUGEPWD) + except Exception: + logger.error('Failed to connect to Deluge') + + if client_agent == 'qbittorrent': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) + except Exception: + logger.error('Failed to connect to qBittorrent') + + return tc + + +def pause_torrent(client_agent, input_hash, input_id, input_name): + logger.debug('Stopping torrent {0} in {1} while processing'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.pause_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.pause(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to stop torrent {0} in {1}'.format(input_name, client_agent)) + + +def resume_torrent(client_agent, input_hash, input_id, input_name): + if not core.TORRENT_RESUME == 1: + return + logger.debug('Starting torrent {0} in {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.resume_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.resume(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to start torrent {0} in {1}'.format(input_name, client_agent)) + + +def remove_torrent(client_agent, input_hash, input_id, input_name): + if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': + logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.removedata(input_hash) + core.TORRENT_CLASS.remove(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.remove_torrent(input_id, True) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.remove_torrent(input_id, True) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.delete_permanently(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to delete torrent {0} in {1}'.format(input_name, client_agent)) + else: + resume_torrent(client_agent, input_hash, input_id, input_name) From bd16f11485c71200ba6a017fe60e198625984e77 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:10:35 -0500 Subject: [PATCH 04/38] Refactor nzbs from utils to utils.nzbs --- core/utils/__init__.py | 64 -------------------------------------- core/utils/nzbs.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 64 deletions(-) create mode 100644 core/utils/nzbs.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 500f1498..8e224974 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -44,22 +44,6 @@ def copyfileobj_fast(fsrc, fdst, length=512 * 1024): shutil.copyfileobj = copyfileobj_fast -def report_nzb(failure_link, client_agent): - # Contact indexer site - logger.info('Sending failure notification to indexer site') - if client_agent == 'nzbget': - headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} - elif client_agent == 'sabnzbd': - headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} - else: - return - try: - requests.post(failure_link, headers=headers, timeout=(30, 300)) - except Exception as e: - logger.error('Unable to open URL {0} due to {1}'.format(failure_link, e)) - return - - def sanitize_name(name): """ >>> sanitize_name('a/b/c') @@ -853,54 +837,6 @@ def find_download(client_agent, download_id): return False -def get_nzoid(input_name): - nzoid = None - slots = [] - logger.debug('Searching for nzoid from SAbnzbd ...') - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - url = base_url - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': 'queue', - 'output': 'json', - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return nzoid # failure - try: - result = r.json() - clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) - except Exception: - logger.warning('Data from SABnzbd queue could not be parsed') - params['mode'] = 'history' - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return nzoid # failure - try: - result = r.json() - clean_name = os.path.splitext(os.path.split(input_name)[1])[0] - slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) - except Exception: - logger.warning('Data from SABnzbd history could not be parsed') - try: - for nzo_id, name in slots: - if name in [input_name, clean_name]: - nzoid = nzo_id - logger.debug('Found nzoid: {0}'.format(nzoid)) - break - except Exception: - logger.warning('Data from SABnzbd could not be parsed') - return nzoid - - def clean_file_name(filename): """Cleans up nzb name by removing any . and _ characters, along with any trailing hyphens. diff --git a/core/utils/nzbs.py b/core/utils/nzbs.py new file mode 100644 index 00000000..ad3b4e04 --- /dev/null +++ b/core/utils/nzbs.py @@ -0,0 +1,70 @@ +import os + +import requests + +import core +from core import logger + + +def get_nzoid(input_name): + nzoid = None + slots = [] + logger.debug('Searching for nzoid from SAbnzbd ...') + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'queue', + 'output': 'json', + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) + except Exception: + logger.warning('Data from SABnzbd queue could not be parsed') + params['mode'] = 'history' + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) + except Exception: + logger.warning('Data from SABnzbd history could not be parsed') + try: + for nzo_id, name in slots: + if name in [input_name, clean_name]: + nzoid = nzo_id + logger.debug('Found nzoid: {0}'.format(nzoid)) + break + except Exception: + logger.warning('Data from SABnzbd could not be parsed') + return nzoid + + +def report_nzb(failure_link, client_agent): + # Contact indexer site + logger.info('Sending failure notification to indexer site') + if client_agent == 'nzbget': + headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} + elif client_agent == 'sabnzbd': + headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} + else: + return + try: + requests.post(failure_link, headers=headers, timeout=(30, 300)) + except Exception as e: + logger.error('Unable to open URL {0} due to {1}'.format(failure_link, e)) + return From 2d6e8034e2a5dd9275e0a8348fc7c97def45aa32 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:19:50 -0500 Subject: [PATCH 05/38] Refactor parses from utils to utils.parsers --- core/utils/__init__.py | 157 ++--------------------------------------- core/utils/parsers.py | 156 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 153 deletions(-) create mode 100644 core/utils/parsers.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 8e224974..130baa95 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,10 @@ import subliminal import core from core import extractor, logger, main_db +from core.utils.parsers import ( + parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, + parse_utorrent, parse_vuze, +) from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -479,159 +483,6 @@ def convert_to_ascii(input_name, dir_name): return input_name, dir_name -def parse_other(args): - return os.path.normpath(args[1]), '', '', '', '' - - -def parse_rtorrent(args): - # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, - # 'execute={/path/to/nzbToMedia/TorrentToMedia.py,\'$d.get_base_path=\',\'$d.get_name=\',\'$d.get_custom1=\',\'$d.get_hash=\'}' - input_directory = os.path.normpath(args[1]) - try: - input_name = args[2] - except Exception: - input_name = '' - try: - input_category = args[3] - except Exception: - input_category = '' - try: - input_hash = args[4] - except Exception: - input_hash = '' - try: - input_id = args[4] - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_utorrent(args): - # uTorrent usage: call TorrentToMedia.py '%D' '%N' '%L' '%I' - input_directory = os.path.normpath(args[1]) - input_name = args[2] - try: - input_category = args[3] - except Exception: - input_category = '' - try: - input_hash = args[4] - except Exception: - input_hash = '' - try: - input_id = args[4] - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_deluge(args): - # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR - input_directory = os.path.normpath(args[3]) - input_name = args[2] - input_hash = args[1] - input_id = args[1] - try: - input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] - except Exception: - input_category = '' - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_transmission(args): - # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) - input_directory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) - input_name = os.getenv('TR_TORRENT_NAME') - input_category = '' # We dont have a category yet - input_hash = os.getenv('TR_TORRENT_HASH') - input_id = os.getenv('TR_TORRENT_ID') - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_vuze(args): - # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D%N%L%I%K%F' - try: - cur_input = args[1].split(',') - except Exception: - cur_input = [] - try: - input_directory = os.path.normpath(cur_input[0]) - except Exception: - input_directory = '' - try: - input_name = cur_input[1] - except Exception: - input_name = '' - try: - input_category = cur_input[2] - except Exception: - input_category = '' - try: - input_hash = cur_input[3] - except Exception: - input_hash = '' - try: - input_id = cur_input[3] - except Exception: - input_id = '' - try: - if cur_input[4] == 'single': - input_name = cur_input[5] - except Exception: - pass - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_qbittorrent(args): - # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D|%N|%L|%I' - try: - cur_input = args[1].split('|') - except Exception: - cur_input = [] - try: - input_directory = os.path.normpath(cur_input[0].replace('\'', '')) - except Exception: - input_directory = '' - try: - input_name = cur_input[1].replace('\'', '') - except Exception: - input_name = '' - try: - input_category = cur_input[2].replace('\'', '') - except Exception: - input_category = '' - try: - input_hash = cur_input[3].replace('\'', '') - except Exception: - input_hash = '' - try: - input_id = cur_input[3].replace('\'', '') - except Exception: - input_id = '' - - return input_directory, input_name, input_category, input_hash, input_id - - -def parse_args(client_agent, args): - clients = { - 'other': parse_other, - 'rtorrent': parse_rtorrent, - 'utorrent': parse_utorrent, - 'deluge': parse_deluge, - 'transmission': parse_transmission, - 'qbittorrent': parse_qbittorrent, - 'vuze': parse_vuze, - } - - try: - return clients[client_agent](args) - except Exception: - return None, None, None, None, None - - def get_dirs(section, subsection, link='hard'): to_return = [] diff --git a/core/utils/parsers.py b/core/utils/parsers.py new file mode 100644 index 00000000..d5f6d933 --- /dev/null +++ b/core/utils/parsers.py @@ -0,0 +1,156 @@ +import os + +import core + + +def parse_other(args): + return os.path.normpath(args[1]), '', '', '', '' + + +def parse_rtorrent(args): + # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, + # 'execute={/path/to/nzbToMedia/TorrentToMedia.py,\'$d.get_base_path=\',\'$d.get_name=\',\'$d.get_custom1=\',\'$d.get_hash=\'}' + input_directory = os.path.normpath(args[1]) + try: + input_name = args[2] + except Exception: + input_name = '' + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_utorrent(args): + # uTorrent usage: call TorrentToMedia.py '%D' '%N' '%L' '%I' + input_directory = os.path.normpath(args[1]) + input_name = args[2] + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_deluge(args): + # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR + input_directory = os.path.normpath(args[3]) + input_name = args[2] + input_hash = args[1] + input_id = args[1] + try: + input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] + except Exception: + input_category = '' + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_transmission(args): + # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) + input_directory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) + input_name = os.getenv('TR_TORRENT_NAME') + input_category = '' # We dont have a category yet + input_hash = os.getenv('TR_TORRENT_HASH') + input_id = os.getenv('TR_TORRENT_ID') + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_vuze(args): + # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D%N%L%I%K%F' + try: + cur_input = args[1].split(',') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0]) + except Exception: + input_directory = '' + try: + input_name = cur_input[1] + except Exception: + input_name = '' + try: + input_category = cur_input[2] + except Exception: + input_category = '' + try: + input_hash = cur_input[3] + except Exception: + input_hash = '' + try: + input_id = cur_input[3] + except Exception: + input_id = '' + try: + if cur_input[4] == 'single': + input_name = cur_input[5] + except Exception: + pass + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_qbittorrent(args): + # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D|%N|%L|%I' + try: + cur_input = args[1].split('|') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0].replace('\'', '')) + except Exception: + input_directory = '' + try: + input_name = cur_input[1].replace('\'', '') + except Exception: + input_name = '' + try: + input_category = cur_input[2].replace('\'', '') + except Exception: + input_category = '' + try: + input_hash = cur_input[3].replace('\'', '') + except Exception: + input_hash = '' + try: + input_id = cur_input[3].replace('\'', '') + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_args(client_agent, args): + clients = { + 'other': parse_other, + 'rtorrent': parse_rtorrent, + 'utorrent': parse_utorrent, + 'deluge': parse_deluge, + 'transmission': parse_transmission, + 'qbittorrent': parse_qbittorrent, + 'vuze': parse_vuze, + } + + try: + return clients[client_agent](args) + except Exception: + return None, None, None, None, None From a50a5edbf70d2ce9941cd52e39c0ee2ceaf329b2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:31:48 -0500 Subject: [PATCH 06/38] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 39 ++------------------------------------ core/utils/paths.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 core/utils/paths.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 130baa95..23867ae1 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import datetime -from functools import partial import os import re import shutil @@ -26,6 +25,7 @@ from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, ) +from core.utils.paths import get_dir_size, make_dir, remote_dir from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -74,33 +74,6 @@ def sanitize_name(name): return name -def make_dir(path): - if not os.path.isdir(path): - try: - os.makedirs(path) - except Exception: - return False - return True - - -def remote_dir(path): - if not core.REMOTEPATHS: - return path - for local, remote in core.REMOTEPATHS: - if local in path: - base_dirs = path.replace(local, '').split(os.sep) - if '/' in remote: - remote_sep = '/' - else: - remote_sep = '\\' - new_path = remote_sep.join([remote] + base_dirs) - new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) - new_path = re.sub(r'(/+)', r'/', new_path) - new_path = re.sub(r'([/\\])$', r'', new_path) - return new_path - return path - - def category_search(input_directory, input_name, input_category, root, categories): tordir = False @@ -195,14 +168,6 @@ def category_search(input_directory, input_name, input_category, root, categorie return input_directory, input_name, input_category, root -def get_dir_size(input_path): - prepend = partial(os.path.join, input_path) - return sum([ - (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) - for f in map(prepend, os.listdir(text_type(input_path))) - ]) - - def is_min_size(input_name, min_size): file_name, file_ext = os.path.splitext(os.path.basename(input_name)) @@ -597,7 +562,7 @@ def onerror(func, path, exc_info): it attempts to add write permission and then retries. If the error is for another reason it re-raises the error. - + Usage : ``shutil.rmtree(path, onerror=onerror)`` """ if not os.access(path, os.W_OK): diff --git a/core/utils/paths.py b/core/utils/paths.py new file mode 100644 index 00000000..04392617 --- /dev/null +++ b/core/utils/paths.py @@ -0,0 +1,43 @@ + +from functools import partial +import os +import re + +from six import text_type + +import core + + +def make_dir(path): + if not os.path.isdir(path): + try: + os.makedirs(path) + except Exception: + return False + return True + + +def remote_dir(path): + if not core.REMOTEPATHS: + return path + for local, remote in core.REMOTEPATHS: + if local in path: + base_dirs = path.replace(local, '').split(os.sep) + if '/' in remote: + remote_sep = '/' + else: + remote_sep = '\\' + new_path = remote_sep.join([remote] + base_dirs) + new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) + new_path = re.sub(r'(/+)', r'/', new_path) + new_path = re.sub(r'([/\\])$', r'', new_path) + return new_path + return path + + +def get_dir_size(input_path): + prepend = partial(os.path.join, input_path) + return sum([ + (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) + for f in map(prepend, os.listdir(text_type(input_path))) + ]) From 04942bf6ad35a4c851f7df6aab5d538c71d17dfd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:51:20 -0500 Subject: [PATCH 07/38] Refactor download info to utils.download_info --- core/utils/__init__.py | 22 ++-------------------- core/utils/download_info.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 core/utils/download_info.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 23867ae1..f4a1f92c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -2,7 +2,6 @@ from __future__ import print_function, unicode_literals -import datetime import os import re import shutil @@ -20,7 +19,8 @@ from six import text_type import subliminal import core -from core import extractor, logger, main_db +from core import extractor, logger +from core.utils.download_info import get_download_info, update_download_info_status from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, @@ -950,21 +950,3 @@ def backup_versioned_file(old_file, version): return False return True - - -def update_download_info_status(input_name, status): - logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - - my_db = main_db.DBConnection() - my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) - - -def get_download_info(input_name, status): - logger.db('Getting download info for {0} from the DB'.format(input_name)) - - my_db = main_db.DBConnection() - sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) - - return sql_results diff --git a/core/utils/download_info.py b/core/utils/download_info.py new file mode 100644 index 00000000..2c2f7da4 --- /dev/null +++ b/core/utils/download_info.py @@ -0,0 +1,23 @@ +import datetime + +from six import text_type + +from core import logger, main_db + + +def update_download_info_status(input_name, status): + logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) + + my_db = main_db.DBConnection() + my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', + [status, datetime.date.today().toordinal(), text_type(input_name)]) + + +def get_download_info(input_name, status): + logger.db('Getting download info for {0} from the DB'.format(input_name)) + + my_db = main_db.DBConnection() + sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', + [text_type(input_name), status]) + + return sql_results From 542893b30bd1a41012e4cf6396c3417f31f69615 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:52:53 -0500 Subject: [PATCH 08/38] Refactor download_info db connection to module variable --- core/utils/download_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index 2c2f7da4..1ef100a2 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -4,11 +4,12 @@ from six import text_type from core import logger, main_db +my_db = main_db.DBConnection() + def update_download_info_status(input_name, status): logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - my_db = main_db.DBConnection() my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', [status, datetime.date.today().toordinal(), text_type(input_name)]) @@ -16,7 +17,6 @@ def update_download_info_status(input_name, status): def get_download_info(input_name, status): logger.db('Getting download info for {0} from the DB'.format(input_name)) - my_db = main_db.DBConnection() sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', [text_type(input_name), status]) From 7b8721b27709a86e596ba18b87e97f21df86f6a6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 14:53:13 -0500 Subject: [PATCH 09/38] Refactor my_db -> database --- core/utils/download_info.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index 1ef100a2..c4cc530e 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -4,20 +4,20 @@ from six import text_type from core import logger, main_db -my_db = main_db.DBConnection() +database = main_db.DBConnection() def update_download_info_status(input_name, status): logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) + database.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', + [status, datetime.date.today().toordinal(), text_type(input_name)]) def get_download_info(input_name, status): logger.db('Getting download info for {0} from the DB'.format(input_name)) - sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) + sql_results = database.select('SELECT * FROM downloads WHERE input_name=? AND status=?', + [text_type(input_name), status]) return sql_results From 84061fea2f05c5f6c24e5221c35e98da43bb7b4b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 15:30:53 -0500 Subject: [PATCH 10/38] Fix PEP8 line length --- core/utils/download_info.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/utils/download_info.py b/core/utils/download_info.py index c4cc530e..ce6e6717 100644 --- a/core/utils/download_info.py +++ b/core/utils/download_info.py @@ -8,16 +8,16 @@ database = main_db.DBConnection() def update_download_info_status(input_name, status): - logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) - - database.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', - [status, datetime.date.today().toordinal(), text_type(input_name)]) + msg = 'Updating DB download status of {0} to {1}' + action = 'UPDATE downloads SET status=?, last_update=? WHERE input_name=?' + args = [status, datetime.date.today().toordinal(), text_type(input_name)] + logger.db(msg.format(input_name, status)) + database.action(action, args) def get_download_info(input_name, status): - logger.db('Getting download info for {0} from the DB'.format(input_name)) - - sql_results = database.select('SELECT * FROM downloads WHERE input_name=? AND status=?', - [text_type(input_name), status]) - - return sql_results + msg = 'Getting download info for {0} from the DB' + action = 'SELECT * FROM downloads WHERE input_name=? AND status=?' + args = [text_type(input_name), status] + logger.db(msg.format(input_name)) + return database.select(action, args) From 42553df5cbfe9c66d454239393fbd508167c6785 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 15:33:56 -0500 Subject: [PATCH 11/38] Refactor network utils to utils.network --- core/utils/__init__.py | 53 +-------------------------------------- core/utils/network.py | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 52 deletions(-) create mode 100644 core/utils/network.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index f4a1f92c..d97e9611 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -5,9 +5,7 @@ from __future__ import print_function, unicode_literals import os import re import shutil -import socket import stat -import struct import time from babelfish import Language @@ -21,6 +19,7 @@ import subliminal import core from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, parse_utorrent, parse_vuze, @@ -319,56 +318,6 @@ def remove_read_only(filename): logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) -# Wake function -def wake_on_lan(ethernet_address): - addr_byte = ethernet_address.split(':') - hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), - int(addr_byte[1], 16), - int(addr_byte[2], 16), - int(addr_byte[3], 16), - int(addr_byte[4], 16), - int(addr_byte[5], 16)) - - # Build the Wake-On-LAN 'Magic Packet'... - - msg = b'\xff' * 6 + hw_addr * 16 - - # ...and send it to the broadcast address using UDP - - ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - ss.sendto(msg, ('', 9)) - ss.close() - - -# Test Connection function -def test_connection(host, port): - try: - socket.create_connection((host, port)) - return 'Up' - except Exception: - return 'Down' - - -def wake_up(): - host = core.CFG['WakeOnLan']['host'] - port = int(core.CFG['WakeOnLan']['port']) - mac = core.CFG['WakeOnLan']['mac'] - - i = 1 - while test_connection(host, port) == 'Down' and i < 4: - logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) - wake_on_lan(mac) - time.sleep(20) - i = i + 1 - - if test_connection(host, port) == 'Down': # final check. - logger.warning('System with mac: {0} has not woken after 3 attempts. ' - 'Continuing with the rest of the script.'.format(mac)) - else: - logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) - - def char_replace(name): # Special character hex range: # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) diff --git a/core/utils/network.py b/core/utils/network.py new file mode 100644 index 00000000..4409c486 --- /dev/null +++ b/core/utils/network.py @@ -0,0 +1,56 @@ +import socket +import struct +import time + +import core +from core import logger + + +# Wake function +def wake_on_lan(ethernet_address): + addr_byte = ethernet_address.split(':') + hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), + int(addr_byte[1], 16), + int(addr_byte[2], 16), + int(addr_byte[3], 16), + int(addr_byte[4], 16), + int(addr_byte[5], 16)) + + # Build the Wake-On-LAN 'Magic Packet'... + + msg = b'\xff' * 6 + hw_addr * 16 + + # ...and send it to the broadcast address using UDP + + ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + ss.sendto(msg, ('', 9)) + ss.close() + + +# Test Connection function +def test_connection(host, port): + try: + socket.create_connection((host, port)) + return 'Up' + except Exception: + return 'Down' + + +def wake_up(): + host = core.CFG['WakeOnLan']['host'] + port = int(core.CFG['WakeOnLan']['port']) + mac = core.CFG['WakeOnLan']['mac'] + + i = 1 + while test_connection(host, port) == 'Down' and i < 4: + logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) + wake_on_lan(mac) + time.sleep(20) + i = i + 1 + + if test_connection(host, port) == 'Down': # final check. + logger.warning('System with mac: {0} has not woken after 3 attempts. ' + 'Continuing with the rest of the script.'.format(mac)) + else: + logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) From 094fe555b8f60acf4c379e659e9075da07ae8ac4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 16:38:33 -0500 Subject: [PATCH 12/38] Clean up network utils --- core/utils/network.py | 74 ++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/core/utils/network.py b/core/utils/network.py index 4409c486..dae99837 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -6,51 +6,61 @@ import core from core import logger -# Wake function +def make_wake_on_lan_packet(mac_address): + """Build the Wake-On-LAN 'Magic Packet'.""" + address = ( + int(value, 16) + for value in mac_address.split(':') + ) + fmt = 'BBBBBB' + hardware_address = struct.pack(fmt, *address) + broadcast_address = b'\xFF' * 6 # FF:FF:FF:FF:FF:FF + return broadcast_address + hardware_address * 16 + + def wake_on_lan(ethernet_address): - addr_byte = ethernet_address.split(':') - hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), - int(addr_byte[1], 16), - int(addr_byte[2], 16), - int(addr_byte[3], 16), - int(addr_byte[4], 16), - int(addr_byte[5], 16)) - - # Build the Wake-On-LAN 'Magic Packet'... - - msg = b'\xff' * 6 + hw_addr * 16 + """Send a WakeOnLan request.""" + # Create the WoL magic packet + magic_packet = make_wake_on_lan_packet(ethernet_address) # ...and send it to the broadcast address using UDP + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + connection.sendto(magic_packet, ('', 9)) - ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - ss.sendto(msg, ('', 9)) - ss.close() + logger.info('WakeOnLan sent for mac: {0}'.format(ethernet_address)) -# Test Connection function def test_connection(host, port): + """Test network connection.""" + address = host, port try: - socket.create_connection((host, port)) - return 'Up' - except Exception: + socket.create_connection(address) + except socket.error: return 'Down' + else: + return 'Up' def wake_up(): - host = core.CFG['WakeOnLan']['host'] - port = int(core.CFG['WakeOnLan']['port']) - mac = core.CFG['WakeOnLan']['mac'] + wol = core.CFG['WakeOnLan'] + host = wol['host'] + port = int(wol['port']) + mac = wol['mac'] + max_attempts = 4 - i = 1 - while test_connection(host, port) == 'Down' and i < 4: - logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) + logger.info('Trying to wake On lan.') + + for attempt in range(0, max_attempts): + logger.info('Attempt {0} of {1}'.format(attempt + 1, max_attempts, mac)) + if test_connection(host, port) == 'Up': + logger.info('System with mac: {0} has been woken.'.format(mac)) + break wake_on_lan(mac) time.sleep(20) - i = i + 1 - - if test_connection(host, port) == 'Down': # final check. - logger.warning('System with mac: {0} has not woken after 3 attempts. ' - 'Continuing with the rest of the script.'.format(mac)) else: - logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) + if test_connection(host, port) == 'Down': # final check. + msg = 'System with mac: {0} has not woken after {1} attempts.' + logger.warning(msg.format(mac, max_attempts)) + + logger.info('Continuing with the rest of the script.') From a6d2c6e96fd871bc999b4f0ccc20e7bb43b46a25 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:12:43 -0500 Subject: [PATCH 13/38] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 52 ++++++++++++------------------------------ core/utils/paths.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d97e9611..376c4898 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -21,10 +21,21 @@ from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( - parse_args, parse_deluge, parse_other, parse_qbittorrent, parse_rtorrent, parse_transmission, - parse_utorrent, parse_vuze, + parse_args, + parse_deluge, + parse_other, + parse_qbittorrent, + parse_rtorrent, + parse_transmission, + parse_utorrent, + parse_vuze, +) +from core.utils.paths import ( + get_dir_size, make_dir, + remote_dir, + remove_empty_folders, + remove_read_only, ) -from core.utils.paths import get_dir_size, make_dir, remote_dir from core.utils.processes import RunningProcess from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent @@ -283,41 +294,6 @@ def flatten(output_destination): remove_empty_folders(output_destination) # Cleanup empty directories -def remove_empty_folders(path, remove_root=True): - """Function to remove empty folders""" - if not os.path.isdir(path): - return - - # remove empty subfolders - logger.debug('Checking for empty folders in:{0}'.format(path)) - files = os.listdir(text_type(path)) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - remove_empty_folders(fullpath) - - # if folder empty, delete it - files = os.listdir(text_type(path)) - if len(files) == 0 and remove_root: - logger.debug('Removing empty folder:{}'.format(path)) - os.rmdir(path) - - -def remove_read_only(filename): - if os.path.isfile(filename): - # check first the read-only attribute - file_attribute = os.stat(filename)[0] - if not file_attribute & stat.S_IWRITE: - # File is read-only, so make it writeable - logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format - (name=filename)) - try: - os.chmod(filename, stat.S_IWRITE) - except Exception: - logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) - - def char_replace(name): # Special character hex range: # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) diff --git a/core/utils/paths.py b/core/utils/paths.py index 04392617..31308650 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -2,10 +2,12 @@ from functools import partial import os import re +import stat from six import text_type import core +from core import logger def make_dir(path): @@ -41,3 +43,38 @@ def get_dir_size(input_path): (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) for f in map(prepend, os.listdir(text_type(input_path))) ]) + + +def remove_empty_folders(path, remove_root=True): + """Function to remove empty folders""" + if not os.path.isdir(path): + return + + # remove empty subfolders + logger.debug('Checking for empty folders in:{0}'.format(path)) + files = os.listdir(text_type(path)) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + remove_empty_folders(fullpath) + + # if folder empty, delete it + files = os.listdir(text_type(path)) + if len(files) == 0 and remove_root: + logger.debug('Removing empty folder:{}'.format(path)) + os.rmdir(path) + + +def remove_read_only(filename): + if os.path.isfile(filename): + # check first the read-only attribute + file_attribute = os.stat(filename)[0] + if not file_attribute & stat.S_IWRITE: + # File is read-only, so make it writeable + logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format + (name=filename)) + try: + os.chmod(filename, stat.S_IWRITE) + except Exception: + logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) From 5c644890e83662e7b749590c70e5ddf0dc7adca5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:02:03 -0500 Subject: [PATCH 14/38] Fix shutil.copyfileobj monkey patching --- core/utils/__init__.py | 11 ++--------- core/utils/shutil_custom.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 core/utils/shutil_custom.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 376c4898..7716011f 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -19,6 +19,7 @@ import subliminal import core from core import extractor, logger from core.utils.download_info import get_download_info, update_download_info_status +from core.utils import shutil_custom from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -47,15 +48,7 @@ except ImportError: requests.packages.urllib3.disable_warnings() -# Monkey Patch shutil.copyfileobj() to adjust the buffer length to 512KB rather than 4KB -shutil.copyfileobjOrig = shutil.copyfileobj - - -def copyfileobj_fast(fsrc, fdst, length=512 * 1024): - shutil.copyfileobjOrig(fsrc, fdst, length=length) - - -shutil.copyfileobj = copyfileobj_fast +shutil_custom.monkey_patch() def sanitize_name(name): diff --git a/core/utils/shutil_custom.py b/core/utils/shutil_custom.py new file mode 100644 index 00000000..5525df1f --- /dev/null +++ b/core/utils/shutil_custom.py @@ -0,0 +1,11 @@ +from functools import partial +import shutil +from six import PY2 + + +def monkey_patch(length=512 * 1024): + if PY2: + # On Python 2 monkey patch shutil.copyfileobj() + # to adjust the buffer length to 512KB rather than 4KB + original_copyfileobj = shutil.copyfileobj + shutil.copyfileobj = partial(original_copyfileobj, length=length) From a14a286a8e66bb51aa8edf0d92d68acf90fac1dd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:07:52 -0500 Subject: [PATCH 15/38] Refactor links to utils.links --- core/utils/__init__.py | 78 +-------------------------------------- core/utils/links.py | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 76 deletions(-) create mode 100644 core/utils/links.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 7716011f..0ad08985 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -11,15 +11,15 @@ import time from babelfish import Language import beets import guessit -import linktastic import requests from six import text_type import subliminal import core from core import extractor, logger -from core.utils.download_info import get_download_info, update_download_info_status from core.utils import shutil_custom +from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.links import copy_link, replace_links from core.utils.network import test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -194,80 +194,6 @@ def is_sample(input_name): return True -def copy_link(src, target_link, use_link): - logger.info('MEDIAFILE: [{0}]'.format(os.path.basename(target_link)), 'COPYLINK') - logger.info('SOURCE FOLDER: [{0}]'.format(os.path.dirname(src)), 'COPYLINK') - logger.info('TARGET FOLDER: [{0}]'.format(os.path.dirname(target_link)), 'COPYLINK') - - if src != target_link and os.path.exists(target_link): - logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') - return True - elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): - logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') - return True - elif src == os.path.dirname(target_link): - logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') - return True - - make_dir(os.path.dirname(target_link)) - try: - if use_link == 'dir': - logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') - linktastic.dirlink(src, target_link) - return True - if use_link == 'junction': - logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') - linktastic.dirlink(src, target_link) - return True - elif use_link == 'hard': - logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - linktastic.link(src, target_link) - return True - elif use_link == 'sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - linktastic.symlink(src, target_link) - return True - elif use_link == 'move-sym': - logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.move(src, target_link) - linktastic.symlink(target_link, src) - return True - elif use_link == 'move': - logger.info('Moving SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.move(src, target_link) - return True - except Exception as e: - logger.warning('Error: {0}, copying instead ... '.format(e), 'COPYLINK') - - logger.info('Copying SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') - shutil.copy(src, target_link) - - return True - - -def replace_links(link): - n = 0 - target = link - if os.name == 'nt': - if not jaraco.windows.filesystem.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while jaraco.windows.filesystem.islink(target): - target = jaraco.windows.filesystem.readlink(target) - n = n + 1 - else: - if not os.path.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while os.path.islink(target): - target = os.readlink(target) - n = n + 1 - if n > 1: - logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') - os.unlink(link) - linktastic.symlink(target, link) - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): diff --git a/core/utils/links.py b/core/utils/links.py new file mode 100644 index 00000000..ca5e99e1 --- /dev/null +++ b/core/utils/links.py @@ -0,0 +1,84 @@ +import os +import shutil + +import linktastic + +from core import logger +from core.utils.paths import make_dir + +if os.name == 'nt': + import jaraco + + +def copy_link(src, target_link, use_link): + logger.info('MEDIAFILE: [{0}]'.format(os.path.basename(target_link)), 'COPYLINK') + logger.info('SOURCE FOLDER: [{0}]'.format(os.path.dirname(src)), 'COPYLINK') + logger.info('TARGET FOLDER: [{0}]'.format(os.path.dirname(target_link)), 'COPYLINK') + + if src != target_link and os.path.exists(target_link): + logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') + return True + elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): + logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') + return True + elif src == os.path.dirname(target_link): + logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') + return True + + make_dir(os.path.dirname(target_link)) + try: + if use_link == 'dir': + logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + if use_link == 'junction': + logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + elif use_link == 'hard': + logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.link(src, target_link) + return True + elif use_link == 'sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.symlink(src, target_link) + return True + elif use_link == 'move-sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + linktastic.symlink(target_link, src) + return True + elif use_link == 'move': + logger.info('Moving SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + return True + except Exception as e: + logger.warning('Error: {0}, copying instead ... '.format(e), 'COPYLINK') + + logger.info('Copying SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.copy(src, target_link) + + return True + + +def replace_links(link): + n = 0 + target = link + if os.name == 'nt': + if not jaraco.windows.filesystem.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while jaraco.windows.filesystem.islink(target): + target = jaraco.windows.filesystem.readlink(target) + n = n + 1 + else: + if not os.path.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while os.path.islink(target): + target = os.readlink(target) + n = n + 1 + if n > 1: + logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') + os.unlink(link) + linktastic.symlink(target, link) From c4d9faeb23f2c8c15b32986676c6dea4e3ef6cca Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:19:21 -0500 Subject: [PATCH 16/38] Refactor network utils to utils.network --- core/utils/__init__.py | 46 +-------------------------------- core/utils/network.py | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 0ad08985..23dfb678 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -20,7 +20,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.links import copy_link, replace_links -from core.utils.network import test_connection, wake_on_lan, wake_up +from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, parse_deluge, @@ -453,50 +453,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def find_download(client_agent, download_id): - logger.debug('Searching for Download on {0} ...'.format(client_agent)) - if client_agent == 'utorrent': - torrents = core.TORRENT_CLASS.list()[1]['torrents'] - for torrent in torrents: - if download_id in torrent: - return True - if client_agent == 'transmission': - torrents = core.TORRENT_CLASS.get_torrents() - for torrent in torrents: - torrent_hash = torrent.hashString - if torrent_hash == download_id: - return True - if client_agent == 'deluge': - return False - if client_agent == 'qbittorrent': - torrents = core.TORRENT_CLASS.torrents() - for torrent in torrents: - if torrent['hash'] == download_id: - return True - if client_agent == 'sabnzbd': - if 'http' in core.SABNZBDHOST: - base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) - url = base_url - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': 'get_files', - 'output': 'json', - 'value': download_id, - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error('Unable to open URL') - return False # failure - - result = r.json() - if result['files']: - return True - return False - - def clean_file_name(filename): """Cleans up nzb name by removing any . and _ characters, along with any trailing hyphens. diff --git a/core/utils/network.py b/core/utils/network.py index dae99837..99a89e1e 100644 --- a/core/utils/network.py +++ b/core/utils/network.py @@ -2,6 +2,8 @@ import socket import struct import time +import requests + import core from core import logger @@ -64,3 +66,59 @@ def wake_up(): logger.warning(msg.format(mac, max_attempts)) logger.info('Continuing with the rest of the script.') + + +def server_responding(base_url): + logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') + try: + requests.get(base_url, timeout=(60, 120), verify=False) + except (requests.ConnectionError, requests.exceptions.Timeout): + logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') + return False + else: + logger.debug('Server responded at {0}'.format(base_url), 'SERVER') + return True + + +def find_download(client_agent, download_id): + logger.debug('Searching for Download on {0} ...'.format(client_agent)) + if client_agent == 'utorrent': + torrents = core.TORRENT_CLASS.list()[1]['torrents'] + for torrent in torrents: + if download_id in torrent: + return True + if client_agent == 'transmission': + torrents = core.TORRENT_CLASS.get_torrents() + for torrent in torrents: + torrent_hash = torrent.hashString + if torrent_hash == download_id: + return True + if client_agent == 'deluge': + return False + if client_agent == 'qbittorrent': + torrents = core.TORRENT_CLASS.torrents() + for torrent in torrents: + if torrent['hash'] == download_id: + return True + if client_agent == 'sabnzbd': + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'get_files', + 'output': 'json', + 'value': download_id, + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return False # failure + + result = r.json() + if result['files']: + return True + return False From f042e014b1f9615ad849abb5f8e5a7d9022c8269 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:22:40 -0500 Subject: [PATCH 17/38] Refactor naming utils to utils.naming --- core/utils/__init__.py | 33 +-------------------------- core/utils/naming.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 core/utils/naming.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 23dfb678..4987314b 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -20,6 +20,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.links import copy_link, replace_links +from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.parsers import ( parse_args, @@ -51,32 +52,6 @@ requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() -def sanitize_name(name): - """ - >>> sanitize_name('a/b/c') - 'a-b-c' - >>> sanitize_name('abc') - 'abc' - >>> sanitize_name('a"b') - 'ab' - >>> sanitize_name('.a.b..') - 'a.b' - """ - - # remove bad chars from the filename - name = re.sub(r'[\\/*]', '-', name) - name = re.sub(r'[:\'<>|?]', '', name) - - # remove leading/trailing periods and spaces - name = name.strip(' .') - try: - name = name.encode(core.SYS_ENCODING) - except Exception: - pass - - return name - - def category_search(input_directory, input_name, input_category, root, categories): tordir = False @@ -188,12 +163,6 @@ def is_min_size(input_name, min_size): return True -def is_sample(input_name): - # Ignore 'sample' in files - if re.search('(^|[\\W_])sample\\d*[\\W_]', input_name.lower()): - return True - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): diff --git a/core/utils/naming.py b/core/utils/naming.py new file mode 100644 index 00000000..352d51ba --- /dev/null +++ b/core/utils/naming.py @@ -0,0 +1,52 @@ +import re +import core + + +def sanitize_name(name): + """ + >>> sanitize_name('a/b/c') + 'a-b-c' + >>> sanitize_name('abc') + 'abc' + >>> sanitize_name('a"b') + 'ab' + >>> sanitize_name('.a.b..') + 'a.b' + """ + + # remove bad chars from the filename + name = re.sub(r'[\\/*]', '-', name) + name = re.sub(r'[:\'<>|?]', '', name) + + # remove leading/trailing periods and spaces + name = name.strip(' .') + try: + name = name.encode(core.SYS_ENCODING) + except Exception: + pass + + return name + + +def clean_file_name(filename): + """Cleans up nzb name by removing any . and _ + characters, along with any trailing hyphens. + + Is basically equivalent to replacing all _ and . with a + space, but handles decimal numbers in string, for example: + """ + + filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) + filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot + filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) + filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) + filename = filename.replace('_', ' ') + filename = re.sub('-$', '', filename) + filename = re.sub(r'^\[.*]', '', filename) + return filename.strip() + + +def is_sample(input_name): + # Ignore 'sample' in files + if re.search('(^|[\\W_])sample\\d*[\\W_]', input_name.lower()): + return True From 9d43e0d60b9d721b723608705ee1640df3174fe0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 16:45:02 -0500 Subject: [PATCH 18/38] Refactor notification utils to utils.notifications --- core/utils/__init__.py | 25 +------------------------ core/utils/notifications.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 core/utils/notifications.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 4987314b..46a30431 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,7 @@ from core.utils.download_info import get_download_info, update_download_info_sta from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up +from core.utils.notifications import plex_update from core.utils.parsers import ( parse_args, parse_deluge, @@ -668,30 +669,6 @@ def server_responding(base_url): return False -def plex_update(category): - if core.FAILED: - return - url = '{scheme}://{host}:{port}/library/sections/'.format( - scheme='https' if core.PLEXSSL else 'http', - host=core.PLEXHOST, - port=core.PLEXPORT, - ) - section = None - if not core.PLEXSEC: - return - logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') - for item in core.PLEXSEC: - if item[0] == category: - section = item[1] - - if section: - url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) - requests.get(url, timeout=(60, 120), verify=False) - logger.debug('Plex Library has been refreshed.', 'PLEX') - else: - logger.debug('Could not identify section for plex update', 'PLEX') - - def backup_versioned_file(old_file, version): num_tries = 0 diff --git a/core/utils/notifications.py b/core/utils/notifications.py new file mode 100644 index 00000000..ed89f65b --- /dev/null +++ b/core/utils/notifications.py @@ -0,0 +1,30 @@ +import requests + +import core +from core import logger + + +def plex_update(category): + if core.FAILED: + return + url = '{scheme}://{host}:{port}/library/sections/'.format( + scheme='https' if core.PLEXSSL else 'http', + host=core.PLEXHOST, + port=core.PLEXPORT, + ) + section = None + if not core.PLEXSEC: + return + logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') + for item in core.PLEXSEC: + if item[0] == category: + section = item[1] + + if section: + url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) + requests.get(url, timeout=(60, 120), verify=False) + logger.debug('Plex Library has been refreshed.', 'PLEX') + else: + logger.debug('Could not identify section for plex update', 'PLEX') + + From 6a9ff96e8c9e4d28d4deff98fb13aeb2016d1b78 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:25:36 -0500 Subject: [PATCH 19/38] Refactor encoding utils to utils.encoding --- core/utils/__init__.py | 80 +-------------------------------------- core/utils/encoding.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 79 deletions(-) create mode 100644 core/utils/encoding.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 46a30431..d0bc778a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -19,6 +19,7 @@ import core from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status +from core.utils.encoding import char_replace, convert_to_ascii from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -183,85 +184,6 @@ def flatten(output_destination): remove_empty_folders(output_destination) # Cleanup empty directories -def char_replace(name): - # Special character hex range: - # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) - # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF - # ISO-8859-15: 0xA6-0xFF - # The function will detect if Name contains a special character - # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding - encoded = False - encoding = None - if isinstance(name, text_type): - return encoded, name.encode(core.SYS_ENCODING) - for Idx in range(len(name)): - # /!\ detection is done 2char by 2char for UTF-8 special character - if (len(name) != 1) & (Idx < (len(name) - 1)): - # Detect UTF-8 - if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & ( - (name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')): - encoding = 'utf-8' - break - # Detect CP850 - elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - else: - # Detect CP850 - if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - if encoding and not encoding == core.SYS_ENCODING: - encoded = True - name = name.decode(encoding).encode(core.SYS_ENCODING) - return encoded, name - - -def convert_to_ascii(input_name, dir_name): - - ascii_convert = int(core.CFG['ASCII']['convert']) - if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. - return input_name, dir_name - - encoded, input_name = char_replace(input_name) - - directory, base = os.path.split(dir_name) - if not base: # ended with '/' - directory, base = os.path.split(directory) - - encoded, base2 = char_replace(base) - if encoded: - dir_name = os.path.join(directory, base2) - logger.info('Renaming directory to: {0}.'.format(base2), 'ENCODER') - os.rename(os.path.join(directory, base), dir_name) - if 'NZBOP_SCRIPTDIR' in os.environ: - print('[NZB] DIRECTORY={0}'.format(dir_name)) - - for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): - for subdirname in dirnames: - encoded, subdirname2 = char_replace(subdirname) - if encoded: - logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') - os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) - - for dirname, dirnames, filenames in os.walk(dir_name): - for filename in filenames: - encoded, filename2 = char_replace(filename) - if encoded: - logger.info('Renaming file to: {0}.'.format(filename2), 'ENCODER') - os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) - - return input_name, dir_name - - def get_dirs(section, subsection, link='hard'): to_return = [] diff --git a/core/utils/encoding.py b/core/utils/encoding.py new file mode 100644 index 00000000..ca19e054 --- /dev/null +++ b/core/utils/encoding.py @@ -0,0 +1,85 @@ +import os + +from six import text_type + +import core +from core import logger + + +def char_replace(name): + # Special character hex range: + # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) + # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF + # ISO-8859-15: 0xA6-0xFF + # The function will detect if Name contains a special character + # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding + encoded = False + encoding = None + if isinstance(name, text_type): + return encoded, name.encode(core.SYS_ENCODING) + for Idx in range(len(name)): + # /!\ detection is done 2char by 2char for UTF-8 special character + if (len(name) != 1) & (Idx < (len(name) - 1)): + # Detect UTF-8 + if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & ( + (name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')): + encoding = 'utf-8' + break + # Detect CP850 + elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + else: + # Detect CP850 + if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + if encoding and not encoding == core.SYS_ENCODING: + encoded = True + name = name.decode(encoding).encode(core.SYS_ENCODING) + return encoded, name + + +def convert_to_ascii(input_name, dir_name): + + ascii_convert = int(core.CFG['ASCII']['convert']) + if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. + return input_name, dir_name + + encoded, input_name = char_replace(input_name) + + directory, base = os.path.split(dir_name) + if not base: # ended with '/' + directory, base = os.path.split(directory) + + encoded, base2 = char_replace(base) + if encoded: + dir_name = os.path.join(directory, base2) + logger.info('Renaming directory to: {0}.'.format(base2), 'ENCODER') + os.rename(os.path.join(directory, base), dir_name) + if 'NZBOP_SCRIPTDIR' in os.environ: + print('[NZB] DIRECTORY={0}'.format(dir_name)) + + for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): + for subdirname in dirnames: + encoded, subdirname2 = char_replace(subdirname) + if encoded: + logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') + os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) + + for dirname, dirnames, filenames in os.walk(dir_name): + for filename in filenames: + encoded, filename2 = char_replace(filename) + if encoded: + logger.info('Renaming file to: {0}.'.format(filename2), 'ENCODER') + os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) + + return input_name, dir_name From 2d4951267bca16ca154ad21f108159be823e9bed Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sat, 5 Jan 2019 17:27:14 -0500 Subject: [PATCH 20/38] Refactor subtitle utils to utils.subtitles --- core/utils/__init__.py | 29 +---------------------------- core/utils/subtitles.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 core/utils/subtitles.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d0bc778a..d04c4f1c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -8,12 +8,10 @@ import shutil import stat import time -from babelfish import Language import beets import guessit import requests from six import text_type -import subliminal import core from core import extractor, logger @@ -41,6 +39,7 @@ from core.utils.paths import ( remove_read_only, ) from core.utils.processes import RunningProcess +from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent try: @@ -554,32 +553,6 @@ def extract_files(src, dst=None, keep_archive=None): logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) -def import_subs(filename): - if not core.GETSUBS: - return - try: - subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) - except Exception: - pass - - languages = set() - for item in core.SLANGUAGES: - try: - languages.add(Language(item)) - except Exception: - pass - if not languages: - return - - logger.info('Attempting to download subtitles for {0}'.format(filename), 'SUBTITLES') - try: - video = subliminal.scan_video(filename) - subtitles = subliminal.download_best_subtitles({video}, languages) - subliminal.save_subtitles(video, subtitles[video]) - except Exception as e: - logger.error('Failed to download subtitles for {0} due to: {1}'.format(filename, e), 'SUBTITLES') - - def server_responding(base_url): logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') try: diff --git a/core/utils/subtitles.py b/core/utils/subtitles.py new file mode 100644 index 00000000..f62f90de --- /dev/null +++ b/core/utils/subtitles.py @@ -0,0 +1,31 @@ +from babelfish import Language +import subliminal + +import core +from core import logger + + +def import_subs(filename): + if not core.GETSUBS: + return + try: + subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) + except Exception: + pass + + languages = set() + for item in core.SLANGUAGES: + try: + languages.add(Language(item)) + except Exception: + pass + if not languages: + return + + logger.info('Attempting to download subtitles for {0}'.format(filename), 'SUBTITLES') + try: + video = subliminal.scan_video(filename) + subtitles = subliminal.download_best_subtitles({video}, languages) + subliminal.save_subtitles(video, subtitles[video]) + except Exception as e: + logger.error('Failed to download subtitles for {0} due to: {1}'.format(filename, e), 'SUBTITLES') From d1f5211e78d9baae5691d6a21186f53eb1bb398c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 02:33:15 -0500 Subject: [PATCH 21/38] Refactor nzbs from utils to utils.nzbs --- core/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index d04c4f1c..fd646d7e 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -22,6 +22,7 @@ from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up from core.utils.notifications import plex_update +from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( parse_args, parse_deluge, From 9b0d53942373ad79383ec5acd30a15235cdfdbef Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:30:58 -0500 Subject: [PATCH 22/38] Make replace_links more DRY and add max_depth for following links --- core/utils/links.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/core/utils/links.py b/core/utils/links.py index ca5e99e1..ab558e9b 100644 --- a/core/utils/links.py +++ b/core/utils/links.py @@ -6,8 +6,14 @@ import linktastic from core import logger from core.utils.paths import make_dir -if os.name == 'nt': - import jaraco +try: + from jaraco.windows.filesystem import islink, readlink +except ImportError: + if os.name == 'nt': + raise + else: + from os.path import islink + from os import readlink def copy_link(src, target_link, use_link): @@ -61,24 +67,21 @@ def copy_link(src, target_link, use_link): return True -def replace_links(link): - n = 0 +def replace_links(link, max_depth=10): + link_depth = 0 target = link - if os.name == 'nt': - if not jaraco.windows.filesystem.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while jaraco.windows.filesystem.islink(target): - target = jaraco.windows.filesystem.readlink(target) - n = n + 1 + + for attempt in range(0, max_depth): + if not islink(target): + break + target = readlink(target) + link_depth = attempt + + if not link_depth: + logger.debug('{0} is not a link'.format(link)) + elif link_depth > max_depth or (link_depth == max_depth and islink(target)): + logger.warning('Exceeded maximum depth {0} while following link {1}'.format(max_depth, link)) else: - if not os.path.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while os.path.islink(target): - target = os.readlink(target) - n = n + 1 - if n > 1: logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') os.unlink(link) linktastic.symlink(target, link) From a074e566297252b0acff7c088c109a4003ecf9b4 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:45:26 -0500 Subject: [PATCH 23/38] Refactor naming utils to utils.naming --- core/utils/__init__.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fd646d7e..a8dbe792 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -345,24 +345,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def clean_file_name(filename): - """Cleans up nzb name by removing any . and _ - characters, along with any trailing hyphens. - - Is basically equivalent to replacing all _ and . with a - space, but handles decimal numbers in string, for example: - """ - - filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) - filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot - filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) - filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) - filename = filename.replace('_', ' ') - filename = re.sub('-$', '', filename) - filename = re.sub(r'^\[.*]', '', filename) - return filename.strip() - - def is_archive_file(filename): """Check if the filename is allowed for the Archive""" for regext in core.COMPRESSEDCONTAINER: From 0cccecb43565eddccdcc21cb3afdd8f1dd889682 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 09:53:01 -0500 Subject: [PATCH 24/38] Refactor file type detection to utils.files --- core/utils/__init__.py | 95 +------------------------------------- core/utils/files.py | 102 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 94 deletions(-) create mode 100644 core/utils/files.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index a8dbe792..cdfb9418 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -18,6 +18,7 @@ from core import extractor, logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii +from core.utils.files import is_archive_file, is_media_file, is_min_size, list_media_files from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -148,23 +149,6 @@ def category_search(input_directory, input_name, input_category, root, categorie return input_directory, input_name, input_category, root -def is_min_size(input_name, min_size): - file_name, file_ext = os.path.splitext(os.path.basename(input_name)) - - # audio files we need to check directory size not file size - input_size = os.path.getsize(input_name) - if file_ext in core.AUDIOCONTAINER: - try: - input_size = get_dir_size(os.path.dirname(input_name)) - except Exception: - logger.error('Failed to get file size for {0}'.format(input_name), 'MINSIZE') - return True - - # Ignore files under a certain size - if input_size > min_size * 1048576: - return True - - def flatten(output_destination): logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) for outputFile in list_media_files(output_destination): @@ -345,83 +329,6 @@ def clean_dir(path, section, subsection): logger.error('Unable to delete directory {0}'.format(path)) -def is_archive_file(filename): - """Check if the filename is allowed for the Archive""" - for regext in core.COMPRESSEDCONTAINER: - if regext.search(filename): - return regext.split(filename)[0] - return False - - -def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): - if otherext is None: - otherext = [] - - file_name, file_ext = os.path.splitext(mediafile) - - try: - # ignore MAC OS's 'resource fork' files - if file_name.startswith('._'): - return False - except Exception: - pass - if (media and file_ext.lower() in core.MEDIACONTAINER) \ - or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ - or (meta and file_ext.lower() in core.METACONTAINER) \ - or (archives and is_archive_file(mediafile)) \ - or (other and (file_ext.lower() in otherext or 'all' in otherext)): - return True - else: - return False - - -def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): - if otherext is None: - otherext = [] - - files = [] - if not os.path.isdir(path): - if os.path.isfile(path): # Single file downloads. - cur_file = os.path.split(path)[1] - if is_media_file(cur_file, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(path) or not is_min_size(path, min_size): - if delete_ignored == 1: - try: - os.unlink(path) - logger.debug('Ignored file {0} has been removed ...'.format - (cur_file)) - except Exception: - pass - else: - files.append(path) - - return files - - for cur_file in os.listdir(text_type(path)): - full_cur_file = os.path.join(path, cur_file) - - # if it's a folder do it recursively - if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): - files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) - - elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): - if delete_ignored == 1: - try: - os.unlink(full_cur_file) - logger.debug('Ignored file {0} has been removed ...'.format - (cur_file)) - except Exception: - pass - continue - - files.append(full_cur_file) - - return sorted(files, key=len) - - def find_imdbid(dir_name, input_name, omdb_api_key): imdbid = None diff --git a/core/utils/files.py b/core/utils/files.py new file mode 100644 index 00000000..1df3114b --- /dev/null +++ b/core/utils/files.py @@ -0,0 +1,102 @@ +import os + +from six import text_type + +import core +from core import logger +from core.utils.naming import is_sample +from core.utils.paths import get_dir_size + + +def is_min_size(input_name, min_size): + file_name, file_ext = os.path.splitext(os.path.basename(input_name)) + + # audio files we need to check directory size not file size + input_size = os.path.getsize(input_name) + if file_ext in core.AUDIOCONTAINER: + try: + input_size = get_dir_size(os.path.dirname(input_name)) + except Exception: + logger.error('Failed to get file size for {0}'.format(input_name), 'MINSIZE') + return True + + # Ignore files under a certain size + if input_size > min_size * 1048576: + return True + + +def is_archive_file(filename): + """Check if the filename is allowed for the Archive""" + for regext in core.COMPRESSEDCONTAINER: + if regext.search(filename): + return regext.split(filename)[0] + return False + + +def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + file_name, file_ext = os.path.splitext(mediafile) + + try: + # ignore MAC OS's 'resource fork' files + if file_name.startswith('._'): + return False + except Exception: + pass + if (media and file_ext.lower() in core.MEDIACONTAINER) \ + or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ + or (meta and file_ext.lower() in core.METACONTAINER) \ + or (archives and is_archive_file(mediafile)) \ + or (other and (file_ext.lower() in otherext or 'all' in otherext)): + return True + else: + return False + + +def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + files = [] + if not os.path.isdir(path): + if os.path.isfile(path): # Single file downloads. + cur_file = os.path.split(path)[1] + if is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(path) or not is_min_size(path, min_size): + if delete_ignored == 1: + try: + os.unlink(path) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + else: + files.append(path) + + return files + + for cur_file in os.listdir(text_type(path)): + full_cur_file = os.path.join(path, cur_file) + + # if it's a folder do it recursively + if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): + files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) + + elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): + if delete_ignored == 1: + try: + os.unlink(full_cur_file) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + continue + + files.append(full_cur_file) + + return sorted(files, key=len) From 4424e217863eb5a419197f43833146949b5d6909 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:03:27 -0500 Subject: [PATCH 25/38] Streamline is_media_file --- core/utils/files.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/utils/files.py b/core/utils/files.py index 1df3114b..ee9c246c 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -45,14 +45,14 @@ def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, o return False except Exception: pass - if (media and file_ext.lower() in core.MEDIACONTAINER) \ - or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ - or (meta and file_ext.lower() in core.METACONTAINER) \ - or (archives and is_archive_file(mediafile)) \ - or (other and (file_ext.lower() in otherext or 'all' in otherext)): - return True - else: - return False + + return any([ + (media and file_ext.lower() in core.MEDIACONTAINER), + (audio and file_ext.lower() in core.AUDIOCONTAINER), + (meta and file_ext.lower() in core.METACONTAINER), + (archives and is_archive_file(mediafile)), + (other and (file_ext.lower() in otherext or 'all' in otherext)), + ]) def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): From 0f7c74dd78f8035fce80f082e27114b354e29a33 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:22:42 -0500 Subject: [PATCH 26/38] Refactor file type detection to utils.files --- core/utils/__init__.py | 78 +++++------------------------------------- core/utils/files.py | 72 +++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index cdfb9418..c3e95150 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -6,7 +6,6 @@ import os import re import shutil import stat -import time import beets import guessit @@ -14,11 +13,18 @@ import requests from six import text_type import core -from core import extractor, logger +from core import logger from core.utils import shutil_custom from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii -from core.utils.files import is_archive_file, is_media_file, is_min_size, list_media_files +from core.utils.files import ( + backup_versioned_file, + extract_files, + is_archive_file, + is_media_file, + is_min_size, + list_media_files, +) from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -406,43 +412,6 @@ def find_imdbid(dir_name, input_name, omdb_api_key): return imdbid -def extract_files(src, dst=None, keep_archive=None): - extracted_folder = [] - extracted_archive = [] - - for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): - dir_path = os.path.dirname(inputFile) - full_file_name = os.path.basename(inputFile) - archive_name = os.path.splitext(full_file_name)[0] - archive_name = re.sub(r'part[0-9]+', '', archive_name) - - if dir_path in extracted_folder and archive_name in extracted_archive: - continue # no need to extract this, but keep going to look for other archives and sub directories. - - try: - if extractor.extract(inputFile, dst or dir_path): - extracted_folder.append(dir_path) - extracted_archive.append(archive_name) - except Exception: - logger.error('Extraction failed for: {0}'.format(full_file_name)) - - for folder in extracted_folder: - for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): - full_file_name = os.path.basename(inputFile) - archive_name = os.path.splitext(full_file_name)[0] - archive_name = re.sub(r'part[0-9]+', '', archive_name) - if archive_name not in extracted_archive or keep_archive: - continue # don't remove if we haven't extracted this archive, or if we want to preserve them. - logger.info('Removing extracted archive {0} from folder {1} ...'.format(full_file_name, folder)) - try: - if not os.access(inputFile, os.W_OK): - os.chmod(inputFile, stat.S_IWUSR) - os.remove(inputFile) - time.sleep(1) - except Exception as e: - logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) - - def server_responding(base_url): logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') try: @@ -452,32 +421,3 @@ def server_responding(base_url): except (requests.ConnectionError, requests.exceptions.Timeout): logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') return False - - -def backup_versioned_file(old_file, version): - num_tries = 0 - - new_file = '{old}.v{version}'.format(old=old_file, version=version) - - while not os.path.isfile(new_file): - if not os.path.isfile(old_file): - logger.log(u'Not creating backup, {file} doesn\'t exist'.format(file=old_file), logger.DEBUG) - break - - try: - logger.log(u'Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) - shutil.copy(old_file, new_file) - logger.log(u'Backup done', logger.DEBUG) - break - except Exception as error: - logger.log(u'Error while trying to back up {old} to {new} : {msg}'.format - (old=old_file, new=new_file, msg=error), logger.WARNING) - num_tries += 1 - time.sleep(1) - logger.log(u'Trying again.', logger.DEBUG) - - if num_tries >= 10: - logger.log(u'Unable to back up {old} to {new} please do it manually.'.format(old=old_file, new=new_file), logger.ERROR) - return False - - return True diff --git a/core/utils/files.py b/core/utils/files.py index ee9c246c..8800d239 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,9 +1,13 @@ +import shutil import os +import re +import stat +import time from six import text_type import core -from core import logger +from core import extractor, logger from core.utils.naming import is_sample from core.utils.paths import get_dir_size @@ -100,3 +104,69 @@ def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, files.append(full_cur_file) return sorted(files, key=len) + + +def extract_files(src, dst=None, keep_archive=None): + extracted_folder = [] + extracted_archive = [] + + for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): + dir_path = os.path.dirname(inputFile) + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + + if dir_path in extracted_folder and archive_name in extracted_archive: + continue # no need to extract this, but keep going to look for other archives and sub directories. + + try: + if extractor.extract(inputFile, dst or dir_path): + extracted_folder.append(dir_path) + extracted_archive.append(archive_name) + except Exception: + logger.error('Extraction failed for: {0}'.format(full_file_name)) + + for folder in extracted_folder: + for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + if archive_name not in extracted_archive or keep_archive: + continue # don't remove if we haven't extracted this archive, or if we want to preserve them. + logger.info('Removing extracted archive {0} from folder {1} ...'.format(full_file_name, folder)) + try: + if not os.access(inputFile, os.W_OK): + os.chmod(inputFile, stat.S_IWUSR) + os.remove(inputFile) + time.sleep(1) + except Exception as e: + logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) + + +def backup_versioned_file(old_file, version): + num_tries = 0 + + new_file = '{old}.v{version}'.format(old=old_file, version=version) + + while not os.path.isfile(new_file): + if not os.path.isfile(old_file): + logger.log(u'Not creating backup, {file} doesn\'t exist'.format(file=old_file), logger.DEBUG) + break + + try: + logger.log(u'Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) + shutil.copy(old_file, new_file) + logger.log(u'Backup done', logger.DEBUG) + break + except Exception as error: + logger.log(u'Error while trying to back up {old} to {new} : {msg}'.format + (old=old_file, new=new_file, msg=error), logger.WARNING) + num_tries += 1 + time.sleep(1) + logger.log(u'Trying again.', logger.DEBUG) + + if num_tries >= 10: + logger.log(u'Unable to back up {old} to {new} please do it manually.'.format(old=old_file, new=new_file), logger.ERROR) + return False + + return True From dade3f669804d3899a6e3c9fb0973c1ad352009b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:26:33 -0500 Subject: [PATCH 27/38] Refactor network utils to utils.network --- core/utils/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index c3e95150..2ef8d8e0 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -410,14 +410,3 @@ def find_imdbid(dir_name, input_name, omdb_api_key): logger.warning('Unable to find a imdbID for {0}'.format(input_name)) return imdbid - - -def server_responding(base_url): - logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') - try: - requests.get(base_url, timeout=(60, 120), verify=False) - logger.debug('Server responded at {0}'.format(base_url), 'SERVER') - return True - except (requests.ConnectionError, requests.exceptions.Timeout): - logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') - return False From 6cc3df73b32f62cdfd0bf0e83a4eaab09797948e Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:32:52 -0500 Subject: [PATCH 28/38] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 50 +++++------------------------------------- core/utils/paths.py | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2ef8d8e0..bea35b3a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -41,7 +41,11 @@ from core.utils.parsers import ( parse_vuze, ) from core.utils.paths import ( - get_dir_size, make_dir, + flatten_dir, + get_dir_size, + make_dir, + onerror, + remove_dir, remote_dir, remove_empty_folders, remove_read_only, @@ -156,22 +160,7 @@ def category_search(input_directory, input_name, input_category, root, categorie def flatten(output_destination): - logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) - for outputFile in list_media_files(output_destination): - dir_path = os.path.dirname(outputFile) - file_name = os.path.basename(outputFile) - - if dir_path == output_destination: - continue - - target = os.path.join(output_destination, file_name) - - try: - shutil.move(outputFile, target) - except Exception: - logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') - - remove_empty_folders(output_destination) # Cleanup empty directories + return flatten_dir(output_destination, list_media_files(output_destination)) def get_dirs(section, subsection, link='hard'): @@ -280,33 +269,6 @@ def get_dirs(section, subsection, link='hard'): return list(set(to_return)) -def onerror(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise Exception - - -def remove_dir(dir_name): - logger.info('Deleting {0}'.format(dir_name)) - try: - shutil.rmtree(text_type(dir_name), onerror=onerror) - except Exception: - logger.error('Unable to delete folder {0}'.format(dir_name)) - - def clean_dir(path, section, subsection): cfg = dict(core.CFG[section][subsection]) if not os.path.exists(path): diff --git a/core/utils/paths.py b/core/utils/paths.py index 31308650..948287b7 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -2,6 +2,7 @@ from functools import partial import os import re +import shutil import stat from six import text_type @@ -10,6 +11,33 @@ import core from core import logger +def onerror(func, path, exc_info): + """ + Error handler for ``shutil.rmtree``. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Usage : ``shutil.rmtree(path, onerror=onerror)`` + """ + if not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise Exception + + +def remove_dir(dir_name): + logger.info('Deleting {0}'.format(dir_name)) + try: + shutil.rmtree(text_type(dir_name), onerror=onerror) + except Exception: + logger.error('Unable to delete folder {0}'.format(dir_name)) + + def make_dir(path): if not os.path.isdir(path): try: @@ -78,3 +106,22 @@ def remove_read_only(filename): os.chmod(filename, stat.S_IWRITE) except Exception: logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) + + +def flatten_dir(destination, files): + logger.info('FLATTEN: Flattening directory: {0}'.format(destination)) + for outputFile in files: + dir_path = os.path.dirname(outputFile) + file_name = os.path.basename(outputFile) + + if dir_path == destination: + continue + + target = os.path.join(destination, file_name) + + try: + shutil.move(outputFile, target) + except Exception: + logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') + + remove_empty_folders(destination) # Cleanup empty directories From 36932e25c6954a84afbcd053da6bd2775aa06cda Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:36:52 -0500 Subject: [PATCH 29/38] Fix clean_dir for Python 3 TypeError when testing str > int --- core/utils/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index bea35b3a..8ab397b7 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -283,7 +283,8 @@ def clean_dir(path, section, subsection): try: num_files = len(list_media_files(path, min_size=min_size, delete_ignored=delete_ignored)) except Exception: - num_files = 'unknown' + num_files = 0 + if num_files > 0: logger.info( 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, num_files), From e44c0bb56ab0564c328d4ac074d308272afcd49d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:40:05 -0500 Subject: [PATCH 30/38] Refactor path functions from utils to utils.paths --- core/utils/__init__.py | 45 ++++++++++++------------------------------ core/utils/paths.py | 24 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 8ab397b7..2dcaaa9a 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,8 +4,6 @@ from __future__ import print_function, unicode_literals import os import re -import shutil -import stat import beets import guessit @@ -41,12 +39,13 @@ from core.utils.parsers import ( parse_vuze, ) from core.utils.paths import ( + clean_directory, flatten_dir, get_dir_size, make_dir, onerror, - remove_dir, remote_dir, + remove_dir, remove_empty_folders, remove_read_only, ) @@ -163,6 +162,17 @@ def flatten(output_destination): return flatten_dir(output_destination, list_media_files(output_destination)) +def clean_dir(path, section, subsection): + cfg = dict(core.CFG[section][subsection]) + min_size = int(cfg.get('minSize', 0)) + delete_ignored = int(cfg.get('delete_ignored', 0)) + try: + files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) + except Exception: + files = [] + return clean_directory(path, files) + + def get_dirs(section, subsection, link='hard'): to_return = [] @@ -269,35 +279,6 @@ def get_dirs(section, subsection, link='hard'): return list(set(to_return)) -def clean_dir(path, section, subsection): - cfg = dict(core.CFG[section][subsection]) - if not os.path.exists(path): - logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') - return - if core.FORCE_CLEAN and not core.FAILED: - logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') - remove_dir(path) - return - min_size = int(cfg.get('minSize', 0)) - delete_ignored = int(cfg.get('delete_ignored', 0)) - try: - num_files = len(list_media_files(path, min_size=min_size, delete_ignored=delete_ignored)) - except Exception: - num_files = 0 - - if num_files > 0: - logger.info( - 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, num_files), - 'CLEANDIRS') - return - - logger.info('Directory {0} has been processed, removing ...'.format(path), 'CLEANDIRS') - try: - shutil.rmtree(path, onerror=onerror) - except Exception: - logger.error('Unable to delete directory {0}'.format(path)) - - def find_imdbid(dir_name, input_name, omdb_api_key): imdbid = None diff --git a/core/utils/paths.py b/core/utils/paths.py index 948287b7..5576206f 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -125,3 +125,27 @@ def flatten_dir(destination, files): logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') remove_empty_folders(destination) # Cleanup empty directories + + +def clean_directory(path, files): + if not os.path.exists(path): + logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') + return + + if core.FORCE_CLEAN and not core.FAILED: + logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') + remove_dir(path) + return + + if files: + logger.info( + 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, len(files)), + 'CLEANDIRS', + ) + return + + logger.info('Directory {0} has been processed, removing ...'.format(path), 'CLEANDIRS') + try: + shutil.rmtree(path, onerror=onerror) + except Exception: + logger.error('Unable to delete directory {0}'.format(path)) From 03cb11dae3107ac3a16cf518707c9c6cdd54a223 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:52:22 -0500 Subject: [PATCH 31/38] Refactor identification utils from utils to utils.identification --- core/utils/__init__.py | 173 +-------------------------------- core/utils/identification.py | 181 +++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 172 deletions(-) create mode 100644 core/utils/identification.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2dcaaa9a..2d5954e6 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import os -import re import beets import guessit @@ -23,6 +22,7 @@ from core.utils.files import ( is_min_size, list_media_files, ) +from core.utils.identification import find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up @@ -64,100 +64,6 @@ requests.packages.urllib3.disable_warnings() shutil_custom.monkey_patch() -def category_search(input_directory, input_name, input_category, root, categories): - tordir = False - - try: - input_name = input_name.encode(core.SYS_ENCODING) - except Exception: - pass - try: - input_directory = input_directory.encode(core.SYS_ENCODING) - except Exception: - pass - - if input_directory is None: # =Nothing to process here. - return input_directory, input_name, input_category, root - - pathlist = os.path.normpath(input_directory).split(os.sep) - - if input_category and input_category in pathlist: - logger.debug('SEARCH: Found the Category: {0} in directory structure'.format(input_category)) - elif input_category: - logger.debug('SEARCH: Could not find the category: {0} in the directory structure'.format(input_category)) - else: - try: - input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. - logger.debug('SEARCH: Found Category: {0} in directory structure'.format(input_category)) - except IndexError: - input_category = '' - logger.debug('SEARCH: Could not find a category in the directory structure') - if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file - if not input_name: - input_name = os.path.split(os.path.normpath(input_directory))[1] - return input_directory, input_name, input_category, root - - if input_category and os.path.isdir(os.path.join(input_directory, input_category)): - logger.info( - 'SEARCH: Found category directory {0} in input directory directory {1}'.format(input_category, input_directory)) - input_directory = os.path.join(input_directory, input_category) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - if input_name and os.path.isdir(os.path.join(input_directory, input_name)): - logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format(input_name, input_directory)) - input_directory = os.path.join(input_directory, input_name) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): - logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format( - sanitize_name(input_name), input_directory)) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): - logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(input_name, input_directory)) - input_directory = os.path.join(input_directory, input_name) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): - logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format( - sanitize_name(input_name), input_directory)) - input_directory = os.path.join(input_directory, sanitize_name(input_name)) - logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) - tordir = True - - imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. - if imdbid and '.cp(tt' not in input_name: - input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP - tordir = True - - if input_category and not tordir: - try: - index = pathlist.index(input_category) - if index + 1 < len(pathlist): - tordir = True - logger.info('SEARCH: Found a unique directory {0} in the category directory'.format - (pathlist[index + 1])) - if not input_name: - input_name = pathlist[index + 1] - except ValueError: - pass - - if input_name and not tordir: - if input_name in pathlist or sanitize_name(input_name) in pathlist: - logger.info('SEARCH: Found torrent directory {0} in the directory structure'.format(input_name)) - tordir = True - else: - root = 1 - if not tordir: - root = 2 - - if root > 0: - logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') - logger.info('SEARCH: We will try and determine which files to process, individually') - - return input_directory, input_name, input_category, root - - def flatten(output_destination): return flatten_dir(output_destination, list_media_files(output_destination)) @@ -277,80 +183,3 @@ def get_dirs(section, subsection, link='hard'): logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) return list(set(to_return)) - - -def find_imdbid(dir_name, input_name, omdb_api_key): - imdbid = None - - logger.info('Attemping imdbID lookup for {0}'.format(input_name)) - - # find imdbid in dirName - logger.info('Searching folder and file names for imdbID ...') - m = re.search(r'(tt\d{7})', dir_name + input_name) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}]'.format(imdbid)) - return imdbid - if os.path.isdir(dir_name): - for file in os.listdir(text_type(dir_name)): - m = re.search(r'(tt\d{7})', file) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}] via file name'.format(imdbid)) - return imdbid - if 'NZBPR__DNZB_MOREINFO' in os.environ: - dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') - if dnzb_more_info != '': - regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) - m = regex.match(dnzb_more_info) - if m: - imdbid = m.group(1) - logger.info('Found imdbID [{0}] from DNZB-MoreInfo'.format(imdbid)) - return imdbid - logger.info('Searching IMDB for imdbID ...') - try: - guess = guessit.guessit(input_name) - except Exception: - guess = None - if guess: - # Movie Title - title = None - if 'title' in guess: - title = guess['title'] - - # Movie Year - year = None - if 'year' in guess: - year = guess['year'] - - url = 'http://www.omdbapi.com' - - if not omdb_api_key: - logger.info('Unable to determine imdbID: No api key provided for ombdapi.com.') - return - - logger.debug('Opening URL: {0}'.format(url)) - - try: - r = requests.get(url, params={'apikey': omdb_api_key, 'y': year, 't': title}, - verify=False, timeout=(60, 300)) - except requests.ConnectionError: - logger.error('Unable to open URL {0}'.format(url)) - return - - try: - results = r.json() - except Exception: - logger.error('No json data returned from omdbapi.com') - - try: - imdbid = results['imdbID'] - except Exception: - logger.error('No imdbID returned from omdbapi.com') - - if imdbid: - logger.info('Found imdbID [{0}]'.format(imdbid)) - return imdbid - - logger.warning('Unable to find a imdbID for {0}'.format(input_name)) - return imdbid diff --git a/core/utils/identification.py b/core/utils/identification.py new file mode 100644 index 00000000..c2dc0352 --- /dev/null +++ b/core/utils/identification.py @@ -0,0 +1,181 @@ +import os +import re + +import guessit +import requests +from six import text_type + +import core +from core import logger +from core.utils.naming import sanitize_name + + +def find_imdbid(dir_name, input_name, omdb_api_key): + imdbid = None + + logger.info('Attemping imdbID lookup for {0}'.format(input_name)) + + # find imdbid in dirName + logger.info('Searching folder and file names for imdbID ...') + m = re.search(r'(tt\d{7})', dir_name + input_name) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + if os.path.isdir(dir_name): + for file in os.listdir(text_type(dir_name)): + m = re.search(r'(tt\d{7})', file) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] via file name'.format(imdbid)) + return imdbid + if 'NZBPR__DNZB_MOREINFO' in os.environ: + dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') + if dnzb_more_info != '': + regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) + m = regex.match(dnzb_more_info) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] from DNZB-MoreInfo'.format(imdbid)) + return imdbid + logger.info('Searching IMDB for imdbID ...') + try: + guess = guessit.guessit(input_name) + except Exception: + guess = None + if guess: + # Movie Title + title = None + if 'title' in guess: + title = guess['title'] + + # Movie Year + year = None + if 'year' in guess: + year = guess['year'] + + url = 'http://www.omdbapi.com' + + if not omdb_api_key: + logger.info('Unable to determine imdbID: No api key provided for ombdapi.com.') + return + + logger.debug('Opening URL: {0}'.format(url)) + + try: + r = requests.get(url, params={'apikey': omdb_api_key, 'y': year, 't': title}, + verify=False, timeout=(60, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url)) + return + + try: + results = r.json() + except Exception: + logger.error('No json data returned from omdbapi.com') + + try: + imdbid = results['imdbID'] + except Exception: + logger.error('No imdbID returned from omdbapi.com') + + if imdbid: + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + + logger.warning('Unable to find a imdbID for {0}'.format(input_name)) + return imdbid + + +def category_search(input_directory, input_name, input_category, root, categories): + tordir = False + + try: + input_name = input_name.encode(core.SYS_ENCODING) + except Exception: + pass + try: + input_directory = input_directory.encode(core.SYS_ENCODING) + except Exception: + pass + + if input_directory is None: # =Nothing to process here. + return input_directory, input_name, input_category, root + + pathlist = os.path.normpath(input_directory).split(os.sep) + + if input_category and input_category in pathlist: + logger.debug('SEARCH: Found the Category: {0} in directory structure'.format(input_category)) + elif input_category: + logger.debug('SEARCH: Could not find the category: {0} in the directory structure'.format(input_category)) + else: + try: + input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. + logger.debug('SEARCH: Found Category: {0} in directory structure'.format(input_category)) + except IndexError: + input_category = '' + logger.debug('SEARCH: Could not find a category in the directory structure') + if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file + if not input_name: + input_name = os.path.split(os.path.normpath(input_directory))[1] + return input_directory, input_name, input_category, root + + if input_category and os.path.isdir(os.path.join(input_directory, input_category)): + logger.info( + 'SEARCH: Found category directory {0} in input directory directory {1}'.format(input_category, input_directory)) + input_directory = os.path.join(input_directory, input_category) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + if input_name and os.path.isdir(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + + imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. + if imdbid and '.cp(tt' not in input_name: + input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP + tordir = True + + if input_category and not tordir: + try: + index = pathlist.index(input_category) + if index + 1 < len(pathlist): + tordir = True + logger.info('SEARCH: Found a unique directory {0} in the category directory'.format + (pathlist[index + 1])) + if not input_name: + input_name = pathlist[index + 1] + except ValueError: + pass + + if input_name and not tordir: + if input_name in pathlist or sanitize_name(input_name) in pathlist: + logger.info('SEARCH: Found torrent directory {0} in the directory structure'.format(input_name)) + tordir = True + else: + root = 1 + if not tordir: + root = 2 + + if root > 0: + logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') + logger.info('SEARCH: We will try and determine which files to process, individually') + + return input_directory, input_name, input_category, root From e67f29cb7bc4d3237ec407edfa35671583de21ef Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 10:58:02 -0500 Subject: [PATCH 32/38] Flatten get_dirs function --- core/utils/__init__.py | 165 +++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 2d5954e6..724c85fe 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -79,94 +79,95 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) +def process_dir(path, link): + folders = [] + + logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) + sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + # search for single files and move them into their own folder for post-processing + for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isfile(os.path.join(path, o))]: + if len(sync) > 0: + break + if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: + continue + try: + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # removeEmptyFolders(path, removeRoot=False) + + if os.listdir(text_type(path)): + for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isdir(os.path.join(path, o))]: + sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: + continue + folders.extend([directory]) + return folders + + def get_dirs(section, subsection, link='hard'): to_return = [] - def process_dir(path): - folders = [] - - logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - # search for single files and move them into their own folder for post-processing - for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isfile(os.path.join(path, o))]: - if len(sync) > 0: - break - if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: - continue - try: - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) - - # removeEmptyFolders(path, removeRoot=False) - - if os.listdir(text_type(path)): - for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isdir(os.path.join(path, o))]: - sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: - continue - folders.extend([directory]) - return folders - try: watch_dir = os.path.join(core.CFG[section][subsection]['watch_dir'], subsection) if os.path.exists(watch_dir): - to_return.extend(process_dir(watch_dir)) + to_return.extend(process_dir(watch_dir, link)) elif os.path.exists(core.CFG[section][subsection]['watch_dir']): - to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'])) + to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'], link)) except Exception as e: logger.error('Failed to add directories from {0} for post-processing: {1}'.format (core.CFG[section][subsection]['watch_dir'], e)) @@ -175,7 +176,7 @@ def get_dirs(section, subsection, link='hard'): try: output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) if os.path.exists(output_directory): - to_return.extend(process_dir(output_directory)) + to_return.extend(process_dir(output_directory, link)) except Exception as e: logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) From 8d458f10ac370c334a2473632af8ff5617d79b48 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:02:54 -0500 Subject: [PATCH 33/38] Refactor get_dirs --- core/utils/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 724c85fe..f0c822cc 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -162,15 +162,16 @@ def process_dir(path, link): def get_dirs(section, subsection, link='hard'): to_return = [] + watch_directory = core.CFG[section][subsection]['watch_dir'] + directory = os.path.join(watch_directory, subsection) + + if not os.path.exists(directory): + directory = watch_directory + try: - watch_dir = os.path.join(core.CFG[section][subsection]['watch_dir'], subsection) - if os.path.exists(watch_dir): - to_return.extend(process_dir(watch_dir, link)) - elif os.path.exists(core.CFG[section][subsection]['watch_dir']): - to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'], link)) + to_return.extend(process_dir(directory, link)) except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format - (core.CFG[section][subsection]['watch_dir'], e)) + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) if core.USELINK == 'move': try: From cb422a0cea75a90fbcbf6089510cb9c546cb8d2f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:09:11 -0500 Subject: [PATCH 34/38] Flatten process_dir --- core/utils/__init__.py | 110 +++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index f0c822cc..665453fe 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,7 +4,7 @@ from __future__ import print_function, unicode_literals import os -import beets +import beets.mediafile import guessit import requests from six import text_type @@ -79,6 +79,61 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) +def move_file(mediafile, path, link): + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) + + def process_dir(path, link): folders = [] @@ -92,58 +147,7 @@ def process_dir(path, link): if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: continue try: - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) + move_file(mediafile, path, link) except Exception as e: logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) From a888d741d3f11ef66cedcf4d4eb9eecd65acb4e0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:12:17 -0500 Subject: [PATCH 35/38] Refactor file type detection to utils.files --- core/utils/__init__.py | 58 +------------------------------------- core/utils/files.py | 64 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 60 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 665453fe..fba6002d 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,8 +4,6 @@ from __future__ import print_function, unicode_literals import os -import beets.mediafile -import guessit import requests from six import text_type @@ -21,6 +19,7 @@ from core.utils.files import ( is_media_file, is_min_size, list_media_files, + move_file, ) from core.utils.identification import find_imdbid from core.utils.links import copy_link, replace_links @@ -79,61 +78,6 @@ def clean_dir(path, section, subsection): return clean_directory(path, files) -def move_file(mediafile, path, link): - logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) - new_path = None - file_ext = os.path.splitext(mediafile)[1] - try: - if file_ext in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) - elif file_ext in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - new_path = os.path.join(path, sanitize_name(title)) - except Exception as e: - logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) - - if not new_path: - title = os.path.splitext(os.path.basename(mediafile))[0] - new_path = os.path.join(path, sanitize_name(title)) - - try: - new_path = new_path.encode(core.SYS_ENCODING) - except Exception: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(new_path): - new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) - new_path = new_path2 - - # create new path if it does not exist - if not os.path.exists(new_path): - make_dir(new_path) - - newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except Exception: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) - - def process_dir(path, link): folders = [] diff --git a/core/utils/files.py b/core/utils/files.py index 8800d239..b753af0c 100644 --- a/core/utils/files.py +++ b/core/utils/files.py @@ -1,15 +1,73 @@ -import shutil import os import re +import shutil import stat import time +import beets.mediafile +import guessit from six import text_type import core from core import extractor, logger -from core.utils.naming import is_sample -from core.utils.paths import get_dir_size +from core.utils.links import copy_link +from core.utils.naming import is_sample, sanitize_name +from core.utils.paths import get_dir_size, make_dir + + +def move_file(mediafile, path, link): + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) def is_min_size(input_name, min_size): From 648ecd40487c6ebcc24f68cde374e34b807183d5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 11:36:32 -0500 Subject: [PATCH 36/38] Refactor process_dir to use generators --- core/utils/__init__.py | 69 +++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fba6002d..fb1ca12c 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -82,28 +82,61 @@ def process_dir(path, link): folders = [] logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + dir_contents = os.listdir(text_type(path)) + # search for single files and move them into their own folder for post-processing - for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isfile(os.path.join(path, o))]: - if len(sync) > 0: - break - if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: - continue - try: - move_file(mediafile, path, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # Generate list of sync files + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + + # Generate a list of file paths + filepaths = ( + os.path.join(path, item) for item in dir_contents + if item not in ['Thumbs.db', 'thumbs.db'] + ) + + # Generate a list of media files + mediafiles = ( + item for item in filepaths + if os.path.isfile(item) + ) + + if any(sync_files): + logger.info('') + else: + for mediafile in mediafiles: + try: + move_file(mediafile, path, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) # removeEmptyFolders(path, removeRoot=False) - if os.listdir(text_type(path)): - for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if - os.path.isdir(os.path.join(path, o))]: - sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: - continue - folders.extend([directory]) + # Generate all path contents + path_contents = ( + os.path.join(path, item) + for item in os.listdir(text_type(path)) + ) + + # Generate all directories from path contents + directories = ( + path for path in path_contents + if os.path.isdir(path) + ) + + for directory in directories: + dir_contents = os.listdir(directory) + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + if not any(dir_contents) or any(sync_files): + continue + folders.append(directory) + return folders From 383eb5eaf2142d54d50393e9d4b8b2358fd1c893 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 12:04:13 -0500 Subject: [PATCH 37/38] Refactor identification utils from utils to utils.identification --- core/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index fb1ca12c..a9c4e6cf 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -21,7 +21,7 @@ from core.utils.files import ( list_media_files, move_file, ) -from core.utils.identification import find_imdbid +from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name from core.utils.network import find_download, test_connection, wake_on_lan, wake_up From bd5b970bc7350fd7d5d62b8ace95357c88bc21b0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Sun, 6 Jan 2019 12:05:26 -0500 Subject: [PATCH 38/38] Refactor network utils to utils.network --- core/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index a9c4e6cf..176c92db 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -24,7 +24,7 @@ from core.utils.files import ( from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name -from core.utils.network import find_download, test_connection, wake_on_lan, wake_up +from core.utils.network import find_download, test_connection, wake_on_lan, wake_up, server_responding from core.utils.notifications import plex_update from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import (