From 0053fdcf3ea6c302bf70f623091489652ed20b31 Mon Sep 17 00:00:00 2001 From: echel0n Date: Sun, 20 Apr 2014 16:28:27 -0700 Subject: [PATCH] Added cleanup of processing folders to start and end of scripts, helps resolve issues where empty folders or misc files left behind cause processing scripts to execute without need. Modified our listMediaFiles function to allow include/exclude of extension containers so that we can chose what extensions are used to create the results list that gets returned back to us. --- TorrentToMedia.py | 12 +++---- nzbToMedia.py | 10 +++--- nzbtomedia/__init__.py | 8 ++--- nzbtomedia/nzbToMediaUtil.py | 67 +++++++++++++++++------------------- 4 files changed, 45 insertions(+), 52 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index f1d4b33e..f217e782 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -2,23 +2,20 @@ import datetime import os import time -import re import shutil import sys import nzbtomedia import platform from subprocess import Popen -from nzbtomedia.Transcoder import Transcoder from nzbtomedia.autoProcess.autoProcessComics import autoProcessComics from nzbtomedia.autoProcess.autoProcessGames import autoProcessGames from nzbtomedia.autoProcess.autoProcessMovie import autoProcessMovie from nzbtomedia.autoProcess.autoProcessMusic import autoProcessMusic from nzbtomedia.autoProcess.autoProcessTV import autoProcessTV -from nzbtomedia.extractor import extractor from nzbtomedia.nzbToMediaUtil import category_search, sanitizeFileName, copy_link, parse_args, flatten, get_dirnames, \ - remove_read_only, cleanup_directories, create_torrent_class, pause_torrent, resume_torrent, listMediaFiles, joinPath, \ - extractFiles + remove_read_only, create_torrent_class, pause_torrent, resume_torrent, listMediaFiles, joinPath, \ + extractFiles, cleanProcDirs from nzbtomedia import logger def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent): @@ -67,7 +64,6 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, if clientAgent != 'manual': resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) - cleanup_directories(inputCategory, processCategories, result, outputDestination) return result processOnly = nzbtomedia.CFG[nzbtomedia.SECTIONS].sections @@ -193,7 +189,6 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, logger.error("A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding") resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) - cleanup_directories(inputCategory, processCategories, result, outputDestination) return result def external_script(outputDestination, torrentName, torrentLabel): @@ -314,6 +309,9 @@ def main(args): logger.warning("%s:%s is DISABLED, you can enable this in autoProcessMedia.cfg ..." % (section, category)) if result == 0: + # cleanup our processing folders of any misc unwanted files and empty directories + cleanProcDirs() + logger.info("The %s script completed successfully." % (args[0])) else: logger.error("A problem was reported in the %s script." % (args[0])) diff --git a/nzbToMedia.py b/nzbToMedia.py index 1115e1eb..90bd7758 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -276,13 +276,12 @@ import os import sys import nzbtomedia -from nzbtomedia.Transcoder import Transcoder from nzbtomedia.autoProcess.autoProcessComics import autoProcessComics from nzbtomedia.autoProcess.autoProcessGames import autoProcessGames from nzbtomedia.autoProcess.autoProcessMovie import autoProcessMovie from nzbtomedia.autoProcess.autoProcessMusic import autoProcessMusic from nzbtomedia.autoProcess.autoProcessTV import autoProcessTV -from nzbtomedia.nzbToMediaUtil import get_dirnames, cleanup_directories, listMediaFiles, extractFiles +from nzbtomedia.nzbToMediaUtil import get_dirnames, extractFiles, cleanProcDirs from nzbtomedia import logger # post-processing @@ -312,10 +311,6 @@ def process(nzbDir, inputName=None, status=0, clientAgent='manual', download_id= logger.error("We could not find a section with containing a download category labeled %s in your autoProcessMedia.cfg, Exiting!" % inputCategory) result = -1 - if result == 0: - # Clean up any leftover files - cleanup_directories(inputCategory, section, result, nzbDir) - return result def main(args, section=None): @@ -432,6 +427,9 @@ def main(args, section=None): logger.debug("nzbToMedia %s:%s is DISABLED" % (section, category)) if result == 0: + # cleanup our processing folders of any misc unwanted files and empty directories + cleanProcDirs() + logger.info("The %s script completed successfully." % args[0]) if os.environ.has_key('NZBOP_SCRIPTDIR'): # return code for nzbget v11 sys.exit(nzbtomedia.NZBGET_POSTPROCESS_SUCCESS) diff --git a/nzbtomedia/__init__.py b/nzbtomedia/__init__.py index e2985c4f..cc652598 100644 --- a/nzbtomedia/__init__.py +++ b/nzbtomedia/__init__.py @@ -21,7 +21,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(PROGRAM_DIR, 'lib'))) from nzbtomedia import logger, versionCheck from nzbtomedia.nzbToMediaConfig import config -from nzbtomedia.nzbToMediaUtil import WakeUp, makeDir, joinPath +from nzbtomedia.nzbToMediaUtil import WakeUp, makeDir, joinPath, cleanProcDirs # sabnzbd constants SABNZB_NO_OF_ARGUMENTS = 8 @@ -250,8 +250,6 @@ def initialize(section=None): if isinstance(AUDIOCONTAINER, str): AUDIOCONTAINER = AUDIOCONTAINER.split(',') if isinstance(METACONTAINER, str): METACONTAINER = METACONTAINER.split(',') - EXTCONTAINER = [y for x in COMPRESSEDCONTAINER,MEDIACONTAINER,AUDIOCONTAINER,METACONTAINER for y in x] - MINSAMPLESIZE = int(CFG["Extensions"]["minSampleSize"]) # 200 (in MB) SAMPLEIDS = CFG["Extensions"]["SampleIDs"] @@ -312,7 +310,9 @@ def initialize(section=None): SUBSECTIONS = CFG[SECTIONS] CATEGORIES += SUBSECTIONS.sections - __INITIALIZED__ = True + # cleanup our processing folders of any misc unwanted files and empty directories + cleanProcDirs() + return True def restart(): diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index 00ad915e..9fed03c2 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -209,7 +209,6 @@ def removeEmptyFolders(path): logger.debug("Removing empty folder: %s" % (path), 'REMOVER') os.rmdir(path) - def remove_read_only(path): if not os.path.isdir(path): return @@ -436,27 +435,22 @@ def rmDir(dirName): except: logger.error("Unable to delete folder %s" % (dirName)) -def cleanup_directories(inputCategory, processCategories, result, directory): - if inputCategory in processCategories and result == 0 and os.path.isdir(directory): - num_files_new = int(0) - file_list = [] - for dirpath, dirnames, filenames in os.walk(directory): - for file in filenames: - filePath = joinPath(dirpath, file) - fileName, fileExtension = os.path.splitext(file) - if fileExtension in nzbtomedia.MEDIACONTAINER or fileExtension in nzbtomedia.METACONTAINER: - num_files_new += 1 - file_list.append(file) - if num_files_new is 0 or int(nzbtomedia.CFG["General"]["force_clean"]) == 1: - logger.info("All files have been processed. Cleaning directory %s" % (directory)) - shutil.rmtree(directory) - else: - logger.info( - "Directory %s still contains %s media and/or meta files. This directory will not be removed." % ( - directory, num_files_new)) - for item in file_list: - logger.debug("media/meta file found: %s" % (item)) +def cleanProcDirs(): + logger.info('Cleaning processing directories ...', 'CLEANDIRS') + for section, subsection in nzbtomedia.SUBSECTIONS.items(): + for category in subsection: + if nzbtomedia.CFG[section][category].isenabled(): + dirNames = get_dirnames(section, category) + for dirName in dirNames: + num_files = len(listMediaFiles(dirName, archives=False)) + if num_files > 0: + logger.info( + "Directory %s still contains %s unprocessed file(s), skipping ..." % (dirName, num_files), + 'CLEANDIRS') + continue + logger.info("Directory %s has already been processed, removing ..." % (dirName), 'CLEANDIRS') + shutil.rmtree(dirName) def create_torrent_class(clientAgent): # Hardlink solution for Torrents @@ -564,7 +558,7 @@ def find_download(clientAgent, download_id): return True -def clean_nzbname(nzbname): +def cleanFileName(filename): """Cleans up nzb name by removing any . and _ characters, along with any trailing hyphens. @@ -572,16 +566,16 @@ def clean_nzbname(nzbname): space, but handles decimal numbers in string, for example: """ - nzbname = re.sub("(\D)\.(?!\s)(\D)", "\\1 \\2", nzbname) - nzbname = re.sub("(\d)\.(\d{4})", "\\1 \\2", nzbname) # if it ends in a year then don't keep the dot - nzbname = re.sub("(\D)\.(?!\s)", "\\1 ", nzbname) - nzbname = re.sub("\.(?!\s)(\D)", " \\1", nzbname) - nzbname = nzbname.replace("_", " ") - nzbname = re.sub("-$", "", nzbname) - nzbname = re.sub("^\[.*\]", "", nzbname) - return nzbname.strip() + filename = re.sub("(\D)\.(?!\s)(\D)", "\\1 \\2", filename) + filename = re.sub("(\d)\.(\d{4})", "\\1 \\2", filename) # if it ends in a year then don't keep the dot + filename = re.sub("(\D)\.(?!\s)", "\\1 ", filename) + filename = re.sub("\.(?!\s)(\D)", " \\1", filename) + filename = filename.replace("_", " ") + filename = re.sub("-$", "", filename) + filename = re.sub("^\[.*\]", "", filename) + return filename.strip() -def isMediaFile(mediafile): +def isMediaFile(mediafile, media=True, audio=True, meta=True, archives=True): fileName, fileExt = os.path.splitext(mediafile) # ignore MAC OS's retarded "resource fork" files @@ -592,12 +586,15 @@ def isMediaFile(mediafile): logger.info("Ignoring extras file: %s " % (mediafile)) return False - if fileExt.lower() in nzbtomedia.EXTCONTAINER: + if (media and fileExt.lower() in nzbtomedia.MEDIACONTAINER)\ + or (audio and fileExt.lower() in nzbtomedia.AUDIOCONTAINER)\ + or (meta and fileExt.lower() in nzbtomedia.METACONTAINER)\ + or (archives and fileExt.lower() in nzbtomedia.COMPRESSEDCONTAINER): return True else: return False -def listMediaFiles(path, ignoreSample=True): +def listMediaFiles(path, media=True, audio=True, meta=True, archives=True, ignoreSample=True): if not dir or not os.path.isdir(path): return [] @@ -607,9 +604,9 @@ def listMediaFiles(path, ignoreSample=True): # if it's a folder do it recursively if os.path.isdir(fullCurFile) and not curFile.startswith('.') and not curFile == 'Extras': - files += listMediaFiles(fullCurFile) + files += listMediaFiles(fullCurFile, media, audio, meta, archives) - elif isMediaFile(curFile): + elif isMediaFile(curFile, media, audio, meta, archives): # Optionally ignore sample files if ignoreSample and is_sample(fullCurFile, nzbtomedia.MINSAMPLESIZE, nzbtomedia.SAMPLEIDS): try: