From d511b54e66fea14fdf85fbdfdf4e23ea67af910a Mon Sep 17 00:00:00 2001 From: echel0n Date: Sun, 20 Apr 2014 13:28:48 -0700 Subject: [PATCH] Fixed issues with checking for video corruption --- TorrentToMedia.py | 60 +++----- autoProcessMedia.cfg.spec | 8 +- nzbToMedia.py | 39 +++-- nzbtomedia/Transcoder.py | 4 + nzbtomedia/autoProcess/autoProcessMovie.py | 10 +- nzbtomedia/autoProcess/autoProcessTV.py | 12 +- .../extractor/bin/{x64 => AMD64}/7z.dll | Bin .../extractor/bin/{x64 => AMD64}/7z.exe | Bin .../extractor/bin/{x64 => AMD64}/license.txt | 0 nzbtomedia/extractor/extractor.py | 69 +++------ nzbtomedia/nzbToMediaConfig.py | 14 ++ nzbtomedia/nzbToMediaUtil.py | 40 ++++- tests/general.py | 145 +----------------- 13 files changed, 145 insertions(+), 256 deletions(-) rename nzbtomedia/extractor/bin/{x64 => AMD64}/7z.dll (100%) rename nzbtomedia/extractor/bin/{x64 => AMD64}/7z.exe (100%) rename nzbtomedia/extractor/bin/{x64 => AMD64}/license.txt (100%) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 94bc000e..8d629f79 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -17,16 +17,15 @@ from nzbtomedia.autoProcess.autoProcessMusic import autoProcessMusic from nzbtomedia.autoProcess.autoProcessTV import autoProcessTV from nzbtomedia.extractor import extractor from nzbtomedia.nzbToMediaUtil import category_search, sanitizeFileName, copy_link, parse_args, flatten, get_dirnames, \ - remove_read_only, cleanup_directories, create_torrent_class, pause_torrent, resume_torrent, listMediaFiles, joinPath + remove_read_only, cleanup_directories, create_torrent_class, pause_torrent, resume_torrent, listMediaFiles, joinPath, \ + extractFiles from nzbtomedia import logger def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent): - status = int(1) # 1 = failed | 0 = success + status = 1 # 1 = failed | 0 = success root = 0 video = 0 - archive = 0 foundFile = 0 - extracted_folder = [] copy_list = [] logger.debug("Received Directory: %s | Name: %s | Category: %s" % (inputDirectory, inputName, inputCategory)) @@ -35,6 +34,16 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, logger.debug("Determined Directory: %s | Name: %s | Category: %s" % (inputDirectory, inputName, inputCategory)) + section = nzbtomedia.CFG.findsection(inputCategory) + if section: + if nzbtomedia.CFG[section]['extract']: + logger.debug('Checking for archives to extract in directory: %s' % (inputDirectory)) + extractFiles(inputDirectory) + else: + logger.error( + "We could not find a section with containing a download category labeled %s in your autoProcessMedia.cfg, Exiting!" % inputCategory) + return -1 + TorrentClass = None if clientAgent != 'manual': TorrentClass = create_torrent_class(clientAgent) @@ -50,14 +59,8 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, if nzbtomedia.CFG["SickBeard"][inputCategory]: Torrent_NoLink = int(nzbtomedia.CFG["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 if Torrent_NoLink == 1: - status = 0 - # Check video files for corruption - for video in listMediaFiles(inputDirectory): - if not Transcoder().isVideoGood(video): - status = 1 - logger.info("Calling autoProcessTV to post-process: %s",inputName) - result = autoProcessTV().processEpisode(inputDirectory, inputName, status, clientAgent=clientAgent, inputCategory=inputCategory) + result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) if result != 0: logger.error("A problem was reported in the autoProcessTV script.") @@ -121,25 +124,6 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, else: continue # This file has not been recently moved or created, skip it - if fileExt in nzbtomedia.COMPRESSEDCONTAINER: - if not (nzbtomedia.CFG["SickBeard"][inputCategory] and nzbtomedia.CFG["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination"): - # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. - if re.search(r'\d+', os.path.splitext(fileName)[1]) and fileDirPath in extracted_folder and not any(item in os.path.splitext(fileName)[1] for item in ['.720p','.1080p','.x264']): - part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) - if part == 1: # we only want to extract the primary part. - logger.debug("Found primary part of a multi-part archive %s. Extracting" % (fullFileName)) - else: - logger.debug("Found part %s of a multi-part archive %s. Ignoring" % (part, fullFileName)) - continue - logger.info("Found compressed archive %s for file %s" % (fileExt, inputFile)) - try: - extractor.extract(inputFile, outputDestination) - extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content) - extracted_folder.append(fileDirPath) - except: - logger.error("Extraction failed for: %s" % (fullFileName)) - continue - try: copy_link(inputFile, targetDirectory, nzbtomedia.USELINK, outputDestination) copy_list.append([inputFile, joinPath(outputDestination, fullFileName)]) @@ -162,19 +146,18 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, if fileExt in nzbtomedia.MEDIACONTAINER: logger.debug("Found media file: %s" % (fullFileName)) video += 1 - if fileExt in nzbtomedia.COMPRESSEDCONTAINER: - logger.debug("Found archive file: %s" % (fullFileName)) - archive += 1 if video > 0: logger.debug("Found %s media files" % (str(video))) status = 0 - elif archive > 0 and not nzbtomedia.CFG["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination": - logger.debug("Found %s archive files to be extracted by SickBeard" % (str(archive))) - status = 0 else: logger.warning("Found no media files in %s" % outputDestination) + + # Only these sections can handling failed downloads so make sure everything else gets through without the check for failed + if not nzbtomedia.CFG['CouchPotato','SickBeard','NzbDrone'][inputCategory]: + status = 0 + result = 0 if (inputCategory in nzbtomedia.USER_SCRIPT_CATEGORIES and not "NONE" in nzbtomedia.USER_SCRIPT_CATEGORIES) or ("ALL" in nzbtomedia.USER_SCRIPT_CATEGORIES and not inputCategory in processCategories): logger.info("Processing user script %s." % (nzbtomedia.USER_SCRIPT)) @@ -183,11 +166,6 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, logger.error("Something failed! Please check logs. Exiting") return status - # Check video files for corruption - for video in listMediaFiles(inputDirectory): - if not Transcoder().isVideoGood(video): - status = 1 - if nzbtomedia.CFG['CouchPotato'][inputCategory]: logger.info("Calling CouchPotato:" + inputCategory + " to post-process: %s" % (inputName)) download_id = inputHash diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 3d56b39f..65c5371f 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -32,6 +32,7 @@ method = renamer delete_failed = 0 wait_for = 2 + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category @@ -51,9 +52,9 @@ ssl = 0 fork = auto delete_failed = 0 - nzbExtractionBy = Downloader Torrent_NoLink = 0 process_method = + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category @@ -73,8 +74,8 @@ web_root = ssl = 0 delete_failed = 0 - nzbExtractionBy = Downloader Torrent_NoLink = 0 + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category @@ -92,6 +93,7 @@ ssl = 0 web_root = wait_for = 2 + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category @@ -109,6 +111,7 @@ ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### web_root= ssl=0 + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category @@ -125,6 +128,7 @@ ###### ADVANCED USE - ONLY EDIT IF YOU KNOW WHAT YOU'RE DOING ###### ssl = 0 web_root = + extract = 1 ##### Set to path where completed downloads are found on remote server for this category remote_path = ##### Set to path where download client places completed downloads locally for this category diff --git a/nzbToMedia.py b/nzbToMedia.py index a29106f1..6639ac4f 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -282,7 +282,7 @@ from nzbtomedia.autoProcess.autoProcessGames import autoProcessGames from nzbtomedia.autoProcess.autoProcessMovie import autoProcessMovie from nzbtomedia.autoProcess.autoProcessMusic import autoProcessMusic from nzbtomedia.autoProcess.autoProcessTV import autoProcessTV -from nzbtomedia.nzbToMediaUtil import get_dirnames, cleanup_directories, listMediaFiles +from nzbtomedia.nzbToMediaUtil import get_dirnames, cleanup_directories, listMediaFiles, extractFiles from nzbtomedia import logger # post-processing @@ -290,32 +290,31 @@ def process(nzbDir, inputName=None, status=0, clientAgent='manual', download_id= # auto-detect section section = nzbtomedia.CFG.findsection(inputCategory) if section: - # Check video files for corruption - for video in listMediaFiles(nzbDir): - if not Transcoder().isVideoGood(video): - status = 1 + if nzbtomedia.CFG[section]['extract']: + logger.debug('Checking for archives to extract in directory: %s' % (nzbDir)) + extractFiles(nzbDir) logger.info("Sending %s to %s for post-processing ..." % (inputName, str(section).upper())) + + if nzbtomedia.CFG["CouchPotato"][inputCategory]: + result = autoProcessMovie().process(nzbDir, inputName, status, clientAgent, download_id, inputCategory) + elif nzbtomedia.CFG["SickBeard", "NzbDrone"][inputCategory]: + result = autoProcessTV().processEpisode(nzbDir, inputName, status, clientAgent, inputCategory) + elif nzbtomedia.CFG["HeadPhones"][inputCategory]: + result = autoProcessMusic().process(nzbDir, inputName, status, clientAgent, inputCategory) + elif nzbtomedia.CFG["Mylar"][inputCategory]: + result = autoProcessComics().processEpisode(nzbDir, inputName, status, clientAgent, inputCategory) + elif nzbtomedia.CFG["Gamez"][inputCategory]: + result = autoProcessGames().process(nzbDir, inputName, status, clientAgent, inputCategory) + else: + result = -1 else: logger.error("We could not find a section with containing a download category labeled %s in your autoProcessMedia.cfg, Exiting!" % inputCategory) - return -1 - - if nzbtomedia.CFG["CouchPotato"][inputCategory]: - result = autoProcessMovie().process(nzbDir, inputName, status, clientAgent, download_id, inputCategory) - elif nzbtomedia.CFG["SickBeard", "NzbDrone"][inputCategory]: - result = autoProcessTV().processEpisode(nzbDir, inputName, status, clientAgent, inputCategory) - elif nzbtomedia.CFG["HeadPhones"][inputCategory]: - result = autoProcessMusic().process(nzbDir, inputName, status, clientAgent, inputCategory) - elif nzbtomedia.CFG["Mylar"][inputCategory]: - result = autoProcessComics().processEpisode(nzbDir, inputName, status, clientAgent, inputCategory) - elif nzbtomedia.CFG["Gamez"][inputCategory]: - result = autoProcessGames().process(nzbDir, inputName, status, clientAgent, inputCategory) - else: result = -1 if result == 0: - # Clean up any leftover files - cleanup_directories(inputCategory, section, result, nzbDir) + # Clean up any leftover files + cleanup_directories(inputCategory, section, result, nzbDir) return result diff --git a/nzbtomedia/Transcoder.py b/nzbtomedia/Transcoder.py index fa734674..039c9062 100644 --- a/nzbtomedia/Transcoder.py +++ b/nzbtomedia/Transcoder.py @@ -8,6 +8,10 @@ from nzbToMediaUtil import listMediaFiles class Transcoder: def isVideoGood(self, videofile): + fileName, fileExt = os.path.splitext(videofile) + if fileExt not in nzbtomedia.MEDIACONTAINER: + return True + if platform.system() == 'Windows': bitbucket = open('NUL') else: diff --git a/nzbtomedia/autoProcess/autoProcessMovie.py b/nzbtomedia/autoProcess/autoProcessMovie.py index 7b45b1a3..c7b1a699 100644 --- a/nzbtomedia/autoProcess/autoProcessMovie.py +++ b/nzbtomedia/autoProcess/autoProcessMovie.py @@ -4,7 +4,7 @@ import nzbtomedia from lib import requests from nzbtomedia.Transcoder import Transcoder from nzbtomedia.nzbToMediaSceneExceptions import process_all_exceptions -from nzbtomedia.nzbToMediaUtil import convert_to_ascii, rmDir, find_imdbid, find_download, joinPath +from nzbtomedia.nzbToMediaUtil import convert_to_ascii, rmDir, find_imdbid, find_download, joinPath, listMediaFiles from nzbtomedia import logger @@ -90,13 +90,17 @@ class autoProcessMovie: "We were unable to find a section for category %s, please check your autoProcessMedia.cfg file." % inputCategory) return 1 + # Check video files for corruption status = int(status) + for video in listMediaFiles(dirName): + if not Transcoder().isVideoGood(video): + status = 1 host = nzbtomedia.CFG[section][inputCategory]["host"] port = nzbtomedia.CFG[section][inputCategory]["port"] apikey = nzbtomedia.CFG[section][inputCategory]["apikey"] method = nzbtomedia.CFG[section][inputCategory]["method"] - rmDir_failed = int(nzbtomedia.CFG[section][inputCategory]["rmDir_failed"]) + delete_failed = int(nzbtomedia.CFG[section][inputCategory]["delete_failed"]) wait_for = int(nzbtomedia.CFG[section][inputCategory]["wait_for"]) try: @@ -186,7 +190,7 @@ class autoProcessMovie: else: logger.postprocess("FAILED DOWNLOAD DETECTED FOR %s" % (nzbName), section) - if rmDir_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: + if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: logger.postprocess("Deleting failed files and folder %s" % dirName, section) rmDir(dirName) diff --git a/nzbtomedia/autoProcess/autoProcessTV.py b/nzbtomedia/autoProcess/autoProcessTV.py index c4c6c441..a8f7aa1d 100644 --- a/nzbtomedia/autoProcess/autoProcessTV.py +++ b/nzbtomedia/autoProcess/autoProcessTV.py @@ -20,7 +20,11 @@ class autoProcessTV: # auto-detect correct fork fork, fork_params = autoFork(inputCategory) + # Check video files for corruption status = int(failed) + for video in listMediaFiles(dirName): + if not Transcoder().isVideoGood(video): + status = 1 host = nzbtomedia.CFG[section][inputCategory]["host"] port = nzbtomedia.CFG[section][inputCategory]["port"] @@ -40,9 +44,9 @@ class autoProcessTV: except: web_root = "" try: - rmDir_failed = int(nzbtomedia.CFG[section][inputCategory]["rmDir_failed"]) + delete_failed = int(nzbtomedia.CFG[section][inputCategory]["rmDir_failed"]) except: - rmDir_failed = 0 + delete_failed = 0 try: nzbExtractionBy = nzbtomedia.CFG[section][inputCategory]["nzbExtractionBy"] except: @@ -128,7 +132,7 @@ class autoProcessTV: logger.postprocess("FAILED: The download failed. Sending 'failed' process request to %s branch" % (fork), section) else: logger.postprocess("FAILED: The download failed. %s branch does not handle failed downloads. Nothing to process" % (fork), section) - if rmDir_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: + if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: logger.postprocess("Deleting failed files and folder %s" % (dirName), section) rmDir(dirName) return 0 # Success (as far as this script is concerned) @@ -168,7 +172,7 @@ class autoProcessTV: for line in r.iter_lines(): if line: logger.postprocess("%s" % (line), section) - if status != 0 and rmDir_failed and not os.path.dirname(dirName) == dirName: + if status != 0 and delete_failed and not os.path.dirname(dirName) == dirName: logger.postprocess("Deleting failed files and folder %s" % (dirName),section) rmDir(dirName) return 0 # Success diff --git a/nzbtomedia/extractor/bin/x64/7z.dll b/nzbtomedia/extractor/bin/AMD64/7z.dll similarity index 100% rename from nzbtomedia/extractor/bin/x64/7z.dll rename to nzbtomedia/extractor/bin/AMD64/7z.dll diff --git a/nzbtomedia/extractor/bin/x64/7z.exe b/nzbtomedia/extractor/bin/AMD64/7z.exe similarity index 100% rename from nzbtomedia/extractor/bin/x64/7z.exe rename to nzbtomedia/extractor/bin/AMD64/7z.exe diff --git a/nzbtomedia/extractor/bin/x64/license.txt b/nzbtomedia/extractor/bin/AMD64/license.txt similarity index 100% rename from nzbtomedia/extractor/bin/x64/license.txt rename to nzbtomedia/extractor/bin/AMD64/license.txt diff --git a/nzbtomedia/extractor/extractor.py b/nzbtomedia/extractor/extractor.py index 4c7d5ae4..1f09f5c1 100644 --- a/nzbtomedia/extractor/extractor.py +++ b/nzbtomedia/extractor/extractor.py @@ -1,41 +1,17 @@ import os -import sys +import platform +from time import sleep import nzbtomedia from subprocess import call, Popen -from nzbtomedia.nzbToMediaUtil import makeDir, joinPath -from nzbtomedia import logger - -# which() and os_platform() breaks when running in Transmission (has to do with os.environ) - -def os_platform(): - # Author Credit: Matthew Scouten @ http://stackoverflow.com/a/7260315 - true_platform = os.environ['PROCESSOR_ARCHITECTURE'] - try: - true_platform = os.environ["PROCESSOR_ARCHITEW6432"] - except KeyError: - pass - #true_platform not assigned to if this does not exist - return true_platform - def extract(filePath, outputDestination): # Using Windows - if os.name == 'nt': - if os_platform() == 'AMD64': - platform = 'x64' - else: - platform = 'x86' - if not os.path.dirname(sys.argv[0]): - chplocation = os.path.normpath(joinPath(os.getcwd(), 'nzbtomedia/extractor/bin/chp.exe')) - sevenzipLocation = os.path.normpath( - joinPath(os.getcwd(), 'nzbtomedia/extractor/bin/' + platform + '/7z.exe')) - else: - chplocation = os.path.normpath( - joinPath(os.path.dirname(sys.argv[0]), 'nzbtomedia/extractor/bin/chp.exe')) - sevenzipLocation = os.path.normpath( - joinPath(os.path.dirname(sys.argv[0]), 'nzbtomedia/extractor/bin/' + platform + '/7z.exe')) + if platform.system() == 'Windows': + chplocation = nzbtomedia.joinPath(nzbtomedia.PROGRAM_DIR, 'nzbtomedia/extractor/bin/chp.exe') + sevenzipLocation = nzbtomedia.joinPath(nzbtomedia.PROGRAM_DIR, 'nzbtomedia/extractor/bin/' + platform.machine() + '/7z.exe') + if not os.path.exists(sevenzipLocation): - logger.error("EXTRACTOR: Could not find 7-zip, Exiting") + nzbtomedia.logger.error("EXTRACTOR: Could not find 7-zip, Exiting") return False else: if not os.path.exists(chplocation): @@ -70,13 +46,13 @@ def extract(filePath, outputDestination): if call(['which', cmd]): #note, returns 0 if exists, or 1 if doesn't exist. for k, v in EXTRACT_COMMANDS.items(): if cmd in v[0]: - logger.error("EXTRACTOR: %s not found, disabling support for %s" % (cmd, k)) + nzbtomedia.logger.error("EXTRACTOR: %s not found, disabling support for %s" % (cmd, k)) del EXTRACT_COMMANDS[k] else: - logger.warning("EXTRACTOR: Cannot determine which tool to use when called from Transmission") + nzbtomedia.logger.warning("EXTRACTOR: Cannot determine which tool to use when called from Transmission") if not EXTRACT_COMMANDS: - logger.warning("EXTRACTOR: No archive extracting programs found, plugin will be disabled") + nzbtomedia.logger.warning("EXTRACTOR: No archive extracting programs found, plugin will be disabled") ext = os.path.splitext(filePath) cmd = [] @@ -90,20 +66,20 @@ def extract(filePath, outputDestination): if ext[1] in EXTRACT_COMMANDS: cmd = EXTRACT_COMMANDS[ext[1]] else: - logger.debug("EXTRACTOR: Unknown file type: %s" % ext[1]) + nzbtomedia.logger.debug("EXTRACTOR: Unknown file type: %s" % ext[1]) return False # Create outputDestination folder - makeDir(outputDestination) + nzbtomedia.makeDir(outputDestination) passwordsfile = nzbtomedia.CFG["passwords"]["PassWordFile"] if passwordsfile != "" and os.path.isfile(os.path.normpath(passwordsfile)): passwords = [line.strip() for line in open(os.path.normpath(passwordsfile))] else: passwords = [] + nzbtomedia.logger.info("Extracting %s to %s" % (filePath, outputDestination)) + nzbtomedia.logger.debug("Extracting %s %s %s" % (cmd, filePath, outputDestination)) - logger.info("Extracting %s to %s" % (filePath, outputDestination)) - logger.debug("Extracting %s %s %s" % (cmd, filePath, outputDestination)) pwd = os.getcwd() # Get our Present Working Directory os.chdir(outputDestination) # Not all unpack commands accept full paths, so just extract into this directory try: # now works same for nt and *nix @@ -114,9 +90,9 @@ def extract(filePath, outputDestination): res = p.wait() if ( res >= 0 and os.name == 'nt') or res == 0: # for windows chp returns process id if successful or -1*Error code. Linux returns 0 for successful. - logger.info("EXTRACTOR: Extraction was successful for %s to %s" % (filePath, outputDestination)) + nzbtomedia.logger.info("EXTRACTOR: Extraction was successful for %s to %s" % (filePath, outputDestination)) elif len(passwords) > 0: - logger.info("EXTRACTOR: Attempting to extract with passwords") + nzbtomedia.logger.info("EXTRACTOR: Attempting to extract with passwords") pass_success = int(0) for password in passwords: if password == "": # if edited in windows or otherwise if blank lines. @@ -128,16 +104,21 @@ def extract(filePath, outputDestination): p = Popen(cmd2) # should extract files fine. res = p.wait() if ( - res >= 0 and os.name == 'nt') or res == 0: # for windows chp returns process id if successful or -1*Error code. Linux returns 0 for successful. - logger.info("EXTRACTOR: Extraction was successful for %s to %s using password: %s" % ( + res >= 0 and platform == 'Windows') or res == 0: # for windows chp returns process id if successful or -1*Error code. Linux returns 0 for successful. + nzbtomedia.logger.info("EXTRACTOR: Extraction was successful for %s to %s using password: %s" % ( filePath, outputDestination, password)) pass_success = int(1) break else: continue if pass_success == int(0): - logger.error("EXTRACTOR: Extraction failed for %s. 7zip result was %s" % (filePath, res)) + nzbtomedia.logger.error("EXTRACTOR: Extraction failed for %s. 7zip result was %s" % (filePath, res)) except: - logger.error("EXTRACTOR: Extraction failed for %s. Could not call command %s" % (filePath, cmd)) + nzbtomedia.logger.error("EXTRACTOR: Extraction failed for %s. Could not call command %s" % (filePath, cmd)) + os.chdir(pwd) # Go back to our Original Working Directory + + # sleep to let files finish writing to disk + sleep (3) + return True diff --git a/nzbtomedia/nzbToMediaConfig.py b/nzbtomedia/nzbToMediaConfig.py index 84bc71cf..0ceae195 100644 --- a/nzbtomedia/nzbToMediaConfig.py +++ b/nzbtomedia/nzbToMediaConfig.py @@ -146,6 +146,20 @@ class ConfigObj(configobj.ConfigObj, Section): if option == "forceClean": CFG_NEW['General']['force_clean'] = value values.pop(option) + + # remove any options that we no longer use or need from new config + def find_key(d, key): + for k, v in d.items(): + if isinstance(v, dict): + p = find_key(v, key) + if p: + return [k] + p + elif v == key: + return [k] + + #if not find_key(CFG_NEW, option): + # values.pop(option) + return values def process_section(section, subsections=None): diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index 4745a11b..00ad915e 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -9,6 +9,7 @@ import nzbtomedia from lib import requests from lib import guessit +from nzbtomedia.extractor import extractor from nzbtomedia.linktastic import linktastic from nzbtomedia import logger from nzbtomedia.synchronousdeluge.client import DelugeClient @@ -668,4 +669,41 @@ def find_imdbid(dirName, nzbName): logger.info("Found imdbID [%s]" % imdbid) return imdbid - logger.warning('Unable to find a imdbID for %s' % (nzbName)) \ No newline at end of file + logger.warning('Unable to find a imdbID for %s' % (nzbName)) + +def extractFiles(src, dst=None): + extracted_folder = [] + + for inputFile in listMediaFiles(src): + dirPath = os.path.dirname(inputFile) + fileName, fileExt = os.path.splitext(os.path.basename(inputFile)) + fullFileName = os.path.basename(inputFile) + + if fileExt in nzbtomedia.COMPRESSEDCONTAINER: + if re.search('part\d+', fullFileName): + if not re.search('^((?!\.part(?!0*1\.rar$)\d+\.rar$).)*\.(?:rar|r?0*1)$', fullFileName): + continue + + logger.info("Found compressed archive %s for file %s" % (fileExt, fullFileName)) + + while(True): + try: + extractor.extract(inputFile, dst or dirPath) + extracted_folder.append(dst or dirPath) + break + except: + logger.error("Extraction failed for: %s" % (fullFileName)) + + if extracted_folder: + for folder in extracted_folder: + for inputFile in listMediaFiles(folder): + fullFileName = os.path.basename(inputFile) + fileName, fileExt = os.path.splitext(fullFileName) + + if fileExt in nzbtomedia.COMPRESSEDCONTAINER: + logger.info("Removing extracted archive %s from folder %s ..." % (fullFileName, folder)) + try: + os.remove(inputFile) + time.sleep(1) + except: + logger.debug("Unable to remove file %s" % (inputFile)) \ No newline at end of file diff --git a/tests/general.py b/tests/general.py index 2f41e850..3988916b 100644 --- a/tests/general.py +++ b/tests/general.py @@ -1,147 +1,10 @@ -import datetime -import os -import re import nzbtomedia -import platform -from nzbtomedia.extractor import extractor -from nzbtomedia.nzbToMediaUtil import listMediaFiles, sanitizeFileName, \ - category_search, copy_link, flatten, remove_read_only, joinPath -from nzbtomedia import logger - -status = int(1) # 1 = failed | 0 = success -root = 0 -video = 0 -archive = 0 -foundFile = 0 -extracted_folder = [] -copy_list = [] +from nzbtomedia.nzbToMediaUtil import extractFiles # Initialize the config nzbtomedia.initialize() -EXTCONTAINER = [] +inputDirectory = "Z:\complete\movie\lego movie" +inputName = "lego movie" -try: - EXTCONTAINER += nzbtomedia.COMPRESSEDCONTAINER.split(',') -except: - EXTCONTAINER.extend(nzbtomedia.COMPRESSEDCONTAINER) - -try: - EXTCONTAINER += nzbtomedia.MEDIACONTAINER.split(',') -except: - EXTCONTAINER.extend(nzbtomedia.MEDIACONTAINER) - -try: - EXTCONTAINER += nzbtomedia.AUDIOCONTAINER.split(',') -except: - EXTCONTAINER.extend(nzbtomedia.AUDIOCONTAINER) - -inputDirectory = "Z:\complete\movie\The.Lego.Movie.2014.R5.x264.English.XviD-vTg.nfo_0166_-_The.Lego.Movie.2014.R5.x264.English.XviD-vTg.nfo_yEn.cp(tt1490017)" -inputName = "The.Lego.Movie.2014.R5.x264.English.XviD-vTg.nfo_0166_-_The.Lego.Movie.2014.R5.x264.English.XviD-vTg.nfo_yEn.cp(tt1490017)" -inputCategory = 'movie' - -inputDirectory, inputName, inputCategory, root, single = category_search(inputDirectory, inputName, inputCategory, root, nzbtomedia.CATEGORIES) # Confirm the category by parsing directory structure -outputDestination = os.path.normpath(joinPath(nzbtomedia.OUTPUTDIRECTORY, inputCategory, sanitizeFileName(inputName))) - -logger.info("Scanning files in directory: %s" % (inputDirectory)) - -if nzbtomedia.CFG["HeadPhones"][inputCategory]: - nzbtomedia.NOFLATTEN.extend( - nzbtomedia.CFG["HeadPhones"].sections) # Make sure we preserve folder structure for HeadPhones. - -outputDestinationMaster = outputDestination # Save the original, so we can change this within the loop below, and reset afterwards. -now = datetime.datetime.now() - -inputFiles = listMediaFiles(inputDirectory) -logger.info("Found %s files in %s" % (str(len(inputFiles)), inputDirectory)) -for inputFile in inputFiles: - fileDirPath = os.path.dirname(inputFile) - fileName, fileExt = os.path.splitext(os.path.basename(inputFile)) - fullFileName = os.path.basename(inputFile) - - if inputCategory in nzbtomedia.NOFLATTEN: - if not fileDirPath == outputDestinationMaster: - outputDestination = joinPath(outputDestinationMaster, - fileDirPath) # join this extra directory to output. - logger.info("Setting outputDestination to %s to preserve folder structure" % (outputDestination)) - - targetDirectory = joinPath(outputDestination, fullFileName) - - if root == 1: - if not foundFile: - logger.info("Looking for %s in: %s" % (inputName, fullFileName)) - if (sanitizeFileName(inputName) in sanitizeFileName(fullFileName)) or ( - sanitizeFileName(fileName) in sanitizeFileName(inputName)): - foundFile = True - logger.info("Found file %s that matches Torrent Name %s" % (fullFileName, inputName)) - else: - continue - - if root == 2: - mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(inputFile)) - ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(inputFile)) - - if not foundFile: - logger.info("Looking for files with modified/created dates less than 5 minutes old.") - if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): - #pass # This file does match the date time criteria - foundFile = True - logger.info("Found file %s with date modifed/created less than 5 minutes ago." % (fullFileName)) - else: - continue # This file has not been recently moved or created, skip it - - if fileExt in nzbtomedia.COMPRESSEDCONTAINER: - if not (nzbtomedia.CFG["SickBeard"][inputCategory] and nzbtomedia.CFG["SickBeard"][inputCategory][ - "nzbExtractionBy"] == "Destination"): - # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. - if re.search(r'\d+', os.path.splitext(fileName)[1]) and fileDirPath in extracted_folder and not any( - item in os.path.splitext(fileName)[1] for item in ['.720p', '.1080p', '.x264']): - part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) - if part == 1: # we only want to extract the primary part. - logger.info("Found primary part of a multi-part archive %s. Extracting" % (fullFileName)) - else: - logger.info("Found part %s of a multi-part archive %s. Ignoring" % (part, fullFileName)) - continue - logger.info("Found compressed archive %s for file %s" % (fileExt, inputFile)) - try: - extractor.extract(inputFile, outputDestination) - extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content) - extracted_folder.append(fileDirPath) - except: - logger.error("Extraction failed for: %s" % (fullFileName)) - continue - - try: - copy_link(inputFile, targetDirectory, nzbtomedia.USELINK, outputDestination) - copy_list.append([inputFile, joinPath(outputDestination, fullFileName)]) - except: - logger.error("Failed to link file: %s" % (fullFileName)) - -outputDestination = outputDestinationMaster # Reset here. -if not inputCategory in nzbtomedia.NOFLATTEN: #don't flatten hp in case multi cd albums, and we need to copy this back later. - flatten(outputDestination) - -if platform.system() == 'Windows': # remove Read Only flag from files in Windows. - remove_read_only(outputDestination) - -# Now check if video files exist in destination: -if nzbtomedia.CFG["SickBeard", "NzbDrone", "CouchPotato"][inputCategory]: - for outputFile in listMediaFiles(outputDestination): - fullFileName = os.path.basename(outputFile) - fileName, fileExt = os.path.splitext(fullFileName) - - if fileExt in nzbtomedia.MEDIACONTAINER: - logger.info("Found media file: %s" % (fullFileName)) - video += 1 - if fileExt in nzbtomedia.COMPRESSEDCONTAINER: - logger.info("Found archive file: %s" % (fullFileName)) - archive += 1 - - if video > 0: - logger.info("Found %s media files" % (str(video))) - status = 0 - elif archive > 0 and not nzbtomedia.CFG["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination": - logger.info("Found %s archive files to be extracted by SickBeard" % (str(archive))) - status = 0 - else: - logger.warning("Found no media files in %s" % outputDestination) +extractFiles(inputDirectory) \ No newline at end of file