From fd4d14be641a8ebe864cae29dbe9fc163f228125 Mon Sep 17 00:00:00 2001 From: echel0n Date: Fri, 18 Apr 2014 03:10:24 -0700 Subject: [PATCH] We now can post-process files not inside of there own folder, we simply confirm there at the root of our category or watch folder then move them into there own seperate foldersand go from there. nzbName's are now cleaned up. --- nzbtomedia/Transcoder.py | 62 ++++++--------- nzbtomedia/__init__.py | 36 ++++++++- nzbtomedia/autoProcess/autoProcessMovie.py | 3 +- nzbtomedia/linktastic/linktastic.py | 54 +++++++------- nzbtomedia/nzbToMediaSceneExceptions.py | 9 ++- nzbtomedia/nzbToMediaUtil.py | 87 ++++++++++++++++++---- tests/general.py | 12 ++- 7 files changed, 175 insertions(+), 88 deletions(-) diff --git a/nzbtomedia/Transcoder.py b/nzbtomedia/Transcoder.py index a26a642c..b8fe8949 100644 --- a/nzbtomedia/Transcoder.py +++ b/nzbtomedia/Transcoder.py @@ -27,25 +27,11 @@ class Transcoder: ffmpeg = 'ffmpeg' useNiceness = True - mediaContainer = (nzbtomedia.CFG["Transcoder"]["duplicate"]) - duplicate = int(nzbtomedia.CFG["Transcoder"]["duplicate"]) - ignoreExtensions = (nzbtomedia.CFG["Transcoder"]["ignoreExtensions"]) - outputVideoExtension = nzbtomedia.CFG["Transcoder"]["outputVideoExtension"].strip() - outputVideoCodec = nzbtomedia.CFG["Transcoder"]["outputVideoCodec"].strip() - outputVideoPreset = nzbtomedia.CFG["Transcoder"]["outputVideoPreset"].strip() - outputVideoFramerate = nzbtomedia.CFG["Transcoder"]["outputVideoFramerate"].strip() - outputVideoBitrate = nzbtomedia.CFG["Transcoder"]["outputVideoBitrate"].strip() - outputAudioCodec = nzbtomedia.CFG["Transcoder"]["outputAudioCodec"].strip() - outputAudioBitrate = nzbtomedia.CFG["Transcoder"]["outputAudioBitrate"].strip() - outputSubtitleCodec = nzbtomedia.CFG["Transcoder"]["outputSubtitleCodec"].strip() - outputFastStart = int(nzbtomedia.CFG["Transcoder"]["outputFastStart"]) - outputQualityPercent = int(nzbtomedia.CFG["Transcoder"]["outputQualityPercent"]) - niceness = None - if useNiceness:niceness = int(nzbtomedia.CFG["Transcoder"]["niceness"]) + if useNiceness:niceness = nzbtomedia.NICENESS - map(lambda ext: ext.strip(), mediaContainer) - map(lambda ext: ext.strip(), ignoreExtensions) + map(lambda ext: ext.strip(), nzbtomedia.MEDIACONTAINER) + map(lambda ext: ext.strip(), nzbtomedia.IGNOREEXTENSIONS) logger.info("Checking for files to be transcoded") final_result = 0 # initialize as successful @@ -53,8 +39,8 @@ class Transcoder: for file in filenames: filePath = os.path.join(dirpath, file) name, ext = os.path.splitext(filePath) - if ext in mediaContainer: # If the file is a video file - if ext in ignoreExtensions: + if ext in nzbtomedia.MEDIACONTAINER: # If the file is a video file + if ext in nzbtomedia.IGNOREEXTENSIONS: logger.info("No need to transcode video type %s", ext) continue if ext == outputVideoExtension: # we need to change the name to prevent overwriting itself. @@ -66,42 +52,42 @@ class Transcoder: if useNiceness: command = ['nice', '-%d' % niceness] + command - if len(outputVideoCodec) > 0: + if len(nzbtomedia.OUTPUTVIDEOCODEC) > 0: command.append('-c:v') - command.append(outputVideoCodec) - if outputVideoCodec == 'libx264' and outputVideoPreset: + command.append(nzbtomedia.OUTPUTVIDEOCODEC) + if nzbtomedia.OUTPUTVIDEOCODEC == 'libx264' and nzbtomedia.OUTPUTVIDEOPRESET: command.append('-preset') - command.append(outputVideoPreset) + command.append(nzbtomedia.OUTPUTVIDEOPRESET) else: command.append('-c:v') command.append('copy') - if len(outputVideoFramerate) > 0: + if len(nzbtomedia.OUTPUTVIDEOFRAMERATE) > 0: command.append('-r') - command.append(str(outputVideoFramerate)) - if len(outputVideoBitrate) > 0: + command.append(str(nzbtomedia.OUTPUTVIDEOFRAMERATE)) + if len(nzbtomedia.OUTPUTVIDEOBITRATE) > 0: command.append('-b:v') - command.append(str(outputVideoBitrate)) - if len(outputAudioCodec) > 0: + command.append(str(nzbtomedia.OUTPUTVIDEOBITRATE)) + if len(nzbtomedia.OUTPUTAUDIOCODEC) > 0: command.append('-c:a') - command.append(outputAudioCodec) - if outputAudioCodec == 'aac': # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg + command.append(nzbtomedia.OUTPUTAUDIOCODEC) + if nzbtomedia.OUTPUTAUDIOCODEC == 'aac': # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg command.append('-strict') command.append('-2') else: command.append('-c:a') command.append('copy') - if len(outputAudioBitrate) > 0: + if len(nzbtomedia.OUTPUTAUDIOBITRATE) > 0: command.append('-b:a') - command.append(str(outputAudioBitrate)) - if outputFastStart > 0: + command.append(str(nzbtomedia.OUTPUTAUDIOBITRATE)) + if nzbtomedia.OUTPUTFASTSTART > 0: command.append('-movflags') command.append('+faststart') - if outputQualityPercent > 0: + if nzbtomedia.OUTPUTQUALITYPERCENT > 0: command.append('-q:a') - command.append(str(outputQualityPercent)) - if len(outputSubtitleCodec) > 0: # Not every subtitle codec can be used for every video container format! + command.append(str(nzbtomedia.OUTPUTQUALITYPERCENT)) + if len(nzbtomedia.OUTPUTSUBTITLECODEC) > 0: # Not every subtitle codec can be used for every video container format! command.append('-c:s') - command.append(outputSubtitleCodec) # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options + command.append(nzbtomedia.OUTPUTSUBTITLECODEC) # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options else: command.append('-sn') # Don't copy the subtitles over command.append(newfilePath) @@ -126,7 +112,7 @@ class Transcoder: logger.error("Transcoding of video %s has failed", filePath) if result == 0: logger.info("Transcoding of video %s to %s succeeded", filePath, newfilePath) - if duplicate == 0: # we get rid of the original file + if nzbtomedia.DUPLICATE == 0: # we get rid of the original file os.unlink(filePath) else: logger.error("Transcoding of video %s to %s failed", filePath, newfilePath) diff --git a/nzbtomedia/__init__.py b/nzbtomedia/__init__.py index 148c03e1..2768d34d 100644 --- a/nzbtomedia/__init__.py +++ b/nzbtomedia/__init__.py @@ -81,6 +81,7 @@ DELUGEUSR = None DELUGEPWD = None COMPRESSEDCONTAINER = None +MEDIAEXTENSIONS = None MEDIACONTAINER = None METACONTAINER = None MINSAMPLESIZE = None @@ -90,7 +91,19 @@ SECTIONS = [] SUBSECTIONS = {} TRANSCODE = None - +DUPLICATE = None +IGNOREEXTENSIONS = None +OUTPUTVIDEOEXTENSION = None +OUTPUTVIDEOCODEC = None +OUTPUTVIDEOPRESET = None +OUTPUTVIDEOFRAMERATE = None +OUTPUTVIDEOBITRATE = None +OUTPUTAUDIOCODEC = None +OUTPUTAUDIOBITRATE = None +OUTPUTSUBTITLECODEC = None +OUTPUTFASTSTART = None +OUTPUTQUALITYPERCENT = None +NICENESS = None USER_SCRIPT_CATEGORIES = None USER_SCRIPT_MEDIAEXTENSIONS = None USER_SCRIPT = None @@ -100,6 +113,7 @@ USER_SCRIPT_CLEAN = None USER_DELAY = None USER_SCRIPT_RUNONCE = None + __INITIALIZED__ = False def initialize(section=None): @@ -112,8 +126,10 @@ def initialize(section=None): TRANSMISSIONPWD, TRANSMISSIONUSR, COMPRESSEDCONTAINER, MEDIACONTAINER, METACONTAINER, MINSAMPLESIZE, SAMPLEIDS, \ SECTIONS, SUBSECTIONS, USER_SCRIPT_CATEGORIES, __INITIALIZED__, AUTO_UPDATE, APP_FILENAME, USER_DELAY, USER_SCRIPT_RUNONCE, \ APP_NAME,USER_SCRIPT_MEDIAEXTENSIONS, USER_SCRIPT, USER_SCRIPT_PARAM, USER_SCRIPT_SUCCESSCODES, USER_SCRIPT_CLEAN, \ - TRANSCODE, GIT_PATH, GIT_USER, GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY - + TRANSCODE, GIT_PATH, GIT_USER, GIT_BRANCH, GIT_REPO, SYS_ENCODING, NZB_CLIENTAGENT, SABNZBDHOST, SABNZBDPORT, SABNZBDAPIKEY, \ + DUPLICATE, IGNOREEXTENSIONS, OUTPUTVIDEOEXTENSION, OUTPUTVIDEOCODEC, OUTPUTVIDEOPRESET, OUTPUTVIDEOFRAMERATE, \ + OUTPUTVIDEOBITRATE, OUTPUTAUDIOCODEC, OUTPUTAUDIOBITRATE, OUTPUTSUBTITLECODEC, OUTPUTFASTSTART, OUTPUTQUALITYPERCENT, \ + NICENESS, MEDIAEXTENSIONS if __INITIALIZED__: return False @@ -227,11 +243,25 @@ def initialize(section=None): DELUGEPWD = CFG["Torrent"]["DelugePWD"] # mysecretpwr COMPRESSEDCONTAINER = (CFG["Extensions"]["compressedExtensions"]) # .zip,.rar,.7z + MEDIAEXTENSIONS = [ 'mkv', 'avi', 'divx', 'xvid', 'mov', 'wmv','mp4', 'mpg', 'mpeg', 'iso' ] MEDIACONTAINER = (CFG["Extensions"]["mediaExtensions"]) # .mkv,.avi,.divx METACONTAINER = (CFG["Extensions"]["metaExtensions"]) # .nfo,.sub,.srt MINSAMPLESIZE = int(CFG["Extensions"]["minSampleSize"]) # 200 (in MB) SAMPLEIDS = (CFG["Extensions"]["SampleIDs"]) # sample,-s. TRANSCODE = int(CFG["Transcoder"]["transcode"]) + DUPLICATE = int(CFG["Transcoder"]["duplicate"]) + IGNOREEXTENSIONS = (CFG["Transcoder"]["ignoreExtensions"]) + OUTPUTVIDEOEXTENSION = CFG["Transcoder"]["outputVideoExtension"].strip() + OUTPUTVIDEOCODEC = CFG["Transcoder"]["outputVideoCodec"].strip() + OUTPUTVIDEOPRESET = CFG["Transcoder"]["outputVideoPreset"].strip() + OUTPUTVIDEOFRAMERATE = CFG["Transcoder"]["outputVideoFramerate"].strip() + OUTPUTVIDEOBITRATE = CFG["Transcoder"]["outputVideoBitrate"].strip() + OUTPUTAUDIOCODEC = CFG["Transcoder"]["outputAudioCodec"].strip() + OUTPUTAUDIOBITRATE = CFG["Transcoder"]["outputAudioBitrate"].strip() + OUTPUTSUBTITLECODEC = CFG["Transcoder"]["outputSubtitleCodec"].strip() + OUTPUTFASTSTART = int(CFG["Transcoder"]["outputFastStart"]) + OUTPUTQUALITYPERCENT = int(CFG["Transcoder"]["outputQualityPercent"]) + NICENESS = int(CFG["Transcoder"]["niceness"]) # check for script-defied section and if None set to allow sections SECTIONS = ("CouchPotato", "SickBeard", "NzbDrone", "HeadPhones", "Mylar", "Gamez") diff --git a/nzbtomedia/autoProcess/autoProcessMovie.py b/nzbtomedia/autoProcess/autoProcessMovie.py index 326424c1..31a9dd39 100644 --- a/nzbtomedia/autoProcess/autoProcessMovie.py +++ b/nzbtomedia/autoProcess/autoProcessMovie.py @@ -5,11 +5,12 @@ import nzbtomedia from lib import requests from nzbtomedia.Transcoder import Transcoder from nzbtomedia.nzbToMediaSceneExceptions import process_all_exceptions -from nzbtomedia.nzbToMediaUtil import convert_to_ascii, delete, create_torrent_class +from nzbtomedia.nzbToMediaUtil import convert_to_ascii, delete, create_torrent_class, clean_nzbname from nzbtomedia import logger class autoProcessMovie: def find_imdbid(self, dirName, nzbName): + nzbName = clean_nzbname(nzbName) # find imdbid in dirName m = re.search('(tt\d{7})', dirName) if m: diff --git a/nzbtomedia/linktastic/linktastic.py b/nzbtomedia/linktastic/linktastic.py index 76687666..cfbcdc1f 100644 --- a/nzbtomedia/linktastic/linktastic.py +++ b/nzbtomedia/linktastic/linktastic.py @@ -28,49 +28,49 @@ import os # Prevent spaces from messing with us! def _escape_param(param): - return '"%s"' % param + return '"%s"' % param # Private function to create link on nt-based systems def _link_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink /H %s %s' % (_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT) - except CalledProcessError as err: + try: + subprocess.check_output( + 'cmd /C mklink /H %s %s' % (_escape_param(dest), _escape_param(src)), + stderr=subprocess.STDOUT) + except CalledProcessError as err: - raise IOError(err.output.decode('utf-8')) + raise IOError(err.output.decode('utf-8')) - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good + # TODO, find out what kind of messages Windows sends us from mklink + # print(stdout) + # assume if they ret-coded 0 we're good def _symlink_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink %s %s' % (_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT) - except CalledProcessError as err: - raise IOError(err.output.decode('utf-8')) + try: + subprocess.check_output( + 'cmd /C mklink %s %s' % (_escape_param(dest), _escape_param(src)), + stderr=subprocess.STDOUT) + except CalledProcessError as err: + raise IOError(err.output.decode('utf-8')) - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good + # TODO, find out what kind of messages Windows sends us from mklink + # print(stdout) + # assume if they ret-coded 0 we're good # Create a hard link to src named as dest # This version of link, unlike os.link, supports nt systems as well def link(src, dest): - if os.name == 'nt': - _link_windows(src, dest) - else: - os.link(src, dest) + if os.name == 'nt': + _link_windows(src, dest) + else: + os.link(src, dest) # Create a symlink to src named as dest, but don't fail if you're on nt def symlink(src, dest): - if os.name == 'nt': - _symlink_windows(src, dest) - else: - os.symlink(src, dest) + if os.name == 'nt': + _symlink_windows(src, dest) + else: + os.symlink(src, dest) diff --git a/nzbtomedia/nzbToMediaSceneExceptions.py b/nzbtomedia/nzbToMediaSceneExceptions.py index 6725810f..b9900dae 100644 --- a/nzbtomedia/nzbToMediaSceneExceptions.py +++ b/nzbtomedia/nzbToMediaSceneExceptions.py @@ -1,6 +1,6 @@ import os import logging -from nzbtomedia.nzbToMediaUtil import iterate_media_files +from nzbtomedia.nzbToMediaUtil import listMediaFiles Logger = logging.getLogger() @@ -11,17 +11,18 @@ def process_all_exceptions(name, dirname): process_exception(exception, name, dirname) def process_exception(exception, name, dirname): - for parentDir, filename in iterate_media_files(dirname): + for filename in listMediaFiles(dirname): + parentDir = os.path.dirname(filename) exception(filename, parentDir) def process_qoq(filename, dirname): - logger.debug("Reversing the file name for a QoQ release %s", filename) + logging.debug("Reversing the file name for a QoQ release %s", filename) head, fileExtension = os.path.splitext(os.path.basename(filename)) newname = head[::-1] newfile = newname + fileExtension newfilePath = os.path.join(dirname, newfile) os.rename(filename, newfilePath) - logger.debug("New file name is %s", newfile) + logging.debug("New file name is %s", newfile) # dict for custom groups # we can add more to this list diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index 86833a60..fb9d2a72 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -188,17 +188,6 @@ def remove_read_only(path): logger.debug("Removing Read Only Flag for: %s", filename) os.chmod(os.path.join(dirpath, filename), stat.S_IWRITE) -def iterate_media_files(dirname): - mediaContainer = [ '.mkv', '.avi', '.divx', '.xvid', '.mov', '.wmv', - '.mp4', '.mpg', '.mpeg', '.iso' ] - - for dirpath, dirnames, filenames in os.walk(dirname): - for filename in filenames: - fileExtension = os.path.splitext(filename)[1] - if not (fileExtension in mediaContainer): - continue - yield dirpath, os.path.join(dirpath, filename) - #Wake function def WakeOnLan(ethernet_address): @@ -374,10 +363,28 @@ def get_dirnames(section, subsections=None): outputDirectory = None if watch_dir: + # search for single files and move them into there own folder for post-processing + for mediafile in listMediaFiles(watch_dir): + parentDir = os.path.dirname(mediafile) + if parentDir == watch_dir: + p = os.path.join(parentDir, (os.path.splitext(os.path.splitext(mediafile)[0])[0])) + if not os.path.exists(p): + os.mkdir(p) + shutil.move(mediafile, p) + dirNames.extend([os.path.join(watch_dir, o) for o in os.listdir(watch_dir) if - os.path.isdir(os.path.join(watch_dir, o))]) + os.path.isdir(os.path.join(watch_dir, o))]) if outputDirectory: + # search for single files and move them into there own folder for post-processing + for mediafile in listMediaFiles(outputDirectory): + parentDir = os.path.dirname(mediafile) + if parentDir == outputDirectory: + p = os.path.join(parentDir, (os.path.splitext(os.path.splitext(mediafile)[0])[0])) + if not os.path.exists(p): + os.mkdir(p) + shutil.move(mediafile, p) + dirNames.extend([os.path.join(outputDirectory, o) for o in os.listdir(outputDirectory) if os.path.isdir(os.path.join(outputDirectory, o))]) @@ -510,4 +517,58 @@ def find_download(clientAgent, nzbName, download_id): return 1 # failure result = r.json() - pass \ No newline at end of file + pass + + +def clean_nzbname(nzbname): + """Cleans up nzb name by removing any . and _ + characters, along with any trailing hyphens. + + Is basically equivalent to replacing all _ and . with a + space, but handles decimal numbers in string, for example: + """ + + nzbname = re.sub("(\D)\.(?!\s)(\D)", "\\1 \\2", nzbname) + nzbname = re.sub("(\d)\.(\d{4})", "\\1 \\2", nzbname) # if it ends in a year then don't keep the dot + nzbname = re.sub("(\D)\.(?!\s)", "\\1 ", nzbname) + nzbname = re.sub("\.(?!\s)(\D)", " \\1", nzbname) + nzbname = nzbname.replace("_", " ") + nzbname = re.sub("-$", "", nzbname) + nzbname = re.sub("^\[.*\]", "", nzbname) + return nzbname.strip() + +def isMediaFile(filename): + # ignore samples + if re.search('(^|[\W_])(sample\d*)[\W_]', filename, re.I): + return False + + # ignore MAC OS's retarded "resource fork" files + if filename.startswith('._'): + return False + + sepFile = filename.rpartition(".") + + if re.search('extras?$', sepFile[0], re.I): + return False + + if sepFile[2].lower() in nzbtomedia.MEDIAEXTENSIONS: + return True + else: + return False + +def listMediaFiles(path): + if not dir or not os.path.isdir(path): + return [] + + files = [] + for curFile in os.listdir(path): + fullCurFile = os.path.join(path, curFile) + + # if it's a folder do it recursively + if os.path.isdir(fullCurFile) and not curFile.startswith('.') and not curFile == 'Extras': + files += listMediaFiles(fullCurFile) + + elif isMediaFile(curFile): + files.append(fullCurFile) + + return files \ No newline at end of file diff --git a/tests/general.py b/tests/general.py index 67aba3bb..618e1c52 100644 --- a/tests/general.py +++ b/tests/general.py @@ -2,18 +2,26 @@ import os import sys import nzbtomedia import TorrentToMedia -from nzbtomedia.nzbToMediaUtil import find_download +from nzbtomedia.nzbToMediaUtil import find_download, clean_nzbname, listMediaFiles os.environ['TR_TORRENT_DIR']="z:/downloads/complete/movie/The.Art.of.the.Steal.2013.LIMITED.1080p.BRRip.h264.AAC-RARBG" os.environ['TR_TORRENT_NAME']="The.Art.of.the.Steal.2013.LIMITED.1080p.BRRip.h264.AAC-RARBG" os.environ['TR_TORRENT_ID']="154206e6390a03bbf01e61f013e1a52494a52dfa" os.environ['TR_TORRENT_HASH']="154206e6390a03bbf01e61f013e1a52494a52dfa" #TorrentToMedia.main(sys.argv) +test = 'The.Art.of.the.Steal.2013.LIMITED.1080p.BRRip.h264.AAC-RARBG' +cleaned = clean_nzbname(test) # Initialize the config nzbtomedia.initialize() -clientAgent = nzbtomedia.NZB_CLIENTAGENT +dirName = 'Z:/complete/tv/' +test = listMediaFiles('Z:/complete/tv/') +for filename in test: + parentDir = os.path.dirname(filename) + if parentDir == dirName: + pass + clientAgent = nzbtomedia.NZB_CLIENTAGENT nzbName = 'Anger.Management.S02E57.HDTV.x264-KILLERS' #download_id = '51C9B415382894727C5C7D8442554D3AC08B390F'