From 98ce939b622e115a61b8ee9424b93cc13ee9df39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 10:35:21 +0100 Subject: [PATCH 01/27] moving "worker" functions to util --- nzbToMediaUtil.py | 157 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index df259695..dd743721 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -26,6 +26,163 @@ def create_destination(outputDestination): Logger.error("CREATE DESTINATION: Not possible to create destination folder: %s. Exiting", e) sys.exit(-1) +def category_search(inputDirectory, inputName, inputCategory, root, categories): + categorySearch = [os.path.normpath(inputDirectory), ""] # initializie + notfound = 0 + for x in range(10): # loop up through 10 directories looking for category. + try: + categorySearch2 = os.path.split(os.path.normpath(categorySearch[0])) + except: # this might happen when we can't go higher. + if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. + Logger.info("SEARCH: Could not find a Torrent Name or category in the directory structure") + Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 1 + break # we are done + elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. + Logger.info("SEARCH: Could not find a Torrent Name or Category in the directory structure") + Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 2 + break # we are done + else: + Logger.error("SEARCH: Could not identify Category of Torrent Name in the directory structure. Please check downloader settings. Exiting") + sys.exit(-1) + + if categorySearch2[1] in categories: + Logger.debug("SEARCH: Found Category: %s in directory structure", categorySearch2[1]) + if not inputCategory: + Logger.info("SEARCH: Determined Category to be: %s", categorySearch2[1]) + inputCategory = categorySearch2[1] + if inputName and categorySearch[0] != os.path.normpath(inputDirectory): # if we are not in the root directory and we have inputName we can continue. + if ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. + Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) + inputName = categorySearch[1] + Logger.info("SEARCH: Identified Category: %s and Torrent Name: %s. We are in a unique directory, so we can proceed.", inputCategory, inputName) + break # we are done + elif categorySearch[1] and not inputName: # assume the the next directory deep is the torrent name. + Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], categorySearch[1]), categorySearch[0]) + inputName = categorySearch[1] + break # we are done + elif ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. + Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) + inputName = categorySearch[1] + break # we are done + elif os.path.isdir(os.path.join(categorySearch[0], inputName)) and inputName: # testing for torrent name in first sub directory + Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], inputName), categorySearch[0]) + if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 + inputDirectory = os.path.join(categorySearch[0], inputName) # we only want to search this next dir up. + break # we are done + elif inputName: # if these exists, we are ok to proceed, but we are in a root/common directory. + Logger.info("SEARCH: Could not find a unique torrent folder in the directory structure") + Logger.info("SEARCH: The directory passed is the root directory for category %s", categorySearch2[1]) + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 1 + break # we are done + else: # this is a problem! if we don't have Torrent name and are in the root category dir, we can't proceed. + Logger.warn("SEARCH: Could not identify a torrent name and the directory passed is common to all downloads for category %s.", categorySearch[1]) + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 2 + break + elif categorySearch2[1] == inputName and inputName: # we have identified a unique directory. + Logger.info("SEARCH: Files appear to be in their own directory") + if inputCategory: # we are ok to proceed. + break # we are done + else: + Logger.debug("SEARCH: Continuing scan to determin category.") + categorySearch = categorySearch2 # ready for next loop + continue # keep going + else: + if x == 9: # This is the last pass in the loop and we didn't find anything. + notfound = 1 + break # we are done + else: + categorySearch = categorySearch2 # ready for next loop + continue # keep going + + if notfound == 1: + if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. + Logger.info("SEARCH: Could not find a category in the directory structure") + Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 1 + elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. + Logger.info("SEARCH: Could not find a Torrent Name or Category in the directory structure") + Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") + Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") + Logger.info("SEARCH: We will try and determine which files to process, individually") + root = 2 + if not inputCategory: # we didn't find this after 10 loops. This is a problem. + Logger.error("SEARCH: Could not identify category and torrent name from the directory structure. Please check downloader settings. Exiting") + sys.exit(-1) # Oh yeah.... WE ARE DONE! + + return inputDirectory, inputName, inputCategory, root + + +def is_sample(filePath, inputName, minSampleSize): + # 200 MB in bytes + SIZE_CUTOFF = minSampleSize * 1024 * 1024 + # Ignore 'sample' in files unless 'sample' in Torrent Name + return ('sample' in filePath.lower()) and (not 'sample' in inputName) and (os.path.getsize(filePath) < SIZE_CUTOFF) + + +def copy_link(source, target, useLink, outputDestination): + create_destination(outputDestination) + if useLink: + try: + Logger.info("COPYLINK: Linking %s to %s", source, target) + linktastic.link(source, target) + except: + if os.path.isfile(target): + Logger.info("COPYLINK: Something went wrong in linktastic.link, but the destination file was created") + else: + Logger.info("COPYLINK: Something went wrong in linktastic.link, copying instead") + Logger.debug("COPYLINK: Copying %s to %s", source, target) + shutil.copy(source, target) + else: + Logger.debug("Copying %s to %s", source, target) + shutil.copy(source, target) + return True + + +def flatten(outputDestination): + Logger.info("FLATTEN: Flattening directory: %s", outputDestination) + for dirpath, dirnames, filenames in os.walk(outputDestination): # Flatten out the directory to make postprocessing easier + if dirpath == outputDestination: + continue # No need to try and move files in the root destination directory + for filename in filenames: + source = os.path.join(dirpath, filename) + target = os.path.join(outputDestination, filename) + try: + shutil.move(source, target) + except OSError: + Logger.error("FLATTEN: Could not flatten %s", source) + removeEmptyFolders(outputDestination) # Cleanup empty directories + + +def removeEmptyFolders(path): + Logger.info("REMOVER: Removing empty folders in: %s", path) + if not os.path.isdir(path): + return + + # Remove empty subfolders + files = os.listdir(path) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + removeEmptyFolders(fullpath) + + # If folder empty, delete it + files = os.listdir(path) + if len(files) == 0: + Logger.debug("REMOVER: Removing empty folder: %s", path) + os.rmdir(path) def iterate_media_files(dirname): mediaContainer = [ '.mkv', '.avi', '.divx', '.xvid', '.mov', '.wmv', From 833d9556d9571446fa04835173198c01a5e75324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 10:54:14 +0100 Subject: [PATCH 02/27] cleanup and more --- TorrentToMedia.py | 476 ++++++++++++++++------------------------------ 1 file changed, 160 insertions(+), 316 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 6dc5cacd..91c92e61 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -12,7 +12,6 @@ from subprocess import call # Custom imports -import linktastic.linktastic as linktastic import extractor.extractor as extractor import autoProcessMovie import autoProcessTV @@ -20,344 +19,189 @@ from nzbToMediaEnv import * from nzbToMediaUtil import * from utorrent.client import UTorrentClient +# Logging nzbtomedia_configure_logging(os.path.dirname(sys.argv[0])) Logger = logging.getLogger(__name__) +def main() -def category_search(inputDirectory, inputName, inputCategory, root, categories): - categorySearch = [os.path.normpath(inputDirectory), ""] # initializie - notfound = 0 - for x in range(10): # loop up through 10 directories looking for category. - try: - categorySearch2 = os.path.split(os.path.normpath(categorySearch[0])) - except: # this might happen when we can't go higher. - if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. - Logger.info("SEARCH: Could not find a Torrent Name or category in the directory structure") - Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - break # we are done - elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. - Logger.info("SEARCH: Could not find a Torrent Name or Category in the directory structure") - Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - break # we are done - else: - Logger.error("SEARCH: Could not identify Category of Torrent Name in the directory structure. Please check downloader settings. Exiting") - sys.exit(-1) + Logger.info("==========================") # Seperate old from new log + Logger.info("TorrentToMedia %s", VERSION) + config = ConfigParser.ConfigParser() + configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg") - if categorySearch2[1] in categories: - Logger.debug("SEARCH: Found Category: %s in directory structure", categorySearch2[1]) - if not inputCategory: - Logger.info("SEARCH: Determined Category to be: %s", categorySearch2[1]) - inputCategory = categorySearch2[1] - if inputName and categorySearch[0] != os.path.normpath(inputDirectory): # if we are not in the root directory and we have inputName we can continue. - if ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. - Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) - inputName = categorySearch[1] - Logger.info("SEARCH: Identified Category: %s and Torrent Name: %s. We are in a unique directory, so we can proceed.", inputCategory, inputName) - break # we are done - elif categorySearch[1] and not inputName: # assume the the next directory deep is the torrent name. - Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], categorySearch[1]), categorySearch[0]) - inputName = categorySearch[1] - break # we are done - elif ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. - Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) - inputName = categorySearch[1] - break # we are done - elif os.path.isdir(os.path.join(categorySearch[0], inputName)) and inputName: # testing for torrent name in first sub directory - Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], inputName), categorySearch[0]) - if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 - inputDirectory = os.path.join(categorySearch[0], inputName) # we only want to search this next dir up. - break # we are done - elif inputName: # if these exists, we are ok to proceed, but we are in a root/common directory. - Logger.info("SEARCH: Could not find a unique torrent folder in the directory structure") - Logger.info("SEARCH: The directory passed is the root directory for category %s", categorySearch2[1]) - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - break # we are done - else: # this is a problem! if we don't have Torrent name and are in the root category dir, we can't proceed. - Logger.warn("SEARCH: Could not identify a torrent name and the directory passed is common to all downloads for category %s.", categorySearch[1]) - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - break - elif categorySearch2[1] == inputName and inputName: # we have identified a unique directory. - Logger.info("SEARCH: Files appear to be in their own directory") - if inputCategory: # we are ok to proceed. - break # we are done - else: - Logger.debug("SEARCH: Continuing scan to determin category.") - categorySearch = categorySearch2 # ready for next loop - continue # keep going - else: - if x == 9: # This is the last pass in the loop and we didn't find anything. - notfound = 1 - break # we are done - else: - categorySearch = categorySearch2 # ready for next loop - continue # keep going + if not os.path.isfile(configFilename): + Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") + sys.exit(-1) - if notfound == 1: - if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. - Logger.info("SEARCH: Could not find a category in the directory structure") - Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. - Logger.info("SEARCH: Could not find a Torrent Name or Category in the directory structure") - Logger.info("SEARCH: We assume the directory passed is the root directory for your downlaoder") - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - if not inputCategory: # we didn't find this after 10 loops. This is a problem. - Logger.error("SEARCH: Could not identify category and torrent name from the directory structure. Please check downloader settings. Exiting") - sys.exit(-1) # Oh yeah.... WE ARE DONE! + Logger.info("MAIN: Loading config from %s", configFilename) + config.read(configFilename) - return inputDirectory, inputName, inputCategory, root + # EXAMPLE VALUES: + clientAgent = config.get("Torrent", "clientAgent") # utorrent | deluge | transmission | other + # Sick-Beard + tvCategory = config.get("SickBeard", "category") # tv + tvDestination = os.path.normpath(config.get("SickBeard", "outputDirectory")) # C:\downloaded\tv | /path/to/downloaded/tv + # CouchPotatoServer + movieCategory = config.get("CouchPotato", "category") # movie + movieDestination = os.path.normpath(config.get("CouchPotato", "outputDirectory")) # C:\downloaded\movies | /path/to/downloaded/movies + # Torrent specific + useLink = config.get("Torrent", "useLink") # true | false + minSampleSize = int(config.get("Torrent", "minSampleSize")) # 200 + uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") # http://localhost:8090/gui/ + uTorrentUSR = config.get("Torrent", "uTorrentUSR") # mysecretusr + uTorrentPWD = config.get("Torrent", "uTorrentPWD") # mysecretpwr + compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') # .zip,.rar,.7z + mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') # .mkv,.avi,.divx + metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt + categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software + categories.append(movieCategory) + categories.append(tvCategory) # now have a list of all categories in use. + try: + inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) + except: + Logger.error("MAIN: There was a problem loading variables: Exiting") + sys.exit(-1) -def is_sample(filePath, inputName, minSampleSize): - # 200 MB in bytes - SIZE_CUTOFF = minSampleSize * 1024 * 1024 - # Ignore 'sample' in files unless 'sample' in Torrent Name - return ('sample' in filePath.lower()) and (not 'sample' in inputName) and (os.path.getsize(filePath) < SIZE_CUTOFF) + Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) + status = int(1) # We start as "failed" until we verify movie file in destination + root = int(0) + video = int(0) + video2 = int(0) + failed_link = int(0) + failed_extract = int(0) -def copy_link(source, target, useLink, outputDestination): - create_destination(outputDestination) - if useLink: - try: - Logger.info("COPYLINK: Linking %s to %s", source, target) - linktastic.link(source, target) - except: - if os.path.isfile(target): - Logger.info("COPYLINK: Something went wrong in linktastic.link, but the destination file was created") - else: - Logger.info("COPYLINK: Something went wrong in linktastic.link, copying instead") - Logger.debug("COPYLINK: Copying %s to %s", source, target) - shutil.copy(source, target) + inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure + if inputCategory == movieCategory: + outputDestination = os.path.normpath(os.path.join(movieDestination, inputName)) + elif inputCategory == tvCategory: + outputDestination = os.path.normpath(os.path.join(tvDestination, inputName)) else: - Logger.debug("Copying %s to %s", source, target) - shutil.copy(source, target) - return True + Logger.error("MAIN: Category of %s does not match either %s or %s: Exiting", inputCategory, movieCategory, tvCategory) + Logger.debug("MAIN: Future versions of this script might do something for Category: %s. Keep updating ;)", inputCategory) + sys.exit(-1) + Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) + if root == 1: + Logger.debug("MAIN: Looking for %s in filename", inputName) + elif root == 2: + Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") -def flatten(outputDestination): - Logger.info("FLATTEN: Flattening directory: %s", outputDestination) - for dirpath, dirnames, filenames in os.walk(outputDestination): # Flatten out the directory to make postprocessing easier - if dirpath == outputDestination: - continue # No need to try and move files in the root destination directory - for filename in filenames: - source = os.path.join(dirpath, filename) - target = os.path.join(outputDestination, filename) - try: - shutil.move(source, target) - except OSError: - Logger.error("FLATTEN: Could not flatten %s", source) - removeEmptyFolders(outputDestination) # Cleanup empty directories - - -def removeEmptyFolders(path): - Logger.info("REMOVER: Removing empty folders in: %s", path) - if not os.path.isdir(path): - return - - # Remove empty subfolders - files = os.listdir(path) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - removeEmptyFolders(fullpath) - - # If folder empty, delete it - files = os.listdir(path) - if len(files) == 0: - Logger.debug("REMOVER: Removing empty folder: %s", path) - os.rmdir(path) - -Logger.info("==========================") # Seperate old from new log -Logger.info("TorrentToMedia %s", VERSION) -config = ConfigParser.ConfigParser() -configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg") - -### TORRENT TO MEDIA ### -if not os.path.isfile(configFilename): - Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") - sys.exit(-1) - -Logger.info("MAIN: Loading config from %s", configFilename) -config.read(configFilename) - -clientAgent = config.get("Torrent", "clientAgent") - -try: - inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) -except: - Logger.error("MAIN: There was a problem loading variables: Exiting") - sys.exit(-1) - -#### Main routine starts here. - -Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) - -# Sick-Beard -tvCategory = config.get("SickBeard", "category") -tvDestination = os.path.normpath(config.get("SickBeard", "outputDirectory")) -# CouchPotatoServer -movieCategory = config.get("CouchPotato", "category") -movieDestination = os.path.normpath(config.get("CouchPotato", "outputDirectory")) -# Torrent specific -useLink = config.get("Torrent", "useLink") -minSampleSize = int(config.get("Torrent", "minSampleSize")) -uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") -uTorrentUSR = config.get("Torrent", "uTorrentUSR") -uTorrentPWD = config.get("Torrent", "uTorrentPWD") -compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') -mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') -metaContainer = (config.get("Torrent", "metaExtentions")).split(',') -categories = (config.get("Torrent", "categories")).split(',') -categories.append(movieCategory) -categories.append(tvCategory) # now have a list of all categories in use. - -status = int(1) # We start as "failed" until we verify movie file in destination -root = int(0) -video = int(0) -video2 = int(0) -failed_link = int(0) -failed_extract = int(0) - -inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure -if inputCategory == movieCategory: - outputDestination = os.path.normpath(os.path.join(movieDestination, inputName)) -elif inputCategory == tvCategory: - outputDestination = os.path.normpath(os.path.join(tvDestination, inputName)) -else: - Logger.error("MAIN: Category of %s does not match either %s or %s: Exiting", inputCategory, movieCategory, tvCategory) - Logger.debug("MAIN: Future versions of this script might do something for Category: %s. Keep updating ;)", inputCategory) - sys.exit(-1) - -Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) -if root == 1: - Logger.debug("MAIN: Looking for %s in filename", inputName) -elif root == 2: - Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") - -now = datetime.datetime.now() -for dirpath, dirnames, filenames in os.walk(inputDirectory): - for file in filenames: - if root == 1: - if (inputName in file) or (os.path.splitext(file)[0] in inputName): - pass # This file does match the Torrent name - Logger.debug("Found file %s that matches Torrent Name %s", file, inputName) - else: - continue # This file does not match the Torrent name, skip it - if root == 2: - mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(dirpath, file))) - ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, file))) - if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): - pass # This file does match the date time criteria - Logger.debug("Found file %s with date modifed/created less than 5 minutes ago.", file) - else: - continue # This file has not been recently moved or created, skip it - filePath = os.path.join(dirpath, file) - fileExtention = os.path.splitext(file)[1] - if fileExtention in mediaContainer: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize): # Ignore samples - Logger.info("MAIN: Ignoring sample file: %s ", filePath) - continue - else: - video = video + 1 + now = datetime.datetime.now() + for dirpath, dirnames, filenames in os.walk(inputDirectory): + for file in filenames: + if root == 1: + if (inputName in file) or (os.path.splitext(file)[0] in inputName): + pass # This file does match the Torrent name + Logger.debug("Found file %s that matches Torrent Name %s", file, inputName) + else: + continue # This file does not match the Torrent name, skip it + if root == 2: + mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(dirpath, file))) + ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, file))) + if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): + pass # This file does match the date time criteria + Logger.debug("Found file %s with date modifed/created less than 5 minutes ago.", file) + else: + continue # This file has not been recently moved or created, skip it + filePath = os.path.join(dirpath, file) + fileExtention = os.path.splitext(file)[1] + if fileExtention in mediaContainer: # If the file is a video file + if is_sample(filePath, inputName, minSampleSize): # Ignore samples + Logger.info("MAIN: Ignoring sample file: %s ", filePath) + continue + else: + video = video + 1 + source = filePath + target = os.path.join(outputDestination, file) + Logger.info("MAIN: Found video file %s in %s", fileExtention, filePath) + state = copy_link(source, target, useLink, outputDestination) + if state == False: + Logger.error("MAIN: Failed to link file %s", file) + failed_link = 1 + elif fileExtention in metaContainer: source = filePath target = os.path.join(outputDestination, file) - Logger.info("MAIN: Found video file %s in %s", fileExtention, filePath) + Logger.info("MAIN: Found metadata file %s for file %s", fileExtention, filePath) state = copy_link(source, target, useLink, outputDestination) if state == False: Logger.error("MAIN: Failed to link file %s", file) failed_link = 1 - elif fileExtention in metaContainer: - source = filePath - target = os.path.join(outputDestination, file) - Logger.info("MAIN: Found metadata file %s for file %s", fileExtention, filePath) - state = copy_link(source, target, useLink, outputDestination) - if state == False: - Logger.error("MAIN: Failed to link file %s", file) - failed_link = 1 - elif fileExtention in compressedContainer: - Logger.info("MAIN: Found compressed archive %s for file %s", fileExtention, filePath) - source = filePath - target = os.path.join(outputDestination, file) - try: - extractor.extract(dirpath, file, outputDestination) - except: - Logger.warn("Extraction failed for %s", file) - else: - Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtention, filePath) - continue -flatten(outputDestination) - -# Now check if movie files exist in destination: -for dirpath, dirnames, filenames in os.walk(outputDestination): - for file in filenames: - filePath = os.path.join(dirpath, file) - fileExtention = os.path.splitext(file)[1] - if fileExtention in mediaContainer: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize): - Logger.debug("Removing sample file: %s", filePath) - os.unlink(filePath) # remove samples + elif fileExtention in compressedContainer: + Logger.info("MAIN: Found compressed archive %s for file %s", fileExtention, filePath) + source = filePath + target = os.path.join(outputDestination, file) + try: + extractor.extract(dirpath, file, outputDestination) + except: + Logger.warn("Extraction failed for %s", file) else: - videofile = filePath - video2 = video2 + 1 -if video2 >= video and video2 > 0: # Check that all video files were moved - status = 0 + Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtention, filePath) + continue + flatten(outputDestination) -if status == 0: #### Maybe we should move this to a more appropriate place? - Logger.info("MAIN: Successful run") - Logger.debug("MAIN: Calling autoProcess script for successful download.") -elif failed_extract == 1 and failed_link == 0: # failed to extract files only. - Logger.info("MAIN: Failed to extract a compressed archive") - Logger.debug("MAIN: Assume this to be password protected file.") - Logger.debug("MAIN: Calling autoProcess script for failed download.") -else: - Logger.error("MAIN: Something failed! Please check logs. Exiting") - sys.exit(-1) + # Now check if movie files exist in destination: + for dirpath, dirnames, filenames in os.walk(outputDestination): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileExtention = os.path.splitext(file)[1] + if fileExtention in mediaContainer: # If the file is a video file + if is_sample(filePath, inputName, minSampleSize): + Logger.debug("Removing sample file: %s", filePath) + os.unlink(filePath) # remove samples + else: + videofile = filePath + video2 = video2 + 1 + if video2 >= video and video2 > 0: # Check that all video files were moved + status = 0 -# Hardlink solution with uTorrent -if inputHash and useLink: - try: - Logger.debug("MAIN: Connecting to uTorrent: %s", uTorrentWEBui) - utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) - except: - Logger.error("MAIN: Failed to connect to uTorrent") - Logger.debug("MAIN: Stoping torrent %s in uTorrent while processing", videofile) - utorrentClass.stop(inputHash) - time.sleep(5) # Give uTorrent some time to catch up with the change + if status == 0: #### Maybe we should move this to a more appropriate place? + Logger.info("MAIN: Successful run") + Logger.debug("MAIN: Calling autoProcess script for successful download.") + elif failed_extract == 1 and failed_link == 0: # failed to extract files only. + Logger.info("MAIN: Failed to extract a compressed archive") + Logger.debug("MAIN: Assume this to be password protected file.") + Logger.debug("MAIN: Calling autoProcess script for failed download.") + else: + Logger.error("MAIN: Something failed! Please check logs. Exiting") + sys.exit(-1) -# Now we pass off to CouchPotato or Sick-Beard -if inputCategory == movieCategory: - Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", videofile) # can we use logger while logfile open? - autoProcessMovie.process(outputDestination, inputName, status) -elif inputCategory == tvCategory: - Logger.info("MAIN: Calling Sick-Beard to post-process: %s", videofile) # can we use logger while logfile open? - autoProcessTV.processEpisode(outputDestination, inputName, status) + # Hardlink solution with uTorrent + if inputHash and useLink: + try: + Logger.debug("MAIN: Connecting to uTorrent: %s", uTorrentWEBui) + utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) + except: + Logger.error("MAIN: Failed to connect to uTorrent") -# Check if the file still exists in the post-process directory -now = datetime.datetime.now() # set time for timeout -while os.path.exists(videofile): # while this file is still here, CPS hasn't finished renaming - if (datetime.datetime.now() - now) > datetime.timedelta(minutes=3): # note; minimum 1 minute delay in autoProcessMovie - Logger.info("MAIN: The file %s has not been moved after 3 minutes.", videofile) - break - time.sleep(10) #Just stop this looping infinitely and hogging resources for 3 minutes ;) -else: # CPS (and SickBeard) have finished. We can now resume seeding. - Logger.info("MAIN: Post-process appears to have succeeded for: %s", videofile) + Logger.debug("MAIN: Stoping torrent %s in uTorrent while processing", videofile) + utorrentClass.stop(inputHash) + time.sleep(5) # Give uTorrent some time to catch up with the change -# Hardlink solution with uTorrent -if inputHash and useLink: - Logger.debug("MAIN: Starting torrent %s in uTorrent", inputName) - utorrentClass.start(inputHash) + # Now we pass off to CouchPotato or Sick-Beard + if inputCategory == movieCategory: + Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", videofile) # can we use logger while logfile open? + autoProcessMovie.process(outputDestination, inputName, status) + elif inputCategory == tvCategory: + Logger.info("MAIN: Calling Sick-Beard to post-process: %s", videofile) # can we use logger while logfile open? + autoProcessTV.processEpisode(outputDestination, inputName, status) + + # Check if the file still exists in the post-process directory + now = datetime.datetime.now() # set time for timeout + while os.path.exists(videofile): # while this file is still here, CPS hasn't finished renaming + if (datetime.datetime.now() - now) > datetime.timedelta(minutes=3): # note; minimum 1 minute delay in autoProcessMovie + Logger.info("MAIN: The file %s has not been moved after 3 minutes.", videofile) + break + time.sleep(10) #Just stop this looping infinitely and hogging resources for 3 minutes ;) + else: # CPS (and SickBeard) have finished. We can now resume seeding. + Logger.info("MAIN: Post-process appears to have succeeded for: %s", videofile) + + # Hardlink solution with uTorrent + if inputHash and useLink: + Logger.debug("MAIN: Starting torrent %s in uTorrent", inputName) + utorrentClass.start(inputHash) + +if __name__ == "__main__": + main() From 42ba5b11e1b8bd06f2c7a7bb013402047f6336b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 11:17:28 +0100 Subject: [PATCH 03/27] more changes, cleanup --- TorrentToMedia.py | 95 ++++++++++++++++++++++++----------------------- nzbToMediaUtil.py | 1 + 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 91c92e61..31bcc6e5 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -19,52 +19,7 @@ from nzbToMediaEnv import * from nzbToMediaUtil import * from utorrent.client import UTorrentClient -# Logging -nzbtomedia_configure_logging(os.path.dirname(sys.argv[0])) -Logger = logging.getLogger(__name__) - -def main() - - Logger.info("==========================") # Seperate old from new log - Logger.info("TorrentToMedia %s", VERSION) - config = ConfigParser.ConfigParser() - configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg") - - if not os.path.isfile(configFilename): - Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") - sys.exit(-1) - - Logger.info("MAIN: Loading config from %s", configFilename) - config.read(configFilename) - - # EXAMPLE VALUES: - clientAgent = config.get("Torrent", "clientAgent") # utorrent | deluge | transmission | other - # Sick-Beard - tvCategory = config.get("SickBeard", "category") # tv - tvDestination = os.path.normpath(config.get("SickBeard", "outputDirectory")) # C:\downloaded\tv | /path/to/downloaded/tv - # CouchPotatoServer - movieCategory = config.get("CouchPotato", "category") # movie - movieDestination = os.path.normpath(config.get("CouchPotato", "outputDirectory")) # C:\downloaded\movies | /path/to/downloaded/movies - # Torrent specific - useLink = config.get("Torrent", "useLink") # true | false - minSampleSize = int(config.get("Torrent", "minSampleSize")) # 200 - uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") # http://localhost:8090/gui/ - uTorrentUSR = config.get("Torrent", "uTorrentUSR") # mysecretusr - uTorrentPWD = config.get("Torrent", "uTorrentPWD") # mysecretpwr - compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') # .zip,.rar,.7z - mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') # .mkv,.avi,.divx - metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt - categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software - categories.append(movieCategory) - categories.append(tvCategory) # now have a list of all categories in use. - - try: - inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) - except: - Logger.error("MAIN: There was a problem loading variables: Exiting") - sys.exit(-1) - - Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) +def main(inputDirectory, inputName, inputCategory, inputHash) status = int(1) # We start as "failed" until we verify movie file in destination root = int(0) @@ -73,6 +28,8 @@ def main() failed_link = int(0) failed_extract = int(0) + Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) + inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure if inputCategory == movieCategory: outputDestination = os.path.normpath(os.path.join(movieDestination, inputName)) @@ -204,4 +161,48 @@ def main() utorrentClass.start(inputHash) if __name__ == "__main__": - main() + + # Logging + nzbtomedia_configure_logging(os.path.dirname(sys.argv[0])) + Logger = logging.getLogger(__name__) + + Logger.info("====================") # Seperate old from new log + Logger.info("TorrentToMedia %s", VERSION) + config = ConfigParser.ConfigParser() + configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg") + + if not os.path.isfile(configFilename): + Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") + sys.exit(-1) + + # CONFIG FILE + Logger.info("MAIN: Loading config from %s", configFilename) + config.read(configFilename) + # EXAMPLE VALUES: + clientAgent = config.get("Torrent", "clientAgent") # utorrent | deluge | transmission | other + # SICK-BEARD + tvCategory = config.get("SickBeard", "category") # tv + tvDestination = os.path.normpath(config.get("SickBeard", "outputDirectory")) # C:\downloaded\tv | /path/to/downloaded/tv + # COUCHPOTATOSERVER + movieCategory = config.get("CouchPotato", "category") # movie + movieDestination = os.path.normpath(config.get("CouchPotato", "outputDirectory")) # C:\downloaded\movies | /path/to/downloaded/movies + # TORRENTS + useLink = config.get("Torrent", "useLink") # true | false + minSampleSize = int(config.get("Torrent", "minSampleSize")) # 200 (in MB) + uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") # http://localhost:8090/gui/ + uTorrentUSR = config.get("Torrent", "uTorrentUSR") # mysecretusr + uTorrentPWD = config.get("Torrent", "uTorrentPWD") # mysecretpwr + compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') # .zip,.rar,.7z + mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') # .mkv,.avi,.divx + metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt + categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software + categories.append(movieCategory) + categories.append(tvCategory) # now have a list of all categories in use. + + try: + inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) + except Exception as e: + Logger.error("MAIN: There was a problem loading variables: %s", e) + sys.exit(-1) + + main(inputDirectory, inputName, inputCategory, inputHash) diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index dd743721..92298288 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -3,6 +3,7 @@ import logging.config import os import sys +import linktastic.linktastic as linktastic Logger = logging.getLogger(__name__) From e322c81392c1ad204c4c4ffe091786c51c6536e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 12:49:22 +0100 Subject: [PATCH 04/27] more cleanup, fixes --- TorrentToMedia.py | 84 +++++++++++++++++++++++------------------- extractor/extractor.py | 3 +- nzbToMediaUtil.py | 16 ++++---- 3 files changed, 55 insertions(+), 48 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 31bcc6e5..ad8d2949 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -21,12 +21,10 @@ from utorrent.client import UTorrentClient def main(inputDirectory, inputName, inputCategory, inputHash) - status = int(1) # We start as "failed" until we verify movie file in destination + status = int(1) # 1 = failed | 0 = success root = int(0) video = int(0) video2 = int(0) - failed_link = int(0) - failed_extract = int(0) Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) @@ -40,22 +38,26 @@ def main(inputDirectory, inputName, inputCategory, inputHash) Logger.debug("MAIN: Future versions of this script might do something for Category: %s. Keep updating ;)", inputCategory) sys.exit(-1) - Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) - if root == 1: - Logger.debug("MAIN: Looking for %s in filename", inputName) - elif root == 2: - Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") + Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) now = datetime.datetime.now() for dirpath, dirnames, filenames in os.walk(inputDirectory): for file in filenames: + + filePath = os.path.join(dirpath, file) + fileExtention = os.path.splitext(file)[1] + targetDirectory = os.path.join(outputDestination, file) + if root == 1: + Logger.debug("MAIN: Looking for %s in filename", inputName) if (inputName in file) or (os.path.splitext(file)[0] in inputName): pass # This file does match the Torrent name Logger.debug("Found file %s that matches Torrent Name %s", file, inputName) else: continue # This file does not match the Torrent name, skip it + if root == 2: + Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(dirpath, file))) ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, file))) if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): @@ -63,37 +65,39 @@ def main(inputDirectory, inputName, inputCategory, inputHash) Logger.debug("Found file %s with date modifed/created less than 5 minutes ago.", file) else: continue # This file has not been recently moved or created, skip it - filePath = os.path.join(dirpath, file) - fileExtention = os.path.splitext(file)[1] + if fileExtention in mediaContainer: # If the file is a video file if is_sample(filePath, inputName, minSampleSize): # Ignore samples Logger.info("MAIN: Ignoring sample file: %s ", filePath) continue else: video = video + 1 - source = filePath - target = os.path.join(outputDestination, file) Logger.info("MAIN: Found video file %s in %s", fileExtention, filePath) - state = copy_link(source, target, useLink, outputDestination) - if state == False: + try: + copy_link(filePath, targetDirectory, useLink, outputDestination) + except Exception as e: Logger.error("MAIN: Failed to link file %s", file) - failed_link = 1 + Logger.debug e + linkFailed = True + elif fileExtention in metaContainer: - source = filePath - target = os.path.join(outputDestination, file) Logger.info("MAIN: Found metadata file %s for file %s", fileExtention, filePath) - state = copy_link(source, target, useLink, outputDestination) - if state == False: + try: + copy_link(filePath, targetDirectory, useLink, outputDestination) + except Exception as e: Logger.error("MAIN: Failed to link file %s", file) - failed_link = 1 + Logger.debug e + linkFailed = True + elif fileExtention in compressedContainer: Logger.info("MAIN: Found compressed archive %s for file %s", fileExtention, filePath) - source = filePath - target = os.path.join(outputDestination, file) try: - extractor.extract(dirpath, file, outputDestination) - except: - Logger.warn("Extraction failed for %s", file) + extractor.extract(filePath, outputDestination) + except Exception as e: + Logger.warn("Extraction failed for: %s", file) + Logger.debug e + extractFailed = True + else: Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtention, filePath) continue @@ -117,7 +121,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) if status == 0: #### Maybe we should move this to a more appropriate place? Logger.info("MAIN: Successful run") Logger.debug("MAIN: Calling autoProcess script for successful download.") - elif failed_extract == 1 and failed_link == 0: # failed to extract files only. + elif extractFailed and linkFailed == False: # failed to extract files only. Logger.info("MAIN: Failed to extract a compressed archive") Logger.debug("MAIN: Assume this to be password protected file.") Logger.debug("MAIN: Calling autoProcess script for failed download.") @@ -125,40 +129,44 @@ def main(inputDirectory, inputName, inputCategory, inputHash) Logger.error("MAIN: Something failed! Please check logs. Exiting") sys.exit(-1) - # Hardlink solution with uTorrent - if inputHash and useLink: + #### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission + if inputHash and useLink and clientAgent == 'utorrent': try: Logger.debug("MAIN: Connecting to uTorrent: %s", uTorrentWEBui) utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) - except: - Logger.error("MAIN: Failed to connect to uTorrent") + except Exception as e: + Logger.error("MAIN: Failed to connect to uTorrent: %s", e) - Logger.debug("MAIN: Stoping torrent %s in uTorrent while processing", videofile) + Logger.debug("MAIN: Stoping torrent %s in uTorrent while processing", inputName) utorrentClass.stop(inputHash) time.sleep(5) # Give uTorrent some time to catch up with the change + ##### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission # Now we pass off to CouchPotato or Sick-Beard if inputCategory == movieCategory: - Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", videofile) # can we use logger while logfile open? + Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName) # can we use logger while logfile open? autoProcessMovie.process(outputDestination, inputName, status) elif inputCategory == tvCategory: - Logger.info("MAIN: Calling Sick-Beard to post-process: %s", videofile) # can we use logger while logfile open? + Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName) # can we use logger while logfile open? autoProcessTV.processEpisode(outputDestination, inputName, status) # Check if the file still exists in the post-process directory now = datetime.datetime.now() # set time for timeout - while os.path.exists(videofile): # while this file is still here, CPS hasn't finished renaming + while os.path.exists(outputDestination): # while this directory is still here, CPS hasn't finished renaming if (datetime.datetime.now() - now) > datetime.timedelta(minutes=3): # note; minimum 1 minute delay in autoProcessMovie - Logger.info("MAIN: The file %s has not been moved after 3 minutes.", videofile) + Logger.info("MAIN: The directory %s has not been moved after 3 minutes.", outputDestination) break time.sleep(10) #Just stop this looping infinitely and hogging resources for 3 minutes ;) else: # CPS (and SickBeard) have finished. We can now resume seeding. - Logger.info("MAIN: Post-process appears to have succeeded for: %s", videofile) + Logger.info("MAIN: Post-process appears to have succeeded for: %s", inputName) - # Hardlink solution with uTorrent - if inputHash and useLink: + #### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission + if inputHash and useLink and clientAgent == 'utorrent': Logger.debug("MAIN: Starting torrent %s in uTorrent", inputName) utorrentClass.start(inputHash) + #### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission + + Logger.info("MAIN: All done.") if __name__ == "__main__": diff --git a/extractor/extractor.py b/extractor/extractor.py index 9835ca7f..4755d988 100644 --- a/extractor/extractor.py +++ b/extractor/extractor.py @@ -36,7 +36,7 @@ def which(program): return None -def extract(dirpath, file, outputDestination): +def extract(filePath, outputDestination): # Using Windows if os.name == 'nt': if os_platform() == 'AMD64': @@ -89,7 +89,6 @@ def extract(dirpath, file, outputDestination): Logger.warn("EXTRACTOR: No archive extracting programs found, plugin will be disabled") ext = os.path.splitext(file) - filePath = os.path.join(dirpath, file) if ext[1] in (".gz", ".bz2", ".lzma"): # Check if this is a tar if os.path.splitext(ext[0])[1] == ".tar": diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index 92298288..12509405 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -132,22 +132,22 @@ def is_sample(filePath, inputName, minSampleSize): return ('sample' in filePath.lower()) and (not 'sample' in inputName) and (os.path.getsize(filePath) < SIZE_CUTOFF) -def copy_link(source, target, useLink, outputDestination): +def copy_link(filePath, targetDirectory, useLink, outputDestination): create_destination(outputDestination) if useLink: try: - Logger.info("COPYLINK: Linking %s to %s", source, target) - linktastic.link(source, target) + Logger.info("COPYLINK: Linking %s to %s", filePath, targetDirectory) + linktastic.link(filePath, targetDirectory) except: - if os.path.isfile(target): + if os.path.isfile(targetDirectory): Logger.info("COPYLINK: Something went wrong in linktastic.link, but the destination file was created") else: Logger.info("COPYLINK: Something went wrong in linktastic.link, copying instead") - Logger.debug("COPYLINK: Copying %s to %s", source, target) - shutil.copy(source, target) + Logger.debug("COPYLINK: Copying %s to %s", filePath, targetDirectory) + shutil.copy(filePath, targetDirectory) else: - Logger.debug("Copying %s to %s", source, target) - shutil.copy(source, target) + Logger.debug("Copying %s to %s", filePath, targetDirectory) + shutil.copy(filePath, targetDirectory) return True From 182841ecbb354cef1ae2667d5324749be08b43e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 13:11:51 +0100 Subject: [PATCH 05/27] match all categorys --- TorrentToMedia.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index ad8d2949..c272fc95 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -29,14 +29,14 @@ def main(inputDirectory, inputName, inputCategory, inputHash) Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure - if inputCategory == movieCategory: - outputDestination = os.path.normpath(os.path.join(movieDestination, inputName)) - elif inputCategory == tvCategory: - outputDestination = os.path.normpath(os.path.join(tvDestination, inputName)) - else: - Logger.error("MAIN: Category of %s does not match either %s or %s: Exiting", inputCategory, movieCategory, tvCategory) - Logger.debug("MAIN: Future versions of this script might do something for Category: %s. Keep updating ;)", inputCategory) - sys.exit(-1) + + for category in categories: + if category == inputCategory: + outputDestination = os.path.normpath(os.path.join(outputDestination, category, inputName)) + Logger.info("MAIN: Output directory set to: %s", outputDestination) + else: + Logger.error("MAIN: Could not match input category with defined categories, Exiting") + sys.exit(-1) Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) @@ -76,7 +76,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) try: copy_link(filePath, targetDirectory, useLink, outputDestination) except Exception as e: - Logger.error("MAIN: Failed to link file %s", file) + Logger.error("MAIN: Failed to link file: %s", file) Logger.debug e linkFailed = True @@ -85,7 +85,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) try: copy_link(filePath, targetDirectory, useLink, outputDestination) except Exception as e: - Logger.error("MAIN: Failed to link file %s", file) + Logger.error("MAIN: Failed to link file: %s", file) Logger.debug e linkFailed = True @@ -94,7 +94,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) try: extractor.extract(filePath, outputDestination) except Exception as e: - Logger.warn("Extraction failed for: %s", file) + Logger.warn("MAIN: Extraction failed for: %s", file) Logger.debug e extractFailed = True @@ -110,7 +110,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) fileExtention = os.path.splitext(file)[1] if fileExtention in mediaContainer: # If the file is a video file if is_sample(filePath, inputName, minSampleSize): - Logger.debug("Removing sample file: %s", filePath) + Logger.debug("MAIN: Removing sample file: %s", filePath) os.unlink(filePath) # remove samples else: videofile = filePath @@ -121,10 +121,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash) if status == 0: #### Maybe we should move this to a more appropriate place? Logger.info("MAIN: Successful run") Logger.debug("MAIN: Calling autoProcess script for successful download.") - elif extractFailed and linkFailed == False: # failed to extract files only. - Logger.info("MAIN: Failed to extract a compressed archive") - Logger.debug("MAIN: Assume this to be password protected file.") - Logger.debug("MAIN: Calling autoProcess script for failed download.") else: Logger.error("MAIN: Something failed! Please check logs. Exiting") sys.exit(-1) From 9fa6a86f1a09b9e68b2531f5c0d5c5b2e13ff61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 13:18:19 +0100 Subject: [PATCH 06/27] fixes to category search --- TorrentToMedia.py | 18 ++++++------------ autoProcessMedia.cfg.sample | 8 +++----- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index c272fc95..ae8f4090 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -184,29 +184,23 @@ if __name__ == "__main__": config.read(configFilename) # EXAMPLE VALUES: clientAgent = config.get("Torrent", "clientAgent") # utorrent | deluge | transmission | other - # SICK-BEARD - tvCategory = config.get("SickBeard", "category") # tv - tvDestination = os.path.normpath(config.get("SickBeard", "outputDirectory")) # C:\downloaded\tv | /path/to/downloaded/tv - # COUCHPOTATOSERVER - movieCategory = config.get("CouchPotato", "category") # movie - movieDestination = os.path.normpath(config.get("CouchPotato", "outputDirectory")) # C:\downloaded\movies | /path/to/downloaded/movies - # TORRENTS useLink = config.get("Torrent", "useLink") # true | false minSampleSize = int(config.get("Torrent", "minSampleSize")) # 200 (in MB) + outputDirectory = config.get("Torrent", "outputDirectory") # /abs/path/to/complete/ + categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software + uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") # http://localhost:8090/gui/ uTorrentUSR = config.get("Torrent", "uTorrentUSR") # mysecretusr uTorrentPWD = config.get("Torrent", "uTorrentPWD") # mysecretpwr + compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') # .zip,.rar,.7z mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') # .mkv,.avi,.divx metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt - categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software - categories.append(movieCategory) - categories.append(tvCategory) # now have a list of all categories in use. - + try: inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) except Exception as e: Logger.error("MAIN: There was a problem loading variables: %s", e) sys.exit(-1) - main(inputDirectory, inputName, inputCategory, inputHash) + main(inputDirectory, inputName, inputCategory, inputHash) \ No newline at end of file diff --git a/autoProcessMedia.cfg.sample b/autoProcessMedia.cfg.sample index e40dd5d3..8a014fe1 100644 --- a/autoProcessMedia.cfg.sample +++ b/autoProcessMedia.cfg.sample @@ -1,6 +1,4 @@ [CouchPotato] -category = movie -outputDirectory = /abs/path/to/complete/movies apikey = host = localhost port = 5050 @@ -15,8 +13,6 @@ delete_failed = 0 [SickBeard] -category = tv -outputDirectory = /abs/path/to/complete/tv host=localhost port=8081 username= @@ -35,8 +31,10 @@ clientAgent = other useLink = false ###### minSampleSize - Minimum required size to consider a file not an sample file (in MB, eg 200mb) minSampleSize = 200 +###### outputDirectory - Default output directory (categories will be appended as sub directory to outputDirectory) +outputDirectory = /abs/path/to/complete/ ###### **insert descriptive comment for categories here** :-) -categories = music,music_videos,pictures,software +categories = movie,tv,music,music_videos,pictures,software ###### uTorrent Hardlink solution (You must edit this if your using TorrentToMedia.py with uTorrent) uTorrentWEBui = http://localhost:8090/gui/ uTorrentUSR = your username From 51a0366906fe4c45256df277bc5fec0e9c88ca44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 13:24:58 +0100 Subject: [PATCH 07/27] my bad :) --- TorrentToMedia.py | 8 ++++++-- autoProcessMedia.cfg.sample | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index ae8f4090..75ef44ef 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -139,10 +139,10 @@ def main(inputDirectory, inputName, inputCategory, inputHash) ##### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission # Now we pass off to CouchPotato or Sick-Beard - if inputCategory == movieCategory: + if inputCategory == cpsCategory: Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName) # can we use logger while logfile open? autoProcessMovie.process(outputDestination, inputName, status) - elif inputCategory == tvCategory: + elif inputCategory == sbCategory: Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName) # can we use logger while logfile open? autoProcessTV.processEpisode(outputDestination, inputName, status) @@ -196,6 +196,10 @@ if __name__ == "__main__": compressedContainer = (config.get("Torrent", "compressedExtentions")).split(',') # .zip,.rar,.7z mediaContainer = (config.get("Torrent", "mediaExtentions")).split(',') # .mkv,.avi,.divx metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt + + cpsCategory = config.get("CouchPotato", "cpsCategory") # movie + sbCategory = config.get("SickBeard", "tvCategory") # tv + categories.append(cpsCategory, sbCategory) try: inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) diff --git a/autoProcessMedia.cfg.sample b/autoProcessMedia.cfg.sample index 8a014fe1..cee0d47a 100644 --- a/autoProcessMedia.cfg.sample +++ b/autoProcessMedia.cfg.sample @@ -1,4 +1,6 @@ [CouchPotato] +#### cpsCategory - category that gets called for post-processing with CPS +cpsCategory = movie apikey = host = localhost port = 5050 @@ -13,6 +15,8 @@ delete_failed = 0 [SickBeard] +#### sbCategory - category that gets called for post-processing with CPS +sbCategory = tv host=localhost port=8081 username= @@ -23,9 +27,9 @@ ssl=0 watch_dir= failed_fork=0 + [Torrent] -###### Set to whatever torrent client you use. -###### Supported values: utorrent, transmission, deluge, other +###### clientAgent - Supported clients: utorrent, transmission, deluge, other clientAgent = other ###### useLink - Set to true or false depending on if you want to use hardlinks useLink = false @@ -34,7 +38,7 @@ minSampleSize = 200 ###### outputDirectory - Default output directory (categories will be appended as sub directory to outputDirectory) outputDirectory = /abs/path/to/complete/ ###### **insert descriptive comment for categories here** :-) -categories = movie,tv,music,music_videos,pictures,software +categories = music,music_videos,pictures,software ###### uTorrent Hardlink solution (You must edit this if your using TorrentToMedia.py with uTorrent) uTorrentWEBui = http://localhost:8090/gui/ uTorrentUSR = your username From b01c6247ddf5efca6f01613ef3528b69aad3c57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sat, 2 Mar 2013 22:20:35 +0100 Subject: [PATCH 08/27] cleanup, unnecessary variables --- TorrentToMedia.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 75ef44ef..abdc4583 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -34,6 +34,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash) if category == inputCategory: outputDestination = os.path.normpath(os.path.join(outputDestination, category, inputName)) Logger.info("MAIN: Output directory set to: %s", outputDestination) + pass else: Logger.error("MAIN: Could not match input category with defined categories, Exiting") sys.exit(-1) @@ -78,8 +79,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash) except Exception as e: Logger.error("MAIN: Failed to link file: %s", file) Logger.debug e - linkFailed = True - elif fileExtention in metaContainer: Logger.info("MAIN: Found metadata file %s for file %s", fileExtention, filePath) try: @@ -87,8 +86,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash) except Exception as e: Logger.error("MAIN: Failed to link file: %s", file) Logger.debug e - linkFailed = True - elif fileExtention in compressedContainer: Logger.info("MAIN: Found compressed archive %s for file %s", fileExtention, filePath) try: @@ -96,8 +93,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash) except Exception as e: Logger.warn("MAIN: Extraction failed for: %s", file) Logger.debug e - extractFailed = True - else: Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtention, filePath) continue @@ -113,7 +108,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash) Logger.debug("MAIN: Removing sample file: %s", filePath) os.unlink(filePath) # remove samples else: - videofile = filePath video2 = video2 + 1 if video2 >= video and video2 > 0: # Check that all video files were moved status = 0 From fb7d1529cc5637e6b9b4fba0d051c2e6520a47cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 3 Mar 2013 12:05:20 +0100 Subject: [PATCH 09/27] minor fix and added license for 7zip --- TorrentToMedia.py | 2 +- extractor/bin/x64/license.txt | 56 +++++++++++++++++++++++++++++++++++ extractor/bin/x86/license.txt | 56 +++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 extractor/bin/x64/license.txt create mode 100644 extractor/bin/x86/license.txt diff --git a/TorrentToMedia.py b/TorrentToMedia.py index abdc4583..d72a3385 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -19,7 +19,7 @@ from nzbToMediaEnv import * from nzbToMediaUtil import * from utorrent.client import UTorrentClient -def main(inputDirectory, inputName, inputCategory, inputHash) +def main(inputDirectory, inputName, inputCategory, inputHash): status = int(1) # 1 = failed | 0 = success root = int(0) diff --git a/extractor/bin/x64/license.txt b/extractor/bin/x64/license.txt new file mode 100644 index 00000000..0be9890a --- /dev/null +++ b/extractor/bin/x64/license.txt @@ -0,0 +1,56 @@ + 7-Zip + ~~~~~ + License for use and distribution + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + + Licenses for files are: + + 1) 7z.dll: GNU LGPL + unRAR restriction + 2) All other files: GNU LGPL + + The GNU LGPL + unRAR restriction means that you must follow both + GNU LGPL rules and unRAR restriction rules. + + + Note: + You can use 7-Zip on any computer, including a computer in a commercial + organization. You don't need to register or pay for 7-Zip. + + + GNU LGPL information + -------------------- + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You can receive a copy of the GNU Lesser General Public License from + http://www.gnu.org/ + + + unRAR restriction + ----------------- + + The decompression engine for RAR archives was developed using source + code of unRAR program. + All copyrights to original unRAR code are owned by Alexander Roshal. + + The license for original unRAR code has the following restriction: + + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form + or as a part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + + -- + Igor Pavlov \ No newline at end of file diff --git a/extractor/bin/x86/license.txt b/extractor/bin/x86/license.txt new file mode 100644 index 00000000..0be9890a --- /dev/null +++ b/extractor/bin/x86/license.txt @@ -0,0 +1,56 @@ + 7-Zip + ~~~~~ + License for use and distribution + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + + Licenses for files are: + + 1) 7z.dll: GNU LGPL + unRAR restriction + 2) All other files: GNU LGPL + + The GNU LGPL + unRAR restriction means that you must follow both + GNU LGPL rules and unRAR restriction rules. + + + Note: + You can use 7-Zip on any computer, including a computer in a commercial + organization. You don't need to register or pay for 7-Zip. + + + GNU LGPL information + -------------------- + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You can receive a copy of the GNU Lesser General Public License from + http://www.gnu.org/ + + + unRAR restriction + ----------------- + + The decompression engine for RAR archives was developed using source + code of unRAR program. + All copyrights to original unRAR code are owned by Alexander Roshal. + + The license for original unRAR code has the following restriction: + + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form + or as a part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + + -- + Igor Pavlov \ No newline at end of file From 6a3bfaf649e7d3b47643027505a9e559f7127c31 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 14:31:42 -0800 Subject: [PATCH 10/27] separate log entires --- nzbToSickBeard.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index e7925693..d8a17b95 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -32,6 +32,7 @@ from nzbToMediaUtil import * nzbtomedia_configure_logging(os.path.dirname(sys.argv[0])) Logger = logging.getLogger(__name__) +Logger.info("====================") # Seperate old from new log Logger.info("nzbToSickBeard %s", VERSION) # SABnzbd From 5e8d5f9fc8850980fb1ff468af3191aecb23e0dc Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 14:32:04 -0800 Subject: [PATCH 11/27] separate log entires --- nzbToCouchPotato.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index c6b5c3cb..cb538c74 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -10,6 +10,7 @@ from nzbToMediaUtil import * nzbtomedia_configure_logging(os.path.dirname(sys.argv[0])) Logger = logging.getLogger(__name__) +Logger.info("====================") # Seperate old from new log Logger.info("nzbToCouchPotato %s", VERSION) # SABnzbd From 2335426bda911849343cf65a402d7eca71993fc9 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 17:39:48 -0800 Subject: [PATCH 12/27] added check for CPS renamer complete. removed sys.exit now use return 1 for failure (so uTorrent can still resume seeding) and return 0 for success --- autoProcessMovie.py | 142 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 33 deletions(-) diff --git a/autoProcessMovie.py b/autoProcessMovie.py index b745d00c..54112a52 100644 --- a/autoProcessMovie.py +++ b/autoProcessMovie.py @@ -30,6 +30,73 @@ class AuthURLOpener(urllib.FancyURLopener): self.numTries = 0 return urllib.FancyURLopener.open(self, url) +def get_imdb(nzbName1, dirName): + + a=nzbName1.find('.cp(')+4 #search for .cptt( in nzbName + b=nzbName1[a:].find(')')+a + imdbid=nzbName1[a:b] + + if imdbid: + Logger.info("Found movie id %s in name", imdbid) + return imdbid + + a=dirName.find('.cp(')+4 #search for .cptt( in dirname + b=dirName[a:].find(')')+a + imdbid=dirName[a:b] + + if imdbid: + Logger.info("Found movie id %s in directory", imdbid) + return imdbid + else: + Logger.info("Could not find movie id in directory or name", imdbid) + imdbid = "" + return imdbid + +def get_movie_info(baseURL, imdbid): + + url = baseURL + "movie.list" + + Logger.debug("Opening URL: %s", url) + + try: + urlObj = myOpener.openit(url) + except IOError, e: + Logger.error("Unable to open URL: %s", str(e)) + return 0, "", "" + + n=0 + movie_id = "" + movie_status = "" + result = json.load(urlObj) + movieid = [item["id"] for item in result["movies"]] + statusid = [item["status_id"] for item in result["movies"]] + library = [item["library"]["identifier"] for item in result["movies"]] + for index in range(len(movieid)): + if library[index] == imdbid: + movie_id = str(movieid[index]) + movie_status = int(statusid[index]) + Logger.info("found movie id %s in CPS database for movie %s", movie_id, imdbid) + n = n + 1 + break + return n, movie_id, movie_status + +def get_status_list(baseURL): + + url = baseURL + "status.list" + + Logger.debug("Opening URL: %s", url) + + try: + urlObj = myOpener.openit(url) + except IOError, e: + Logger.error("Unable to open URL: %s", str(e)) + return [], [] + + result = json.load(urlObj) + status = [item["identifier"] for item in result["list"]] + status_id = [item["id"] for item in result["list"]] + + return status, status_id def process(dirName, nzbName=None, status=0): @@ -40,7 +107,7 @@ def process(dirName, nzbName=None, status=0): if not os.path.isfile(configFilename): Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") - sys.exit(-1) + return 1 # failure config.read(configFilename) @@ -66,6 +133,8 @@ def process(dirName, nzbName=None, status=0): myOpener = AuthURLOpener(username, password) nzbName1 = str(nzbName) + + imdbid = get_imdb(nzbName1, dirName) if ssl: protocol = "https://" @@ -75,6 +144,21 @@ def process(dirName, nzbName=None, status=0): if nzbName == "Manual Run": delay = 0 + baseURL = protocol + host + ":" + port + web_root + "/api/" + apikey + "/" + + status, status_id = get_status_list(baseURL) + if not status: + return 1 # failure + + n, movie_id, movie_status = get_movie_info(baseURL, imdbid) # get the initial status fo this movie. + if not movie_id: + initial status = "" + for index in range(len(status): + if movie_status == status_id[index] + initial_status = movie_status + Logger.debug("This movie is marked as status %s in CouchPotatoServer", status[index]) + break + process_all_exceptions(nzbName.lower(), dirName) if status == 0: @@ -83,7 +167,7 @@ def process(dirName, nzbName=None, status=0): else: command = "renamer.scan" - url = protocol + host + ":" + port + web_root + "/api/" + apikey + "/" + command + url = baseURL + command Logger.info("waiting for %s seconds to allow CPS to process newly extracted files", str(delay)) @@ -95,51 +179,27 @@ def process(dirName, nzbName=None, status=0): urlObj = myOpener.openit(url) except IOError, e: Logger.error("Unable to open URL: %s", str(e)) - sys.exit(1) + return 1 # failure result = json.load(urlObj) Logger.info("CouchPotatoServer returned %s", result) if result['success']: Logger.info("%s started on CouchPotatoServer for %s", command, nzbName1) else: - Logger.error("%s has NOT started on CouchPotatoServer for %s", command, nzbName1) + Logger.error("%s has NOT started on CouchPotatoServer for %s. Exiting", command, nzbName1) + return 1 # failure else: Logger.info("download of %s has failed.", nzbName1) Logger.info("trying to re-cue the next highest ranked release") - a=nzbName1.find('.cp(')+4 - b=nzbName1[a:].find(')')+a - imdbid=nzbName1[a:b] - - url = protocol + host + ":" + port + web_root + "/api/" + apikey + "/movie.list" - - Logger.debug("Opening URL: %s", url) - - try: - urlObj = myOpener.openit(url) - except IOError, e: - Logger.error("Unable to open URL: %s", str(e)) - sys.exit(1) - - n=0 - result = json.load(urlObj) - movieid = [item["id"] for item in result["movies"]] - library = [item["library"] for item in result["movies"]] - identifier = [item["identifier"] for item in library] - for index in range(len(movieid)): - if identifier[index] == imdbid: - movid = str(movieid[index]) - Logger.info("found movie id %s in database for release %s", movid, nzbName1) - n = n + 1 - break - + if n == 0: Logger.warning("cound not find a movie in the database for release %s", nzbName1) Logger.warning("please manually ignore this release and refresh the wanted movie") Logger.error("exiting postprocessing script") - sys.exit(1) + return 1 # failure - url = protocol + host + ":" + port + web_root + "/api/" + apikey + "/searcher.try_next/?id=" + movid + url = baseURL + "searcher.try_next/?id=" + movie_id Logger.debug("Opening URL: %s", url) @@ -147,7 +207,7 @@ def process(dirName, nzbName=None, status=0): urlObj = myOpener.openit(url) except IOError, e: Logger.error("Unable to open URL: %s", str(e)) - sys.exit(1) + return 1 # failure result = urlObj.readlines() for line in result: @@ -157,3 +217,19 @@ def process(dirName, nzbName=None, status=0): if delete_failed: Logger.info("Deleting failed files and folder %s", dirName) shutil.rmtree(dirName) + return 0 # success + + # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. + start = datetime.datetime.now() # set time for timeout + while (datetime.datetime.now() - start) > datetime.timedelta(minutes=2): # only wait 2 minutes, then return to TorrentToMedia + n, movie_id, movie_status = get_movie_info(baseURL, imdbid) # get the current status fo this movie. + if movie_status != initial_status: # Something has changed + for index in range(len(status): + if movie_status == status_id[index] + Logger.info("SUCCESS: This movie is now marked as status %s in CouchPotatoServer", status[index]) + break # leave the for loop + return 0 # success + time.sleep(20) # Just stop this looping infinitely and hogging resources for 2 minutes ;) + else: # The status hasn't changed. we have waited 2 minutes which is more than enough. uTorrent can resule seeding now. + Logger.warning("The movie does not appear to have changed status after 2 minutes. Please check CouchPotato Logs") + return 1 # failure From 6818cb30b64bc1b8ec0657a1007b91c2838b6b03 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 17:48:08 -0800 Subject: [PATCH 13/27] moved CPS renamer complete check to autoprocess --- TorrentToMedia.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index d72a3385..48fdc4fe 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -134,24 +134,17 @@ def main(inputDirectory, inputName, inputCategory, inputHash): # Now we pass off to CouchPotato or Sick-Beard if inputCategory == cpsCategory: - Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName) # can we use logger while logfile open? - autoProcessMovie.process(outputDestination, inputName, status) + Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName) + result = autoProcessMovie.process(outputDestination, inputName, status) elif inputCategory == sbCategory: - Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName) # can we use logger while logfile open? - autoProcessTV.processEpisode(outputDestination, inputName, status) + Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName) + result = autoProcessTV.processEpisode(outputDestination, inputName, status) - # Check if the file still exists in the post-process directory - now = datetime.datetime.now() # set time for timeout - while os.path.exists(outputDestination): # while this directory is still here, CPS hasn't finished renaming - if (datetime.datetime.now() - now) > datetime.timedelta(minutes=3): # note; minimum 1 minute delay in autoProcessMovie - Logger.info("MAIN: The directory %s has not been moved after 3 minutes.", outputDestination) - break - time.sleep(10) #Just stop this looping infinitely and hogging resources for 3 minutes ;) - else: # CPS (and SickBeard) have finished. We can now resume seeding. - Logger.info("MAIN: Post-process appears to have succeeded for: %s", inputName) + if result == 1: + Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was pasued we will resume seeding") #### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission - if inputHash and useLink and clientAgent == 'utorrent': + if inputHash and useLink and clientAgent == 'utorrent' and status == 0: # only resume seeding for successfully extracted files? Logger.debug("MAIN: Starting torrent %s in uTorrent", inputName) utorrentClass.start(inputHash) #### quick 'n dirt hardlink solution for uTorrent, need to implent support for deluge, transmission @@ -201,4 +194,4 @@ if __name__ == "__main__": Logger.error("MAIN: There was a problem loading variables: %s", e) sys.exit(-1) - main(inputDirectory, inputName, inputCategory, inputHash) \ No newline at end of file + main(inputDirectory, inputName, inputCategory, inputHash) From 927c0c41c1ced9af102781798ea93d22e737529b Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 17:51:23 -0800 Subject: [PATCH 14/27] return 1 or 0. no sys.exit allows uTorrent to resume seeding --- autoProcessTV.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/autoProcessTV.py b/autoProcessTV.py index 39f09f0e..7352ba00 100644 --- a/autoProcessTV.py +++ b/autoProcessTV.py @@ -57,7 +57,7 @@ def processEpisode(dirName, nzbName=None, failed=False): if not os.path.isfile(configFilename): Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") - sys.exit(-1) + return 1 # failure try: fp = open(configFilename, "r") @@ -65,7 +65,7 @@ def processEpisode(dirName, nzbName=None, failed=False): fp.close() except IOError, e: Logger.error("Could not read configuration file: %s", str(e)) - sys.exit(1) + return 1 # failure watch_dir = "" host = config.get("SickBeard", "host") @@ -98,7 +98,7 @@ def processEpisode(dirName, nzbName=None, failed=False): #allows manual call of postprocess script if we have specified a watch_dir. Check that here. if nzbName == "Manual Run" and watch_dir == "": Logger.error("In order to run this script manually you must specify a watch_dir in autoProcessTV.cfg") - sys.exit(-1) + return 1 # failure #allows us to specify the default watch directory and call the postproecssing on another PC with different directory structure. if watch_dir != "": dirName = watch_dir @@ -126,7 +126,7 @@ def processEpisode(dirName, nzbName=None, failed=False): # the standard Master bamch of SickBeard cannot process failed downloads. So Exit here. if status: Logger.info("The download failed. Nothing to process") - sys.exit() + return 0 # Success (as far as this script is concerned) else: Logger.info("The download succeeded. Sending process request to SickBeard") @@ -145,8 +145,9 @@ def processEpisode(dirName, nzbName=None, failed=False): urlObj = myOpener.openit(url) except IOError, e: Logger.error("Unable to open URL: %s", str(e)) - sys.exit(1) + return 1 # failure result = urlObj.readlines() for line in result: Logger.info("%s", line) + return 0 # Success From 5d4629b2531e17b755b510f12f39df007d333b78 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 17:53:08 -0800 Subject: [PATCH 15/27] Get result back from autoProcessTV we don't do anything with it... yet. --- nzbToSickBeard.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index d8a17b95..ce6bf683 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -46,7 +46,7 @@ if len(sys.argv) == 8: # 6 Group that the NZB was posted in e.g. alt.binaries.x # 7 Status of post processing. 0 = OK, 1=failed verification, 2=failed unpack, 3=1+2 Logger.info("Script triggered from SABnzbd, starting autoProcessTV...") - autoProcessTV.processEpisode(sys.argv[1], sys.argv[2], sys.argv[7]) + result = autoProcessTV.processEpisode(sys.argv[1], sys.argv[2], sys.argv[7]) # NZBGet elif len(sys.argv) == 4: @@ -55,9 +55,9 @@ elif len(sys.argv) == 4: # 2 The original name of the NZB file # 3 The status of the download: 0 == successful Logger.info("Script triggered from NZBGet, starting autoProcessTV...") - autoProcessTV.processEpisode(sys.argv[1], sys.argv[2], sys.argv[3]) + result = autoProcessTV.processEpisode(sys.argv[1], sys.argv[2], sys.argv[3]) else: Logger.debug("Invalid number of arguments received from client.") Logger.info("Running autoProcessTV as a manual run...") - autoProcessTV.processEpisode('Manual Run', 'Manual Run', 0) + result = autoProcessTV.processEpisode('Manual Run', 'Manual Run', 0) From f532091de592c39cfd863e799d0f699430d8433a Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 17:54:29 -0800 Subject: [PATCH 16/27] Get result back from autoProcessMovie We don't do anything with it... yet ;) --- nzbToCouchPotato.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index cb538c74..e33575cf 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -24,7 +24,7 @@ if len(sys.argv) == 8: # 6 Group that the NZB was posted in e.g. alt.binaries.x # 7 Status of post processing. 0 = OK, 1=failed verification, 2=failed unpack, 3=1+2 Logger.info("Script triggered from SABnzbd, starting autoProcessMovie...") - autoProcessMovie.process(sys.argv[1], sys.argv[2], sys.argv[7]) + result = autoProcessMovie.process(sys.argv[1], sys.argv[2], sys.argv[7]) # NZBGet elif len(sys.argv) == 4: @@ -33,10 +33,9 @@ elif len(sys.argv) == 4: # 2 The original name of the NZB file # 3 The status of the download: 0 == successful Logger.info("Script triggered from NZBGet, starting autoProcessMovie...") - - autoProcessMovie.process(sys.argv[1], sys.argv[2], sys.argv[3]) + result = autoProcessMovie.process(sys.argv[1], sys.argv[2], sys.argv[3]) else: Logger.warn("Invalid number of arguments received from client.") Logger.info("Running autoProcessMovie as a manual run...") - autoProcessMovie.process('Manual Run', 'Manual Run', 0) + result = autoProcessMovie.process('Manual Run', 'Manual Run', 0) From db56bdd9664b1e6576ab4d9dbe6bf36b20af3ba4 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 18:57:55 -0800 Subject: [PATCH 17/27] fixed to status check --- autoProcessMovie.py | 57 +++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/autoProcessMovie.py b/autoProcessMovie.py index 54112a52..7c614c00 100644 --- a/autoProcessMovie.py +++ b/autoProcessMovie.py @@ -48,12 +48,15 @@ def get_imdb(nzbName1, dirName): Logger.info("Found movie id %s in directory", imdbid) return imdbid else: - Logger.info("Could not find movie id in directory or name", imdbid) + Logger.warning("Could not find movie id in directory or name", imdbid) + Logger.info("Postprocessing will continue, but the movie may not be identified correctly by CouchPotato") imdbid = "" return imdbid def get_movie_info(baseURL, imdbid): + if not imdbid: + return 0, "", "" url = baseURL + "movie.list" Logger.debug("Opening URL: %s", url) @@ -64,25 +67,23 @@ def get_movie_info(baseURL, imdbid): Logger.error("Unable to open URL: %s", str(e)) return 0, "", "" - n=0 movie_id = "" movie_status = "" result = json.load(urlObj) movieid = [item["id"] for item in result["movies"]] - statusid = [item["status_id"] for item in result["movies"]] library = [item["library"]["identifier"] for item in result["movies"]] for index in range(len(movieid)): if library[index] == imdbid: movie_id = str(movieid[index]) - movie_status = int(statusid[index]) Logger.info("found movie id %s in CPS database for movie %s", movie_id, imdbid) - n = n + 1 break - return n, movie_id, movie_status + return movie_id -def get_status_list(baseURL): +def get_status(movie_id): - url = baseURL + "status.list" + if not movie_id: + return "" + url = baseURL + "movie.get/?id=" + movie_id Logger.debug("Opening URL: %s", url) @@ -90,13 +91,13 @@ def get_status_list(baseURL): urlObj = myOpener.openit(url) except IOError, e: Logger.error("Unable to open URL: %s", str(e)) - return [], [] - + return "" result = json.load(urlObj) - status = [item["identifier"] for item in result["list"]] - status_id = [item["id"] for item in result["list"]] - - return status, status_id + try: + movie_status = result["movie"]["status"]["identifier"] + return movie_status + except: + return "" def process(dirName, nzbName=None, status=0): @@ -146,18 +147,12 @@ def process(dirName, nzbName=None, status=0): baseURL = protocol + host + ":" + port + web_root + "/api/" + apikey + "/" - status, status_id = get_status_list(baseURL) - if not status: - return 1 # failure - - n, movie_id, movie_status = get_movie_info(baseURL, imdbid) # get the initial status fo this movie. + movie_id = get_movie_info(baseURL, imdbid) # get the CPS database movie id this movie. if not movie_id: initial status = "" - for index in range(len(status): - if movie_status == status_id[index] - initial_status = movie_status - Logger.debug("This movie is marked as status %s in CouchPotatoServer", status[index]) - break + else: + initial_status = get_status(movie_id) + Logger.debug("This movie is marked as status %s in CouchPotatoServer", initial_status) process_all_exceptions(nzbName.lower(), dirName) @@ -193,7 +188,7 @@ def process(dirName, nzbName=None, status=0): Logger.info("download of %s has failed.", nzbName1) Logger.info("trying to re-cue the next highest ranked release") - if n == 0: + if not movie_id: Logger.warning("cound not find a movie in the database for release %s", nzbName1) Logger.warning("please manually ignore this release and refresh the wanted movie") Logger.error("exiting postprocessing script") @@ -219,15 +214,15 @@ def process(dirName, nzbName=None, status=0): shutil.rmtree(dirName) return 0 # success + if nzbName == "Manual Run": + return 0 # success + # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. start = datetime.datetime.now() # set time for timeout while (datetime.datetime.now() - start) > datetime.timedelta(minutes=2): # only wait 2 minutes, then return to TorrentToMedia - n, movie_id, movie_status = get_movie_info(baseURL, imdbid) # get the current status fo this movie. - if movie_status != initial_status: # Something has changed - for index in range(len(status): - if movie_status == status_id[index] - Logger.info("SUCCESS: This movie is now marked as status %s in CouchPotatoServer", status[index]) - break # leave the for loop + movie_status = get_status(movie_id) # get the current status fo this movie. + if movie_status != initial_status: # Something has changed. CPS must have processed this movie. + Logger.info("SUCCESS: This movie is now marked as status %s in CouchPotatoServer", movie_status) return 0 # success time.sleep(20) # Just stop this looping infinitely and hogging resources for 2 minutes ;) else: # The status hasn't changed. we have waited 2 minutes which is more than enough. uTorrent can resule seeding now. From 8dceb863c5ff0070c0b368c26434a855bbd12a7d Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 20:48:28 -0800 Subject: [PATCH 18/27] improved status check Tests show renamenr usually moves files in less than 20 seconds and updates the movie status in less than 30. When the renamer fails to get correct quality etc, the status won't be updated, so the 2 minute timeout will occur and then torrent seeding can resume. --- autoProcessMovie.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/autoProcessMovie.py b/autoProcessMovie.py index 7c614c00..06653f1a 100644 --- a/autoProcessMovie.py +++ b/autoProcessMovie.py @@ -3,6 +3,7 @@ import urllib import os import shutil import ConfigParser +import datetime import time import json import logging @@ -48,15 +49,15 @@ def get_imdb(nzbName1, dirName): Logger.info("Found movie id %s in directory", imdbid) return imdbid else: - Logger.warning("Could not find movie id in directory or name", imdbid) + Logger.warning("Could not find an imdb id in directory or name") Logger.info("Postprocessing will continue, but the movie may not be identified correctly by CouchPotato") imdbid = "" return imdbid -def get_movie_info(baseURL, imdbid): +def get_movie_info(myOpener, baseURL, imdbid): if not imdbid: - return 0, "", "" + return "" url = baseURL + "movie.list" Logger.debug("Opening URL: %s", url) @@ -65,7 +66,7 @@ def get_movie_info(baseURL, imdbid): urlObj = myOpener.openit(url) except IOError, e: Logger.error("Unable to open URL: %s", str(e)) - return 0, "", "" + return "" movie_id = "" movie_status = "" @@ -79,11 +80,11 @@ def get_movie_info(baseURL, imdbid): break return movie_id -def get_status(movie_id): +def get_status(myOpener, baseURL, movie_id): if not movie_id: return "" - url = baseURL + "movie.get/?id=" + movie_id + url = baseURL + "movie.get/?id=" + str(movie_id) Logger.debug("Opening URL: %s", url) @@ -147,11 +148,11 @@ def process(dirName, nzbName=None, status=0): baseURL = protocol + host + ":" + port + web_root + "/api/" + apikey + "/" - movie_id = get_movie_info(baseURL, imdbid) # get the CPS database movie id this movie. + movie_id = get_movie_info(myOpener, baseURL, imdbid) # get the CPS database movie id this movie. if not movie_id: - initial status = "" + initial_status = "" else: - initial_status = get_status(movie_id) + initial_status = get_status(myOpener, baseURL, movie_id) Logger.debug("This movie is marked as status %s in CouchPotatoServer", initial_status) process_all_exceptions(nzbName.lower(), dirName) @@ -208,7 +209,7 @@ def process(dirName, nzbName=None, status=0): for line in result: Logger.info("%s", line) - Logger.info("movie %s set to try the next best release on CouchPotatoServer", movid) + Logger.info("movie %s set to try the next best release on CouchPotatoServer", movie_id) if delete_failed: Logger.info("Deleting failed files and folder %s", dirName) shutil.rmtree(dirName) @@ -219,8 +220,8 @@ def process(dirName, nzbName=None, status=0): # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. start = datetime.datetime.now() # set time for timeout - while (datetime.datetime.now() - start) > datetime.timedelta(minutes=2): # only wait 2 minutes, then return to TorrentToMedia - movie_status = get_status(movie_id) # get the current status fo this movie. + while (datetime.datetime.now() - start) < datetime.timedelta(minutes=2): # only wait 2 minutes, then return to TorrentToMedia + movie_status = get_status(myOpener, baseURL, movie_id) # get the current status fo this movie. if movie_status != initial_status: # Something has changed. CPS must have processed this movie. Logger.info("SUCCESS: This movie is now marked as status %s in CouchPotatoServer", movie_status) return 0 # success From 8be53264c83bfe4b0425ae6bff6cdcc2a4d05c82 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 21:02:23 -0800 Subject: [PATCH 19/27] minor improvements --- autoProcessMovie.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/autoProcessMovie.py b/autoProcessMovie.py index 06653f1a..6a8ee432 100644 --- a/autoProcessMovie.py +++ b/autoProcessMovie.py @@ -76,7 +76,7 @@ def get_movie_info(myOpener, baseURL, imdbid): for index in range(len(movieid)): if library[index] == imdbid: movie_id = str(movieid[index]) - Logger.info("found movie id %s in CPS database for movie %s", movie_id, imdbid) + Logger.info("Found movie id %s in CPS database for movie %s", movie_id, imdbid) break return movie_id @@ -96,6 +96,7 @@ def get_status(myOpener, baseURL, movie_id): result = json.load(urlObj) try: movie_status = result["movie"]["status"]["identifier"] + Logger.debug("This movie is marked as status %s in CouchPotatoServer", movie_status) return movie_status except: return "" @@ -149,11 +150,8 @@ def process(dirName, nzbName=None, status=0): baseURL = protocol + host + ":" + port + web_root + "/api/" + apikey + "/" movie_id = get_movie_info(myOpener, baseURL, imdbid) # get the CPS database movie id this movie. - if not movie_id: - initial_status = "" - else: - initial_status = get_status(myOpener, baseURL, movie_id) - Logger.debug("This movie is marked as status %s in CouchPotatoServer", initial_status) + + initial_status = get_status(myOpener, baseURL, movie_id) process_all_exceptions(nzbName.lower(), dirName) @@ -165,7 +163,7 @@ def process(dirName, nzbName=None, status=0): url = baseURL + command - Logger.info("waiting for %s seconds to allow CPS to process newly extracted files", str(delay)) + Logger.info("Waiting for %s seconds to allow CPS to process newly extracted files", str(delay)) time.sleep(delay) @@ -186,13 +184,13 @@ def process(dirName, nzbName=None, status=0): return 1 # failure else: - Logger.info("download of %s has failed.", nzbName1) - Logger.info("trying to re-cue the next highest ranked release") + Logger.info("Download of %s has failed.", nzbName1) + Logger.info("Trying to re-cue the next highest ranked release") if not movie_id: - Logger.warning("cound not find a movie in the database for release %s", nzbName1) - Logger.warning("please manually ignore this release and refresh the wanted movie") - Logger.error("exiting postprocessing script") + Logger.warning("Cound not find a movie in the database for release %s", nzbName1) + Logger.warning("Please manually ignore this release and refresh the wanted movie") + Logger.error("Exiting autoProcessMovie script") return 1 # failure url = baseURL + "searcher.try_next/?id=" + movie_id @@ -209,10 +207,13 @@ def process(dirName, nzbName=None, status=0): for line in result: Logger.info("%s", line) - Logger.info("movie %s set to try the next best release on CouchPotatoServer", movie_id) + Logger.info("Movie %s set to try the next best release on CouchPotatoServer", movie_id) if delete_failed: Logger.info("Deleting failed files and folder %s", dirName) - shutil.rmtree(dirName) + try: + shutil.rmtree(dirName) + except e: + Logger.error("Unable to delete folder %s due to: %s", dirName, str(e)) return 0 # success if nzbName == "Manual Run": From aea30191a54c9adce3aac617b4b372eddf2242bd Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Sun, 3 Mar 2013 22:22:47 -0800 Subject: [PATCH 20/27] More cleaning --- autoProcessMovie.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/autoProcessMovie.py b/autoProcessMovie.py index 6a8ee432..10c70316 100644 --- a/autoProcessMovie.py +++ b/autoProcessMovie.py @@ -31,11 +31,11 @@ class AuthURLOpener(urllib.FancyURLopener): self.numTries = 0 return urllib.FancyURLopener.open(self, url) -def get_imdb(nzbName1, dirName): +def get_imdb(nzbName, dirName): - a=nzbName1.find('.cp(')+4 #search for .cptt( in nzbName - b=nzbName1[a:].find(')')+a - imdbid=nzbName1[a:b] + a=nzbName.find('.cp(')+4 #search for .cptt( in nzbName + b=nzbName[a:].find(')')+a + imdbid=nzbName[a:b] if imdbid: Logger.info("Found movie id %s in name", imdbid) @@ -51,8 +51,7 @@ def get_imdb(nzbName1, dirName): else: Logger.warning("Could not find an imdb id in directory or name") Logger.info("Postprocessing will continue, but the movie may not be identified correctly by CouchPotato") - imdbid = "" - return imdbid + return "" def get_movie_info(myOpener, baseURL, imdbid): @@ -69,7 +68,6 @@ def get_movie_info(myOpener, baseURL, imdbid): return "" movie_id = "" - movie_status = "" result = json.load(urlObj) movieid = [item["id"] for item in result["movies"]] library = [item["library"]["identifier"] for item in result["movies"]] @@ -98,7 +96,8 @@ def get_status(myOpener, baseURL, movie_id): movie_status = result["movie"]["status"]["identifier"] Logger.debug("This movie is marked as status %s in CouchPotatoServer", movie_status) return movie_status - except: + except e: # index out of range/doesn't exist? + Logger.error("Could not find a status for this movie due to: %s", str(e)) return "" def process(dirName, nzbName=None, status=0): @@ -135,9 +134,9 @@ def process(dirName, nzbName=None, status=0): myOpener = AuthURLOpener(username, password) - nzbName1 = str(nzbName) + nzbName = str(nzbName) # make sure it is a string - imdbid = get_imdb(nzbName1, dirName) + imdbid = get_imdb(nzbName, dirName) if ssl: protocol = "https://" @@ -178,17 +177,17 @@ def process(dirName, nzbName=None, status=0): result = json.load(urlObj) Logger.info("CouchPotatoServer returned %s", result) if result['success']: - Logger.info("%s started on CouchPotatoServer for %s", command, nzbName1) + Logger.info("%s started on CouchPotatoServer for %s", command, nzbName) else: - Logger.error("%s has NOT started on CouchPotatoServer for %s. Exiting", command, nzbName1) + Logger.error("%s has NOT started on CouchPotatoServer for %s. Exiting", command, nzbName) return 1 # failure else: - Logger.info("Download of %s has failed.", nzbName1) + Logger.info("Download of %s has failed.", nzbName) Logger.info("Trying to re-cue the next highest ranked release") if not movie_id: - Logger.warning("Cound not find a movie in the database for release %s", nzbName1) + Logger.warning("Cound not find a movie in the database for release %s", nzbName) Logger.warning("Please manually ignore this release and refresh the wanted movie") Logger.error("Exiting autoProcessMovie script") return 1 # failure From 9d32800a504991aa21aec1da18ba3bc470651869 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 15:05:32 -0800 Subject: [PATCH 21/27] a few fixes. --- TorrentToMedia.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 48fdc4fe..4541feb3 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -78,21 +78,21 @@ def main(inputDirectory, inputName, inputCategory, inputHash): copy_link(filePath, targetDirectory, useLink, outputDestination) except Exception as e: Logger.error("MAIN: Failed to link file: %s", file) - Logger.debug e + Logger.debug(e) elif fileExtention in metaContainer: Logger.info("MAIN: Found metadata file %s for file %s", fileExtention, filePath) try: copy_link(filePath, targetDirectory, useLink, outputDestination) except Exception as e: Logger.error("MAIN: Failed to link file: %s", file) - Logger.debug e + Logger.debug(e) elif fileExtention in compressedContainer: Logger.info("MAIN: Found compressed archive %s for file %s", fileExtention, filePath) try: extractor.extract(filePath, outputDestination) except Exception as e: Logger.warn("MAIN: Extraction failed for: %s", file) - Logger.debug e + Logger.debug(e) else: Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtention, filePath) continue @@ -185,8 +185,9 @@ if __name__ == "__main__": metaContainer = (config.get("Torrent", "metaExtentions")).split(',') # .nfo,.sub,.srt cpsCategory = config.get("CouchPotato", "cpsCategory") # movie - sbCategory = config.get("SickBeard", "tvCategory") # tv - categories.append(cpsCategory, sbCategory) + sbCategory = config.get("SickBeard", "sbCategory") # tv + categories.append(cpsCategory) + categories.append(sbCategory) try: inputDirectory, inputName, inputCategory, inputHash = parse_args(clientAgent) From 6909e0edb5105d827e5fd51f486d290e6c1df7ab Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 15:13:55 -0800 Subject: [PATCH 22/27] another small fix --- TorrentToMedia.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 4541feb3..73faab5f 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -32,12 +32,11 @@ def main(inputDirectory, inputName, inputCategory, inputHash): for category in categories: if category == inputCategory: - outputDestination = os.path.normpath(os.path.join(outputDestination, category, inputName)) + outputDestination = os.path.normpath(os.path.join(outputDirectory, category, inputName)) Logger.info("MAIN: Output directory set to: %s", outputDestination) - pass + break else: - Logger.error("MAIN: Could not match input category with defined categories, Exiting") - sys.exit(-1) + continue Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) From 8c238118efe614e1a48fcb392cc3238c49cc911f Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 15:15:24 -0800 Subject: [PATCH 23/27] fix logger --- nzbToMediaUtil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index 12509405..b2c63b1c 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -5,7 +5,7 @@ import sys import linktastic.linktastic as linktastic -Logger = logging.getLogger(__name__) +Logger = logging.getLogger() def nzbtomedia_configure_logging(dirname): From f8e887b853e08e75a9b97f901b01ff1f45e590db Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 15:37:26 -0800 Subject: [PATCH 24/27] fix file extension check --- nzbToMediaUtil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index b2c63b1c..d43e5b9f 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -191,7 +191,7 @@ def iterate_media_files(dirname): for dirpath, dirnames, filesnames in os.walk(dirname): for filename in filesnames: - fileExtention = os.path.splitext(filename)[0] + fileExtention = os.path.splitext(filename)[1] if not (fileExtention in mediaContainer): continue yield dirpath, os.path.join(dirpath, filename) From 628539da08747dee5f24dd521653960d36f4e573 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 15:39:24 -0800 Subject: [PATCH 25/27] only reverse the filename, simplify match --- nzbToMediaSceneExceptions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nzbToMediaSceneExceptions.py b/nzbToMediaSceneExceptions.py index bb8d103c..d0a327f1 100644 --- a/nzbToMediaSceneExceptions.py +++ b/nzbToMediaSceneExceptions.py @@ -11,7 +11,7 @@ Logger = logging.getLogger() def process_all_exceptions(name, dirname): for group, exception in __customgroups__.items(): - if not group in name: + if not (group in name or group in dirname): continue process_exception(exception, name, dirname) @@ -23,7 +23,7 @@ def process_exception(exception, name, dirname): def process_qoq(filename, dirname): Logger.debug("Reversing the file name for a QoQ release %s", filename) - head, fileExtention = os.path.splitext(filename) + head, fileExtention = os.path.splitext(os.path.basename(filename)) newname = head[::-1] newfile = newname + fileExtention newfilePath = os.path.join(dirname, newfile) @@ -32,4 +32,4 @@ def process_qoq(filename, dirname): # dict for custom groups # we can add more to this list -__customgroups__ = {'[=-< Q o Q >-=]': process_qoq} +__customgroups__ = {'Q o Q': process_qoq} From 7accf74fb960ddb92d4bdbdfa306d99977fe7d52 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 16:21:18 -0800 Subject: [PATCH 26/27] added safeName for windows directory --- TorrentToMedia.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 73faab5f..e998b150 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -29,10 +29,10 @@ def main(inputDirectory, inputName, inputCategory, inputHash): Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure - + for category in categories: if category == inputCategory: - outputDestination = os.path.normpath(os.path.join(outputDirectory, category, inputName)) + outputDestination = os.path.normpath(os.path.join(outputDirectory, category, safeName(inputName))) Logger.info("MAIN: Output directory set to: %s", outputDestination) break else: @@ -50,7 +50,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash): if root == 1: Logger.debug("MAIN: Looking for %s in filename", inputName) - if (inputName in file) or (os.path.splitext(file)[0] in inputName): + if (safeName(inputName) in safeName(file)) or (safeName(os.path.splitext(file)[0]) in safeName(inputName)): pass # This file does match the Torrent name Logger.debug("Found file %s that matches Torrent Name %s", file, inputName) else: From 7c56ba6b1d5ada8ed3236f9b945288108bebc0f4 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Mon, 4 Mar 2013 16:24:52 -0800 Subject: [PATCH 27/27] added safe name --- nzbToMediaUtil.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nzbToMediaUtil.py b/nzbToMediaUtil.py index d43e5b9f..0ce4cc39 100644 --- a/nzbToMediaUtil.py +++ b/nzbToMediaUtil.py @@ -1,6 +1,7 @@ import logging import logging.config import os +import re import sys import linktastic.linktastic as linktastic @@ -8,6 +9,11 @@ import linktastic.linktastic as linktastic Logger = logging.getLogger() +def safeName(name): + safename = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "", name) #make this name safe for use in directories for windows etc. + return safename + + def nzbtomedia_configure_logging(dirname): logFile = os.path.join(dirname, "postprocess.log") logging.config.fileConfig(os.path.join(dirname, "autoProcessMedia.cfg")) @@ -76,6 +82,11 @@ def category_search(inputDirectory, inputName, inputCategory, root, categories): if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 inputDirectory = os.path.join(categorySearch[0], inputName) # we only want to search this next dir up. break # we are done + elif os.path.isdir(os.path.join(categorySearch[0], safeName(inputName))) and inputName: # testing for torrent name in first sub directory + Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], safeName(inputName)), categorySearch[0]) + if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 + inputDirectory = os.path.join(categorySearch[0], safeName(inputName)) # we only want to search this next dir up. + break # we are done elif inputName: # if these exists, we are ok to proceed, but we are in a root/common directory. Logger.info("SEARCH: Could not find a unique torrent folder in the directory structure") Logger.info("SEARCH: The directory passed is the root directory for category %s", categorySearch2[1]) @@ -89,7 +100,7 @@ def category_search(inputDirectory, inputName, inputCategory, root, categories): Logger.info("SEARCH: We will try and determine which files to process, individually") root = 2 break - elif categorySearch2[1] == inputName and inputName: # we have identified a unique directory. + elif safeName(categorySearch2[1]) == safeName(inputName) and inputName: # we have identified a unique directory. Logger.info("SEARCH: Files appear to be in their own directory") if inputCategory: # we are ok to proceed. break # we are done