From cb435de3318f0b0e718bf3bc7cc7c2ea2e832a41 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Mon, 9 Dec 2013 14:33:26 +1030 Subject: [PATCH] Re-order torrentToMedia. fixes #185 #210 #218 --- TorrentToMedia.py | 178 ++++++++++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 70 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 79d5e4f1..dc4fbbc1 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -34,6 +34,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): foundFile = int(0) extracted_folder = [] extractionSuccess = False + copy_list = [] Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) if inputCategory == sbCategory and sbFork in SICKBEARD_TORRENT: @@ -49,9 +50,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): outputDestination = "" for category in categories: if category == inputCategory: - if inputCategory == hpCategory: - outputDestination = inputDirectory #HP needs to scan the same dir as passed to downloader. - break if os.path.basename(inputDirectory) == inputName: Logger.info("MAIN: Download is a directory") outputDestination = os.path.normpath(os.path.join(outputDirectory, category, safeName(inputName))) @@ -63,14 +61,52 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): else: continue if outputDestination == "": + if inputCategory == "": + inputCategory = "UNCAT" if os.path.basename(inputDirectory) == inputName: Logger.info("MAIN: Download is a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, safeName(inputName))) + outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, safeName(inputName))) else: Logger.info("MAIN: Download is not a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, os.path.splitext(safeName(inputName))[0])) + outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, os.path.splitext(safeName(inputName))[0])) Logger.info("MAIN: Output directory set to: %s", outputDestination) + processOnly = [cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory] + if not "NONE" in user_script_categories: # if None, we only process the 5 listed. + if "ALL" in user_script_categories: # All defined categories + processOnly = categories + processOnly.extend(user_script_categories) # Adds all categories to be processed by userscript. + + if not inputCategory in processOnly: + Logger.info("MAIN: No processing to be done for category: %s. Exiting", inputCategory) + Logger.info("MAIN: All done.") + sys.exit() + + # Hardlink solution for uTorrent, need to implent support for deluge, transmission + if clientAgent in ['utorrent', 'transmission'] and inputHash and useLink != "no": + if clientAgent == 'utorrent': + try: + Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) + utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) + except: + Logger.exception("MAIN: Failed to connect to uTorrent") + utorrentClass = "" + if clientAgent == 'transmission': + try: + Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) + TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) + except: + Logger.exception("MAIN: Failed to connect to Transmission") + TransmissionClass = "" + + # if we are using links with uTorrent it means we need to pause it in order to access the files + Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) + if clientAgent == 'utorrent' and utorrentClass != "": + utorrentClass.stop(inputHash) + if clientAgent == 'transmission' and TransmissionClass !="": + TransmissionClass.stop_torrent(inputID) + time.sleep(5) # Give Torrent client some time to catch up with the change + Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) now = datetime.datetime.now() @@ -102,11 +138,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): else: continue # This file has not been recently moved or created, skip it - if not inputCategory in [cpsCategory, sbCategory, hpCategory]: #process all for non-video categories, except HP. - Logger.info("MAIN: Found file %s for category %s", filePath, inputCategory) - copy_link(filePath, targetDirectory, useLink, outputDestination) - elif fileExtension in mediaContainer and not inputCategory == hpCategory: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize): # Ignore samples + if fileExtension in mediaContainer: # If the file is a video file + if is_sample(filePath, inputName, minSampleSize) and not inputCategory == hpCategory: # Ignore samples Logger.info("MAIN: Ignoring sample file: %s ", filePath) continue else: @@ -114,15 +147,25 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.info("MAIN: Found video file %s in %s", fileExtension, filePath) try: copy_link(filePath, targetDirectory, useLink, outputDestination) + copy_list.append([filePath, os.path.join(outputDestination, file)]) except: Logger.exception("MAIN: Failed to link file: %s", file) - elif fileExtension in metaContainer and not inputCategory == hpCategory: + elif fileExtension in metaContainer: Logger.info("MAIN: Found metadata file %s for file %s", fileExtension, filePath) try: copy_link(filePath, targetDirectory, useLink, outputDestination) + copy_list.append([filePath, os.path.join(outputDestination, file)]) except: Logger.exception("MAIN: Failed to link file: %s", file) + continue elif fileExtension in compressedContainer: + if inputCategory in [hpCategory]: # We need to link all files for HP in order to move these back to support seeding. + Logger.info("MAIN: Linking compressed archive file %s for file %s", fileExtension, filePath) + try: + copy_link(filePath, targetDirectory, useLink, outputDestination) + copy_list.append([filePath, os.path.join(outputDestination, file)]) + except: + Logger.exception("MAIN: Failed to link file: %s", file) # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. if re.search(r'\d+', os.path.splitext(fileName)[1]) and os.path.dirname(filePath) in extracted_folder and not (os.path.splitext(fileName)[1] in ['.720p','.1080p']): part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) @@ -133,69 +176,50 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): continue Logger.info("MAIN: Found compressed archive %s for file %s", fileExtension, filePath) try: - extractor.extract(filePath, outputDestination) + if inputCategory == hpCategory: # HP needs to scan the same dir as passed to downloader. + extractor.extract(filePath, inputDirectory) + else: + extractor.extract(filePath, outputDestination) extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content) extracted_folder.append(os.path.dirname(filePath)) except: Logger.exception("MAIN: Extraction failed for: %s", file) - elif inputCategory == hpCategory: + continue + elif not inputCategory in [cpsCategory, sbCategory]: #process all for non-video categories. + Logger.info("MAIN: Found file %s for category %s", filePath, inputCategory) + copy_link(filePath, targetDirectory, useLink, outputDestination) + copy_list.append([filePath, os.path.join(outputDestination, file)]) continue else: Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtension, filePath) continue - if not inputCategory in [hpCategory]: #don't flatten hp in case multi cd albums. + if not inputCategory in [hpCategory]: #don't flatten hp in case multi cd albums, and we need to copy this back later. flatten(outputDestination) # Now check if movie files exist in destination: - for dirpath, dirnames, filenames in os.walk(outputDestination): - for file in filenames: - filePath = os.path.join(dirpath, file) - fileName, fileExtension = os.path.splitext(file) - if fileExtension in mediaContainer: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize): - Logger.debug("MAIN: Removing sample file: %s", filePath) - os.unlink(filePath) # remove samples + if inputCategory in [cpsCategory, sbCategory]: + for dirpath, dirnames, filenames in os.walk(outputDestination): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileName, fileExtension = os.path.splitext(file) + if fileExtension in mediaContainer: # If the file is a video file + if is_sample(filePath, inputName, minSampleSize): + Logger.debug("MAIN: Removing sample file: %s", filePath) + os.unlink(filePath) # remove samples + else: + Logger.debug("MAIN: Found media file: %s", filePath) + video2 = video2 + 1 else: - Logger.debug("MAIN: Found media file: %s", filePath) - video2 = video2 + 1 - else: - Logger.debug("MAIN: File %s is not a media file", filePath) - if video2 >= video and video2 > int(0): # Check that all video files were moved - Logger.debug("MAIN: Found %s media files", str(video2)) - status = int(0) - else: - Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video)) - - # Hardlink solution for uTorrent, need to implent support for deluge, transmission - if clientAgent in ['utorrent', 'transmission'] and inputHash and useLink != "no": - if clientAgent == 'utorrent': - try: - Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) - utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) - except: - Logger.exception("MAIN: Failed to connect to uTorrent") - utorrentClass = "" - if clientAgent == 'transmission': - try: - Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) - TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) - except: - Logger.exception("MAIN: Failed to connect to Transmission") - TransmissionClass = "" - - # if we are using links with uTorrent it means we need to pause it in order to access the files - Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.stop(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.stop_torrent(inputID) - time.sleep(5) # Give Torrent client some time to catch up with the change + Logger.debug("MAIN: File %s is not a media file", filePath) + if video2 >= video and video2 > int(0): # Check that all video files were moved + Logger.debug("MAIN: Found %s media files", str(video2)) + status = int(0) + else: + Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video)) processCategories = Set([cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory]) - if inputCategory == "": - inputCategory = "UNCAT" - if (user_script_categories != "NONE" and inputCategory in user_script_categories) or (user_script_categories == "ALL" and not inputCategory in processCategories): + if (inputCategory in user_script_categories and not "NONE" in user_script_categories) or ("ALL" in user_script_categories and not inputCategory in processCategories): Logger.info("MAIN: Processing user script %s.", user_script) result = external_script(outputDestination) elif status == int(0) or (inputCategory in [hpCategory, mlCategory, gzCategory]): # if movies linked/extracted or for other categories. @@ -214,7 +238,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): result = autoProcessTV.processEpisode(outputDestination, inputName, status) elif inputCategory == hpCategory: Logger.info("MAIN: Calling HeadPhones to post-process: %s", inputName) - result = autoProcessMusic.process(outputDestination, inputName, status) + result = autoProcessMusic.process(inputDirectory, inputName, status) elif inputCategory == mlCategory: Logger.info("MAIN: Calling Mylar to post-process: %s", inputName) result = autoProcessComics.processEpisode(outputDestination, inputName, status) @@ -225,15 +249,22 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): if result == 1: Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding") + if inputCategory == hpCategory: + # we need to move the output dir files back... + Logger.debug("MAIN: Moving temporary HeadPhones files back to allow seeding.") + for item in copy_list: + if os.path.isfile(item[1]): # check to ensure temp files still exist. + if os.path.isfile(item[0]): # both exist, remove temp version + Logger.debug("MAIN: File %s still present. Removing tempoary file %s", str(item[0]), str(item[1]) + os.unlink(item[1]) + continue + else: # move temp version back to allow seeding or Torrent removal. + Logger.debug("MAIN: Moving %s to %s", str(item[1]), str(item[0]) + shutil.move(item[1], item[0]) + continue + # Hardlink solution for uTorrent, need to implent support for deluge, transmission if clientAgent in ['utorrent', 'transmission'] and inputHash and useLink != "no": - # we always want to resume seeding, for now manually find out what is wrong when extraction fails - if deleteOriginal == 0: - Logger.debug("MAIN: Starting torrent %s in %s", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.start(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.start_torrent(inputID) # Delete torrent and torrentdata from Torrent client if processing was successful. if deleteOriginal == 1 and result != 1: Logger.debug("MAIN: Deleting torrent %s from %s", inputName, clientAgent) @@ -246,7 +277,14 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): TransmissionClass.remove_torrent(inputID, False) else: TransmissionClass.remove_torrent(inputID, True) - time.sleep(5) + # we always want to resume seeding, for now manually find out what is wrong when extraction fails + else: + Logger.debug("MAIN: Starting torrent %s in %s", inputName, clientAgent) + if clientAgent == 'utorrent' and utorrentClass != "": + utorrentClass.start(inputHash) + if clientAgent == 'transmission' and TransmissionClass !="": + TransmissionClass.start_torrent(inputID) + time.sleep(5) #cleanup if inputCategory in processCategories and result == 0 and os.path.isdir(outputDestination): num_files_new = int(0) @@ -386,8 +424,8 @@ if __name__ == "__main__": categories.append(mlCategory) categories.append(gzCategory) - user_script_categories = config.get("UserScript", "user_script_categories") # NONE - if user_script_categories != "None": + user_script_categories = config.get("UserScript", "user_script_categories").split(',') # NONE + if not "NONE" in user_script_categories: user_script_mediaExtensions = (config.get("UserScript", "user_script_mediaExtensions")).split(',') user_script = config.get("UserScript", "user_script_path") user_script_param = (config.get("UserScript", "user_script_param")).split(',')