From e23d39d4581047ed607fe4d19666da023ec942fc Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 10:08:00 +0930 Subject: [PATCH 01/11] fixed TorrentToMedia flow. All NZBGet categories are enabled. fixes #309 #314 #315 #316 --- TorrentToMedia.py | 109 +++++++++++++++++++-------------- nzbtomedia/nzbToMediaConfig.py | 6 ++ 2 files changed, 70 insertions(+), 45 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 76f9f0ab..a9f38ee2 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -48,13 +48,52 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.debug("MAIN: Determined Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) - if config().issubsection(inputCategory,["SickBeard","NzbDrone"]): - Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) - result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) - if result != 0: - Logger.info("MAIN: A problem was reported in the autoProcessTV script.") - Logger.info("MAIN: All done.") - sys.exit() + # Hardlink solution for Torrents + if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: + if clientAgent == 'utorrent': + try: + Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) + utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) + except: + Logger.exception("MAIN: Failed to connect to uTorrent") + + if clientAgent == 'transmission': + try: + Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) + TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) + except: + Logger.exception("MAIN: Failed to connect to Transmission") + + if clientAgent == 'deluge': + try: + Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, DelugeHost, DelugePort) + delugeClient = DelugeClient() + delugeClient.connect(host = DelugeHost, port = DelugePort, username = DelugeUSR, password = DelugePWD) + except: + Logger.exception("MAIN: Failed to connect to deluge") + + # if we are using links with Torrents it means we need to pause it in order to access the files + Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) + if clientAgent == 'utorrent' and utorrentClass != "": + utorrentClass.stop(inputHash) + if clientAgent == 'transmission' and TransmissionClass !="": + TransmissionClass.stop_torrent(inputID) + if clientAgent == 'deluge' and delugeClient != "": + delugeClient.core.pause_torrent([inputID]) + time.sleep(5) # Give Torrent client some time to catch up with the change + + if config.issubsection(inputCategory,["SickBeard"): + fork, fork_params = autoFork("SickBeard", inputCategory) + Torrent_NoLink = int(config()["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 + if fork in config.SICKBEARD_TORRENT and Torrent_NoLink == 1: + Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) + result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) + if result != 0: + Logger.info("MAIN: A problem was reported in the autoProcessTV script.") + resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) + cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) + Logger.info("MAIN: All done.") + sys.exit() outputDestination = "" for category in categories: @@ -92,40 +131,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.info("MAIN: All done.") sys.exit() - # Hardlink solution for uTorrent, need to implent support for deluge, transmission - if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: - if clientAgent == 'utorrent': - try: - Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) - utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) - except: - Logger.exception("MAIN: Failed to connect to uTorrent") - - if clientAgent == 'transmission': - try: - Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) - TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) - except: - Logger.exception("MAIN: Failed to connect to Transmission") - - if clientAgent == 'deluge': - try: - Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, DelugeHost, DelugePort) - delugeClient = DelugeClient() - delugeClient.connect(host = DelugeHost, port = DelugePort, username = DelugeUSR, password = DelugePWD) - except: - Logger.exception("MAIN: Failed to connect to deluge") - - # if we are using links with uTorrent it means we need to pause it in order to access the files - Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.stop(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.stop_torrent(inputID) - if clientAgent == 'deluge' and delugeClient != "": - delugeClient.core.pause_torrent([inputID]) - time.sleep(5) # Give Torrent client some time to catch up with the change - Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory) if config().issubsection(inputCategory, "HeadPhones"): @@ -190,6 +195,14 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.exception("MAIN: Failed to link file: %s", file) continue elif fileExtension in compressedContainer: + if config().issubsection(inputCategory,["SickBeard"]) and config()["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination": + Logger.info("MAIN: Found archive file %s in %s", fileExtension, filePath) + try: + copy_link(filePath, targetDirectory, useLink, outputDestination) + copy_list.append([filePath, os.path.join(outputDestination, file)]) + except: + Logger.exception("MAIN: Failed to link file: %s", file) + continue # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. if re.search(r'\d+', os.path.splitext(fileName)[1]) and os.path.dirname(filePath) in extracted_folder and not any(item in os.path.splitext(fileName)[1] for item in ['.720p','.1080p','.x264']): part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) @@ -220,7 +233,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): flatten(outputDestination) # Now check if video files exist in destination: - if config().issubsection(inputCategory,["SickBeard","NzbDrone"]): + if config().issubsection(inputCategory,["SickBeard","NzbDrone","CouchPotato"]): for dirpath, dirnames, filenames in os.walk(outputDestination): for file in filenames: filePath = os.path.join(dirpath, file) @@ -276,6 +289,12 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): if result == 1: Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding") + resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) + cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) + Logger.info("MAIN: All done.") + +def resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) + # Hardlink solution for uTorrent, need to implent support for deluge, transmission if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: # Delete torrent and torrentdata from Torrent client if processing was successful. @@ -298,7 +317,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): if clientAgent == 'deluge' and delugeClient != "": delugeClient.core.resume_torrent([inputID]) time.sleep(5) - #cleanup + +def cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) if inputCategory in processCategories and result == 0 and os.path.isdir(outputDestination): num_files_new = int(0) file_list = [] @@ -316,9 +336,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.info("outputDirectory %s still contains %s media and/or meta files. This directory will not be removed.", outputDestination, num_files_new) for item in file_list: Logger.debug("media/meta file found: %s", item) - Logger.info("MAIN: All done.") -def external_script(outputDestination,torrentName,torrentLabel): +def external_script(outputDestination, torrentName, torrentLabel): final_result = int(0) # start at 0. num_files = int(0) diff --git a/nzbtomedia/nzbToMediaConfig.py b/nzbtomedia/nzbToMediaConfig.py index 6d074dec..70270646 100644 --- a/nzbtomedia/nzbToMediaConfig.py +++ b/nzbtomedia/nzbToMediaConfig.py @@ -249,6 +249,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "SickBeard" envCatKey = 'NZBPO_SBCATEGORY' @@ -263,6 +264,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "HeadPhones" envCatKey = 'NZBPO_HPCATEGORY' @@ -277,6 +279,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "Mylar" envCatKey = 'NZBPO_MYCATEGORY' @@ -291,6 +294,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "Gamez" envCatKey = 'NZBPO_GZCATEGORY' @@ -305,6 +309,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "NzbDrone" envCatKey = 'NZBPO_NDCATEGORY' @@ -319,6 +324,7 @@ class config(original_ConfigObj): if os.environ[envCatKey] not in config_new[section].sections: config_new[section][os.environ[envCatKey]] = {} config_new[section][os.environ[envCatKey]][option] = value + config_new[section][os.environ[envCatKey]]['enabled'] = 1 section = "Extensions" envKeys = ['COMPRESSEDEXTENSIONS', 'MEDIAEXTENSIONS', 'METAEXTENSIONS'] From 8132ab57697046411c5da37550e1c8a3d44cecb4 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 12:30:26 +0930 Subject: [PATCH 02/11] cleanup the way we handle torrents --- TorrentToMedia.py | 137 +++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index a9f38ee2..8e21d20f 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -35,52 +35,18 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): extracted_folder = [] extractionSuccess = False copy_list = [] - useLink = useLink_in file = None - delugeClient = "" - utorrentClass = "" - TransmissionClass = "" - Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure Logger.debug("MAIN: Determined Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) - # Hardlink solution for Torrents - if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: - if clientAgent == 'utorrent': - try: - Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) - utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) - except: - Logger.exception("MAIN: Failed to connect to uTorrent") + TorrentClass = create_torrent_class(clientAgent, inputHash) + pause_torrent(clientAgent, TorrentClass, inputHash, inputID, inputName) - if clientAgent == 'transmission': - try: - Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) - TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) - except: - Logger.exception("MAIN: Failed to connect to Transmission") - - if clientAgent == 'deluge': - try: - Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, DelugeHost, DelugePort) - delugeClient = DelugeClient() - delugeClient.connect(host = DelugeHost, port = DelugePort, username = DelugeUSR, password = DelugePWD) - except: - Logger.exception("MAIN: Failed to connect to deluge") - - # if we are using links with Torrents it means we need to pause it in order to access the files - Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.stop(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.stop_torrent(inputID) - if clientAgent == 'deluge' and delugeClient != "": - delugeClient.core.pause_torrent([inputID]) - time.sleep(5) # Give Torrent client some time to catch up with the change + processCategories = list(chain.from_iterable(subsections.values())) if config.issubsection(inputCategory,["SickBeard"): fork, fork_params = autoFork("SickBeard", inputCategory) @@ -90,8 +56,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) if result != 0: Logger.info("MAIN: A problem was reported in the autoProcessTV script.") - resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) - cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) + resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) + cleanup_output(inputCategory, processCategories, result, outputDestination) Logger.info("MAIN: All done.") sys.exit() @@ -253,8 +219,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): else: Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video)) - processCategories = list(chain.from_iterable(subsections.values())) - if (inputCategory in user_script_categories and not "NONE" in user_script_categories) or ("ALL" in user_script_categories and not inputCategory in processCategories): Logger.info("MAIN: Processing user script %s.", user_script) result = external_script(outputDestination,inputName,inputCategory) @@ -289,36 +253,74 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): if result == 1: Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding") - resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) - cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) + resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) + cleanup_output(inputCategory, processCategories, result, outputDestination) Logger.info("MAIN: All done.") -def resume(clientAgent, inputHash, inputID, deleteOriginal, result, useLink, utorrentClass, TransmissionClass, delugeClient) +def create_torrent_class(clientAgent, inputHash) # Hardlink solution for Torrents + TorrentClass = "" + if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: + if clientAgent == 'utorrent': + try: + Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui) + TorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD) + except: + Logger.exception("MAIN: Failed to connect to uTorrent") + + if clientAgent == 'transmission': + try: + Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort) + TorrentClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD) + except: + Logger.exception("MAIN: Failed to connect to Transmission") + + if clientAgent == 'deluge': + try: + Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, DelugeHost, DelugePort) + TorrentClass = DelugeClient() + TorrentClass.connect(host = DelugeHost, port = DelugePort, username = DelugeUSR, password = DelugePWD) + except: + Logger.exception("MAIN: Failed to connect to deluge") + + return TorrentClass + +def pause_torrent(clientAgent, TorrentClass, inputHash, inputID, inputName) + # if we are using links with Torrents it means we need to pause it in order to access the files + Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) + if clientAgent == 'utorrent' and TorrentClass != "": + TorrentClass.stop(inputHash) + if clientAgent == 'transmission' and TorrentClass !="": + TorrentClass.stop_torrent(inputID) + if clientAgent == 'deluge' and TorrentClass != "": + TorrentClass.core.pause_torrent([inputID]) + time.sleep(5) # Give Torrent client some time to catch up with the change + +def resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) # Hardlink solution for uTorrent, need to implent support for deluge, transmission if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: # Delete torrent and torrentdata from Torrent client if processing was successful. - if (deleteOriginal == 1 and result != 1) or useLink == 'move': # added uselink = move, if we move files, nothing to resume seeding. + if (int(config()["Torrent"]["deleteOriginal"]) is 1 and result != 1) or useLink == 'move': # if we move files, nothing to resume seeding. Logger.debug("MAIN: Deleting torrent %s from %s", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.removedata(inputHash) - utorrentClass.remove(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.remove_torrent(inputID, True) - if clientAgent == 'deluge' and delugeClient != "": - delugeClient.core.remove_torrent(inputID, True) + if clientAgent == 'utorrent' and TorrentClass != "": + TorrentClass.removedata(inputHash) + TorrentClass.remove(inputHash) + if clientAgent == 'transmission' and TorrentClass !="": + TorrentClass.remove_torrent(inputID, True) + if clientAgent == 'deluge' and TorrentClass != "": + TorrentClass.core.remove_torrent(inputID, True) # we always want to resume seeding, for now manually find out what is wrong when extraction fails else: Logger.debug("MAIN: Starting torrent %s in %s", inputName, clientAgent) - if clientAgent == 'utorrent' and utorrentClass != "": - utorrentClass.start(inputHash) - if clientAgent == 'transmission' and TransmissionClass !="": - TransmissionClass.start_torrent(inputID) - if clientAgent == 'deluge' and delugeClient != "": - delugeClient.core.resume_torrent([inputID]) + if clientAgent == 'utorrent' and TorrentClass != "": + TorrentClass.start(inputHash) + if clientAgent == 'transmission' and TorrentClass !="": + TorrentClass.start_torrent(inputID) + if clientAgent == 'deluge' and TorrentClass != "": + TorrentClass.core.resume_torrent([inputID]) time.sleep(5) -def cleanup_output(inputCategory, processCategories, result, outputDestination, mediaContainer, metaContainer) +def cleanup_output(inputCategory, processCategories, result, outputDestination) if inputCategory in processCategories and result == 0 and os.path.isdir(outputDestination): num_files_new = int(0) file_list = [] @@ -327,9 +329,9 @@ def cleanup_output(inputCategory, processCategories, result, outputDestination, filePath = os.path.join(dirpath, file) fileName, fileExtension = os.path.splitext(file) if fileExtension in mediaContainer or fileExtension in metaContainer: - num_files_new = num_files_new + 1 + num_files_new += 1 file_list.append(file) - if num_files_new == int(0) or forceClean == 1: + if num_files_new is 0 or int(config()["Torrent"]["forceClean"]) is 1: Logger.info("All files have been processed. Cleaning outputDirectory %s", outputDestination) shutil.rmtree(outputDestination) else: @@ -426,7 +428,7 @@ if __name__ == "__main__": # EXAMPLE VALUES: clientAgent = config()["Torrent"]["clientAgent"] # utorrent | deluge | transmission | rtorrent | other - useLink_in = config()["Torrent"]["useLink"] # no | hard | sym + useLink = config()["Torrent"]["useLink"] # no | hard | sym outputDirectory = config()["Torrent"]["outputDirectory"] # /abs/path/to/complete/ categories = (config()["Torrent"]["categories"]) # music,music_videos,pictures,software noFlatten = (config()["Torrent"]["noFlatten"]) @@ -445,14 +447,11 @@ if __name__ == "__main__": DelugeUSR = config()["Torrent"]["DelugeUSR"] # mysecretusr DelugePWD = config()["Torrent"]["DelugePWD"] # mysecretpwr - deleteOriginal = int(config()["Torrent"]["deleteOriginal"]) # 0 - forceClean = int(config()["Torrent"]["forceClean"]) # 0 - - compressedContainer = (config()["Extensions"]["compressedExtensions"]) # .zip,.rar,.7z - mediaContainer = (config()["Extensions"]["mediaExtensions"]) # .mkv,.avi,.divx - metaContainer = (config()["Extensions"]["metaExtensions"]) # .nfo,.sub,.srt + compressedContainer = (config()["Extensions"]["compressedExtensions"]) # .zip,.rar,.7z + mediaContainer = (config()["Extensions"]["mediaExtensions"]) # .mkv,.avi,.divx + metaContainer = (config()["Extensions"]["metaExtensions"]) # .nfo,.sub,.srt minSampleSize = int(config()["Extensions"]["minSampleSize"]) # 200 (in MB) - SampleIDs = (config()["Extensions"]["SampleIDs"]) # sample,-s. + SampleIDs = (config()["Extensions"]["SampleIDs"]) # sample,-s. subsections = config().get_subsections(["CouchPotato", "SickBeard", "NzbDrone", "HeadPhones", "Mylar", "Gamez"]) categories += list(chain.from_iterable(subsections.values())) From 4104aa91cd9dba6760d606cdb5fca1bd93c19cc3 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 12:38:56 +0930 Subject: [PATCH 03/11] format and context fixes --- TorrentToMedia.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 8e21d20f..345d3bf6 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -257,7 +257,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): cleanup_output(inputCategory, processCategories, result, outputDestination) Logger.info("MAIN: All done.") -def create_torrent_class(clientAgent, inputHash) # Hardlink solution for Torrents +def create_torrent_class(clientAgent, inputHash): + # Hardlink solution for Torrents TorrentClass = "" if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: if clientAgent == 'utorrent': @@ -284,19 +285,18 @@ def create_torrent_class(clientAgent, inputHash) # Hardlink solution for Torr return TorrentClass -def pause_torrent(clientAgent, TorrentClass, inputHash, inputID, inputName) - # if we are using links with Torrents it means we need to pause it in order to access the files - Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) - if clientAgent == 'utorrent' and TorrentClass != "": - TorrentClass.stop(inputHash) - if clientAgent == 'transmission' and TorrentClass !="": - TorrentClass.stop_torrent(inputID) - if clientAgent == 'deluge' and TorrentClass != "": - TorrentClass.core.pause_torrent([inputID]) - time.sleep(5) # Give Torrent client some time to catch up with the change - -def resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) +def pause_torrent(clientAgent, TorrentClass, inputHash, inputID, inputName): + # if we are using links with Torrents it means we need to pause it in order to access the files + Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent) + if clientAgent == 'utorrent' and TorrentClass != "": + TorrentClass.stop(inputHash) + if clientAgent == 'transmission' and TorrentClass !="": + TorrentClass.stop_torrent(inputID) + if clientAgent == 'deluge' and TorrentClass != "": + TorrentClass.core.pause_torrent([inputID]) + time.sleep(5) # Give Torrent client some time to catch up with the change +def resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName): # Hardlink solution for uTorrent, need to implent support for deluge, transmission if clientAgent in ['utorrent', 'transmission', 'deluge'] and inputHash: # Delete torrent and torrentdata from Torrent client if processing was successful. @@ -320,7 +320,7 @@ def resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputN TorrentClass.core.resume_torrent([inputID]) time.sleep(5) -def cleanup_output(inputCategory, processCategories, result, outputDestination) +def cleanup_output(inputCategory, processCategories, result, outputDestination): if inputCategory in processCategories and result == 0 and os.path.isdir(outputDestination): num_files_new = int(0) file_list = [] From 0857d8c7cab2038a764fd5ef8777e19b78837144 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 13:42:58 +0930 Subject: [PATCH 04/11] fix extractor path for windows --- nzbtomedia/extractor/extractor.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/nzbtomedia/extractor/extractor.py b/nzbtomedia/extractor/extractor.py index 6de9cf24..4c89e2dd 100644 --- a/nzbtomedia/extractor/extractor.py +++ b/nzbtomedia/extractor/extractor.py @@ -1,8 +1,9 @@ +import os import sys import logging from subprocess import call, Popen -from nzbtomedia.nzbToMediaConfig import * +from nzbtomedia.nzbToMediaConfig import config from nzbtomedia.nzbToMediaUtil import create_destination @@ -10,6 +11,7 @@ Logger = logging.getLogger() # which() and os_platform() breaks when running in Transmission (has to do with os.environ) + def os_platform(): # Author Credit: Matthew Scouten @ http://stackoverflow.com/a/7260315 true_platform = os.environ['PROCESSOR_ARCHITECTURE'] @@ -20,22 +22,6 @@ def os_platform(): #true_platform not assigned to if this does not exist return true_platform -def which(program): - # Author Credit: Jay @ http://stackoverflow.com/a/377028 - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None def extract(filePath, outputDestination): # Using Windows @@ -45,11 +31,11 @@ def extract(filePath, outputDestination): else: platform = 'x86' if not os.path.dirname(sys.argv[0]): - chplocation = os.path.normpath(os.path.join(os.getcwd(), 'extractor/bin/chp.exe')) - sevenzipLocation = os.path.normpath(os.path.join(os.getcwd(), 'extractor/bin/' + platform + '/7z.exe')) + chplocation = os.path.normpath(os.path.join(os.getcwd(), 'nzbtomedia/extractor/bin/chp.exe')) + sevenzipLocation = os.path.normpath(os.path.join(os.getcwd(), 'nzbtomedia/extractor/bin/' + platform + '/7z.exe')) else: - chplocation = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), 'extractor/bin/chp.exe')) - sevenzipLocation = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), 'extractor/bin/' + platform + '/7z.exe')) + chplocation = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), 'nzbtomedia/extractor/bin/chp.exe')) + sevenzipLocation = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), 'nzbtomedia/extractor/bin/' + platform + '/7z.exe')) if not os.path.exists(sevenzipLocation): Logger.error("EXTRACTOR: Could not find 7-zip, Exiting") return False From 3457fbf100f3d8def62039f831b079950d934eda Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 15:12:41 +0930 Subject: [PATCH 05/11] link all files unless archive. Cleanup flow. --- TorrentToMedia.py | 140 +++++++++++++++------------------------------- 1 file changed, 45 insertions(+), 95 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 345d3bf6..0cb0c64b 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -30,7 +30,6 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): status = int(1) # 1 = failed | 0 = success root = int(0) video = int(0) - video2 = int(0) foundFile = int(0) extracted_folder = [] extractionSuccess = False @@ -48,43 +47,24 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): processCategories = list(chain.from_iterable(subsections.values())) - if config.issubsection(inputCategory,["SickBeard"): - fork, fork_params = autoFork("SickBeard", inputCategory) - Torrent_NoLink = int(config()["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 - if fork in config.SICKBEARD_TORRENT and Torrent_NoLink == 1: - Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) - result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) - if result != 0: - Logger.info("MAIN: A problem was reported in the autoProcessTV script.") - resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) - cleanup_output(inputCategory, processCategories, result, outputDestination) - Logger.info("MAIN: All done.") - sys.exit() - outputDestination = "" - for category in categories: - if category == inputCategory: - if os.path.basename(inputDirectory) == inputName and os.path.isdir(inputDirectory): - Logger.info("MAIN: Download is a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, category, safeName(inputName))) - else: - Logger.info("MAIN: Download is not a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, category, os.path.splitext(safeName(inputName))[0])) - Logger.info("MAIN: Output directory set to: %s", outputDestination) - break - else: - continue + if inputCategory == "": + inputCategory = "UNCAT" + outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, safeName(inputName))) + Logger.info("MAIN: Output directory set to: %s", outputDestination) - if outputDestination == "": - if inputCategory == "": - inputCategory = "UNCAT" - if os.path.basename(inputDirectory) == inputName and os.path.isdir(inputDirectory): - Logger.info("MAIN: Download is a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, safeName(inputName))) - else: - Logger.info("MAIN: Download is not a directory") - outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, os.path.splitext(safeName(inputName))[0])) - Logger.info("MAIN: Output directory set to: %s", outputDestination) + if config.issubsection(inputCategory,["SickBeard"]): + fork, fork_params = autoFork("SickBeard", inputCategory) + Torrent_NoLink = int(config()["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 + if fork in config.SICKBEARD_TORRENT and Torrent_NoLink == 1: + Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) + result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) + if result != 0: + Logger.info("MAIN: A problem was reported in the autoProcessTV script.") + resume_torrent(clientAgent, TorrentClass, inputHash, inputID, result, inputName) + cleanup_output(inputCategory, processCategories, result, outputDestination) + Logger.info("MAIN: All done.") + sys.exit() processOnly = list(chain.from_iterable(subsections.values())) if not "NONE" in user_script_categories: # if None, we only process the 5 listed. @@ -130,7 +110,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): continue # This file does not match the Torrent name, skip it if root == 2: - Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") + if foundFile == int(0): + Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.") mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(dirpath, file))) ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, file))) if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): @@ -140,59 +121,34 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): else: continue # This file has not been recently moved or created, skip it - if fileExtension in mediaContainer: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize, SampleIDs) and not config().issubsection(inputCategory, ["HeadPhones"]): # Ignore samples - Logger.info("MAIN: Ignoring sample file: %s ", filePath) - continue - else: - video = video + 1 - Logger.info("MAIN: Found media file %s in %s", fileExtension, filePath) - try: - copy_link(filePath, targetDirectory, useLink, outputDestination) - copy_list.append([filePath, os.path.join(outputDestination, file)]) - except: - Logger.exception("MAIN: Failed to link file: %s", file) - elif fileExtension in metaContainer: - Logger.info("MAIN: Found metadata file %s for file %s", fileExtension, filePath) - try: - copy_link(filePath, targetDirectory, useLink, outputDestination) - copy_list.append([filePath, os.path.join(outputDestination, file)]) - except: - Logger.exception("MAIN: Failed to link file: %s", file) + if fileExtension in mediaContainer and is_sample(filePath, inputName, minSampleSize, SampleIDs) and not config().issubsection(inputCategory, ["HeadPhones"]): # Ignore samples + Logger.info("MAIN: Ignoring sample file: %s ", filePath) continue - elif fileExtension in compressedContainer: - if config().issubsection(inputCategory,["SickBeard"]) and config()["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination": - Logger.info("MAIN: Found archive file %s in %s", fileExtension, filePath) + + if fileExtension in compressedContainer: + if not (config().issubsection(inputCategory,["SickBeard"]) and config()["SickBeard"][inputCategory]["nzbExtractionBy"] == "Destination"): + # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. + if re.search(r'\d+', os.path.splitext(fileName)[1]) and os.path.dirname(filePath) in extracted_folder and not any(item in os.path.splitext(fileName)[1] for item in ['.720p','.1080p','.x264']): + part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) + if part == 1: # we only want to extract the primary part. + Logger.debug("MAIN: Found primary part of a multi-part archive %s. Extracting", file) + else: + Logger.debug("MAIN: Found part %s of a multi-part archive %s. Ignoring", part, file) + continue + Logger.info("MAIN: Found compressed archive %s for file %s", fileExtension, filePath) try: - copy_link(filePath, targetDirectory, useLink, outputDestination) - copy_list.append([filePath, os.path.join(outputDestination, file)]) + extractor.extract(filePath, outputDestination) + extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content) + extracted_folder.append(os.path.dirname(filePath)) except: - Logger.exception("MAIN: Failed to link file: %s", file) + Logger.exception("MAIN: Extraction failed for: %s", file) continue - # find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory. - if re.search(r'\d+', os.path.splitext(fileName)[1]) and os.path.dirname(filePath) in extracted_folder and not any(item in os.path.splitext(fileName)[1] for item in ['.720p','.1080p','.x264']): - part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group()) - if part == 1: # we only want to extract the primary part. - Logger.debug("MAIN: Found primary part of a multi-part archive %s. Extracting", file) - else: - Logger.debug("MAIN: Found part %s of a multi-part archive %s. Ignoring", part, file) - continue - Logger.info("MAIN: Found compressed archive %s for file %s", fileExtension, filePath) - try: - extractor.extract(filePath, outputDestination) - extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content) - extracted_folder.append(os.path.dirname(filePath)) - except: - Logger.exception("MAIN: Extraction failed for: %s", file) - continue - elif not config().issubsection(inputCategory,['CouchPotato','SickBeard','NzbDrone']): #process all for non-video categories. - Logger.info("MAIN: Found file %s for category %s", filePath, inputCategory) + + try: copy_link(filePath, targetDirectory, useLink, outputDestination) copy_list.append([filePath, os.path.join(outputDestination, file)]) - continue - else: - Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtension, filePath) - continue + except: + Logger.exception("MAIN: Failed to link file: %s", file) outputDestination = outputDestinationMaster # Reset here. if not inputCategory in noFlatten: #don't flatten hp in case multi cd albums, and we need to copy this back later. @@ -205,19 +161,13 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): filePath = os.path.join(dirpath, file) fileName, fileExtension = os.path.splitext(file) if fileExtension in mediaContainer: # If the file is a video file - if is_sample(filePath, inputName, minSampleSize, SampleIDs): - Logger.debug("MAIN: Removing sample file: %s", filePath) - os.unlink(filePath) # remove samples - else: - Logger.debug("MAIN: Found media file: %s", filePath) - video2 = video2 + 1 - else: - Logger.debug("MAIN: File %s is not a media file", filePath) - if video2 >= video and video2 > int(0): # Check that all video files were moved - Logger.debug("MAIN: Found %s media files", str(video2)) + Logger.debug("MAIN: Found media file: %s", filePath) + video += 1 + if video > int(0): # Check that media files exist + Logger.debug("MAIN: Found %s media files", str(video)) status = int(0) else: - Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video)) + Logger.warning("MAIN: Found no media files in output.", str(video)) if (inputCategory in user_script_categories and not "NONE" in user_script_categories) or ("ALL" in user_script_categories and not inputCategory in processCategories): Logger.info("MAIN: Processing user script %s.", user_script) From 432bc87d108b15205bdf18540f0084b0ed275cd2 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 16:13:13 +0930 Subject: [PATCH 06/11] change check for Torren_NoLink. fixes #314 --- TorrentToMedia.py | 3 +-- nzbtomedia/autoProcess/autoProcessTV.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 0cb0c64b..7476bd73 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -54,9 +54,8 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.info("MAIN: Output directory set to: %s", outputDestination) if config.issubsection(inputCategory,["SickBeard"]): - fork, fork_params = autoFork("SickBeard", inputCategory) Torrent_NoLink = int(config()["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 - if fork in config.SICKBEARD_TORRENT and Torrent_NoLink == 1: + if Torrent_NoLink == 1: Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) result = autoProcessTV().processEpisode(inputDirectory, inputName, 0, clientAgent=clientAgent, inputCategory=inputCategory) if result != 0: diff --git a/nzbtomedia/autoProcess/autoProcessTV.py b/nzbtomedia/autoProcess/autoProcessTV.py index c32b4661..656d82b7 100644 --- a/nzbtomedia/autoProcess/autoProcessTV.py +++ b/nzbtomedia/autoProcess/autoProcessTV.py @@ -28,10 +28,6 @@ class autoProcessTV: return 1 fork, fork_params = autoFork(section, inputCategory) - Torrent_NoLink = int(config()[section][inputCategory]["Torrent_NoLink"]) # 0 - if not fork in config.SICKBEARD_TORRENT and not Torrent_NoLink == 1: - if clientAgent in ['utorrent', 'transmission', 'deluge']: - return 1 socket.setdefaulttimeout(int(config.NZBTOMEDIA_TIMEOUT)) #initialize socket timeout. From d4cc70eaceb8431b63ede0a3d16ca27144700ba7 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 19:26:41 +0930 Subject: [PATCH 07/11] fixed config call. fixes 314 --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 7476bd73..36bc71e7 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -53,7 +53,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, safeName(inputName))) Logger.info("MAIN: Output directory set to: %s", outputDestination) - if config.issubsection(inputCategory,["SickBeard"]): + if config().issubsection(inputCategory,["SickBeard"]): Torrent_NoLink = int(config()["SickBeard"][inputCategory]["Torrent_NoLink"]) # 0 if Torrent_NoLink == 1: Logger.info("MAIN: Calling autoProcessTV to post-process: %s",inputName) From 1829fe05489658dfad88ca910088ca6d6982c8b0 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Wed, 9 Apr 2014 22:31:15 +0930 Subject: [PATCH 08/11] clean category_search --- nzbtomedia/nzbToMediaUtil.py | 171 ++++++++++------------------------- 1 file changed, 47 insertions(+), 124 deletions(-) diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index 3c9f2263..c7c613ed 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -54,149 +54,72 @@ def category_search(inputDirectory, inputName, inputCategory, root, categories): if inputDirectory is None: return inputDirectory, inputName, inputCategory, root - if not os.path.isdir(inputDirectory) and os.path.isfile(inputDirectory): # If the input directory is a file, assume single file downlaod and split dir/name. + catdir = False + tordir = False + + if not os.path.isdir(inputDirectory) and os.path.isfile(inputDirectory): # If the input directory is a file, assume single file download and split dir/name. inputDirectory,inputName = os.path.split(os.path.normpath(inputDirectory)) if inputCategory and os.path.isdir(os.path.join(inputDirectory, inputCategory)): Logger.info("SEARCH: Found category directory %s in input directory directory %s", inputCategory, inputDirectory) inputDirectory = os.path.join(inputDirectory, inputCategory) Logger.info("SEARCH: Setting inputDirectory to %s", inputDirectory) + catdir = True if inputName and os.path.isdir(os.path.join(inputDirectory, inputName)): Logger.info("SEARCH: Found torrent directory %s in input directory directory %s", inputName, inputDirectory) inputDirectory = os.path.join(inputDirectory, inputName) Logger.info("SEARCH: Setting inputDirectory to %s", inputDirectory) + tordir = True if inputName and os.path.isdir(os.path.join(inputDirectory, safeName(inputName))): Logger.info("SEARCH: Found torrent directory %s in input directory directory %s", safeName(inputName), inputDirectory) inputDirectory = os.path.join(inputDirectory, safeName(inputName)) Logger.info("SEARCH: Setting inputDirectory to %s", inputDirectory) - - categorySearch = [os.path.normpath(inputDirectory), ""] # initializie - notfound = 0 - unique = int(0) - for x in range(10): # loop up through 10 directories looking for category. - try: - categorySearch2 = os.path.split(os.path.normpath(categorySearch[0])) - except: # this might happen when we can't go higher. - if unique == int(0): - if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. - Logger.info("SEARCH: Could not find a category in the directory structure") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - break # we are done - elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. - Logger.info("SEARCH: Could not find a torrent name or category in the directory structure") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - break # we are done - elif inputName: # we didn't find category after 10 loops. This is a problem. - Logger.info("SEARCH: Could not find a category in the directory structure") - Logger.info("SEARCH: Files will be linked and will only be processed by the userscript if enabled for UNCAT or ALL") - root = 1 - break # we are done - else: # we didn't find this after 10 loops. This is a problem. - Logger.info("SEARCH: Could not identify category or torrent name from the directory structure.") - Logger.info("SEARCH: Files will be linked and will only be processed by the userscript if enabled for UNCAT or ALL") - root = 2 - break # we are done + tordir = True - if categorySearch2[1] in categories: - Logger.debug("SEARCH: Found Category: %s in directory structure", categorySearch2[1]) - if not inputCategory: - Logger.info("SEARCH: Determined Category to be: %s", categorySearch2[1]) - inputCategory = categorySearch2[1] - if inputName and categorySearch[0] != os.path.normpath(inputDirectory): # if we are not in the root directory and we have inputName we can continue. - if ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. - Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) - inputName = categorySearch[1] - Logger.info("SEARCH: Identified Category: %s and Torrent Name: %s. We are in a unique directory, so we can proceed.", inputCategory, inputName) - break # we are done - elif categorySearch[1] and not inputName: # assume the the next directory deep is the torrent name. - inputName = categorySearch[1] - Logger.info("SEARCH: Found torrent name: %s", categorySearch[1]) - if os.path.isdir(os.path.join(categorySearch[0], categorySearch[1])): - Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], categorySearch[1]), categorySearch[0]) - inputDirectory = os.path.normpath(os.path.join(categorySearch[0], categorySearch[1])) - elif os.path.isfile(os.path.join(categorySearch[0], categorySearch[1])): # Our inputdirectory is actually the full file path for single file download. - Logger.info("SEARCH: %s is a file, not a directory.", os.path.join(categorySearch[0], categorySearch[1])) - Logger.info("SEARCH: Setting input directory to %s", categorySearch[0]) - root = 1 - inputDirectory = os.path.normpath(categorySearch[0]) - else: # The inputdirectory given can't have been valid. Start at the category directory and search for date modified. - Logger.info("SEARCH: Input Directory %s doesn't exist as a directory or file", inputDirectory) - Logger.info("SEARCH: Setting input directory to %s and checking for files by date modified.", categorySearch[0]) - root = 2 - inputDirectory = os.path.normpath(categorySearch[0]) - break # we are done - elif ('.cp(tt' in categorySearch[1]) and (not '.cp(tt' in inputName): # if the directory was created by CouchPotato, and this tag is not in Torrent name, we want to add it. - Logger.info("SEARCH: Changing Torrent Name to %s to preserve imdb id.", categorySearch[1]) - inputName = categorySearch[1] - break # we are done - elif inputName and os.path.isdir(os.path.join(categorySearch[0], inputName)): # testing for torrent name in first sub directory - Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], inputName), categorySearch[0]) - if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 - inputDirectory = os.path.join(categorySearch[0], inputName) # we only want to search this next dir up. - break # we are done - elif inputName and os.path.isdir(os.path.join(categorySearch[0], safeName(inputName))): # testing for torrent name in first sub directory - Logger.info("SEARCH: Found torrent directory %s in category directory %s", os.path.join(categorySearch[0], safeName(inputName)), categorySearch[0]) - if categorySearch[0] == os.path.normpath(inputDirectory): # only true on first pass, x =0 - inputDirectory = os.path.join(categorySearch[0], safeName(inputName)) # we only want to search this next dir up. - break # we are done - elif inputName and os.path.isfile(os.path.join(categorySearch[0], inputName)) or os.path.isfile(os.path.join(categorySearch[0], safeName(inputName))): # testing for torrent name name as file inside category directory - Logger.info("SEARCH: Found torrent file %s in category directory %s", os.path.join(categorySearch[0], safeName(inputName)), categorySearch[0]) - root = 1 - inputDirectory = os.path.normpath(categorySearch[0]) - break # we are done - elif inputName: # if these exists, we are ok to proceed, but we are in a root/common directory. - Logger.info("SEARCH: Could not find a unique torrent folder in the directory structure") - Logger.info("SEARCH: The directory passed is the root directory for category %s", categorySearch2[1]) - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - break # we are done - else: # this is a problem! if we don't have Torrent name and are in the root category dir, we can't proceed. - Logger.warn("SEARCH: Could not identify a torrent name and the directory passed is common to all downloads for category %s.", categorySearch[1]) - Logger.warn("SEARCH: You should change settings to download torrents to their own directory if possible") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - break - elif inputName and safeName(categorySearch2[1]) == safeName(inputName) and os.path.isdir(categorySearch[0]): # we have identified a unique directory. - Logger.info("SEARCH: Files appear to be in their own directory") - unique = int(1) - if inputCategory: # we are ok to proceed. - break # we are done - else: - Logger.debug("SEARCH: Continuing scan to determin category.") - categorySearch = categorySearch2 # ready for next loop - continue # keep going + pathlist = os.path.normpath(inputDirectory).split(os.sep) + + if not catdir: + if inputCategory and inputCategory in pathlist: + Logger.debug("SEARCH: Found Category: %s in directory structure", inputCategory) + catdir = True else: - if x == 9: # This is the last pass in the loop and we didn't find anything. - notfound = 1 - break # we are done - else: - categorySearch = categorySearch2 # ready for next loop - continue # keep going - - if notfound == 1 and not unique == int(1): - if inputCategory and inputName: # if these exists, we are ok to proceed, but assume we are in a root/common directory. - Logger.info("SEARCH: Could not find a category in the directory structure") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 1 - elif inputCategory: # if this exists, we are ok to proceed, but assume we are in a root/common directory and we have to check file dates. - Logger.info("SEARCH: Could not find a torrent name or category in the directory structure") - Logger.info("SEARCH: We will try and determine which files to process, individually") - root = 2 - elif inputName: # we didn't find category after 10 loops. This is a problem. - Logger.info("SEARCH: Could not find a category in the directory structure") - Logger.info("SEARCH: Files will be linked and will only be processed by the userscript if enabled for UNCAT or ALL") - root = 1 - else: # we didn't find this after 10 loops. This is a problem. - Logger.info("SEARCH: Could not identify category or torrent name from the directory structure.") - Logger.info("SEARCH: Files will be linked and will only be processed by the userscript if enabled for UNCAT or ALL") - root = 2 + for item in pathlist: + if item in categories: + if not inputCategory: inputCategory = item + Logger.debug("SEARCH: Found Category: %s in directory structure", item) + catdir = True + break + if not inputCategory: + Logger.info("SEARCH: Could not find a category in the directory structure") + if not tordir: + if inputName and (inputName in pathlist or safeName(inputName) in pathlist): + Logger.info("SEARCH: Found torrent directory %s in the directory structure", inputName) + tordir = True + elif catdir: + try: + index = pathlist.index(inputCategory) + except ValueError: + index = 0 + if index != 0 and index < (len(pathlist)-1): # if there is another path after category. + tordir = True + if not inputName: inputName = pathlist[index+1] + Logger.info("SEARCH: Found a unique directory %s in the category directory", pathlist[index+1]) + else: + Logger.info("SEARCH: Could not find a unique directory for this download. Assume a common directory.") + Logger.info("SEARCH: We will try and determine which files to process, individually") + + if tordir: # in unique directory. + root = 0 + + elif inputName: # not unique directory, but we know the name to search for. + root = 1 + + else: # we need to search the input directory for recently changed files. + root = 2 + return inputDirectory, inputName, inputCategory, root - def is_sample(filePath, inputName, minSampleSize, SampleIDs): # 200 MB in bytes SIZE_CUTOFF = minSampleSize * 1024 * 1024 From 810f37edbf059cf12490f2ae267c85b849eeb293 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 10 Apr 2014 11:24:26 +0930 Subject: [PATCH 09/11] added single folder handling and improved category search --- TorrentToMedia.py | 8 +++- nzbtomedia/nzbToMediaUtil.py | 84 +++++++++++++++++------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 36bc71e7..8b5404e4 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -38,7 +38,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) - inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure + inputDirectory, inputName, inputCategory, root, single = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure Logger.debug("MAIN: Determined Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory) @@ -83,8 +83,12 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): outputDestinationMaster = outputDestination # Save the original, so we can change this within the loop below, and reset afterwards. now = datetime.datetime.now() + if single: inputDirectory,filename = os.path.split(inputDirectory) for dirpath, dirnames, filenames in os.walk(inputDirectory): - Logger.debug("MAIN: Found %s files in %s", str(len(filenames)), dirpath) + if single: + dirnames[:] = [] + filenames[:] = [filename] # we just want to work with this one file if single = True + Logger.debug("MAIN: Found %s files in %s", str(len(filenames)), dirpath)if for file in filenames: filePath = os.path.join(dirpath, file) fileName, fileExtension = os.path.splitext(file) diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index c7c613ed..909c3e27 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -51,20 +51,30 @@ def create_destination(outputDestination): sys.exit(-1) def category_search(inputDirectory, inputName, inputCategory, root, categories): - if inputDirectory is None: - return inputDirectory, inputName, inputCategory, root - - catdir = False + single = False tordir = False - if not os.path.isdir(inputDirectory) and os.path.isfile(inputDirectory): # If the input directory is a file, assume single file download and split dir/name. - inputDirectory,inputName = os.path.split(os.path.normpath(inputDirectory)) + if inputDirectory is None: # =Nothing to process here. + return inputDirectory, inputName, inputCategory, root, single + + pathlist = os.path.normpath(inputDirectory).split(os.sep) + + try: + inputCategory = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. + Logger.debug("SEARCH: Found Category: %s in directory structure", inputCategory) + except IndexError: + inputCategory = "" + Logger.debug("SEARCH: Could not find a category in the directory structure") + + if not os.path.isdir(inputDirectory) and os.path.isfile(inputDirectory): # If the input directory is a file + single = True + if not inputName: inputName = os.path.split(os.path.normpath(inputDirectory))[1] + return inputDirectory, inputName, inputCategory, root, single if inputCategory and os.path.isdir(os.path.join(inputDirectory, inputCategory)): Logger.info("SEARCH: Found category directory %s in input directory directory %s", inputCategory, inputDirectory) inputDirectory = os.path.join(inputDirectory, inputCategory) Logger.info("SEARCH: Setting inputDirectory to %s", inputDirectory) - catdir = True if inputName and os.path.isdir(os.path.join(inputDirectory, inputName)): Logger.info("SEARCH: Found torrent directory %s in input directory directory %s", inputName, inputDirectory) inputDirectory = os.path.join(inputDirectory, inputName) @@ -76,49 +86,35 @@ def category_search(inputDirectory, inputName, inputCategory, root, categories): Logger.info("SEARCH: Setting inputDirectory to %s", inputDirectory) tordir = True - pathlist = os.path.normpath(inputDirectory).split(os.sep) + imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. + if imdbid and not '.cp(tt' in inputName: + inputName = imdbid[0] # This ensures the imdb id is preserved and passed to CP + tordir = True - if not catdir: - if inputCategory and inputCategory in pathlist: - Logger.debug("SEARCH: Found Category: %s in directory structure", inputCategory) - catdir = True - else: - for item in pathlist: - if item in categories: - if not inputCategory: inputCategory = item - Logger.debug("SEARCH: Found Category: %s in directory structure", item) - catdir = True - break - if not inputCategory: - Logger.info("SEARCH: Could not find a category in the directory structure") + if inputCategory and not tordir: + try: + index = pathlist.index(inputCategory) + if index + 1 < len(pathlist): + tordir = True + Logger.info("SEARCH: Found a unique directory %s in the category directory", pathlist[index+1]) + if not inputName: inputName = pathlist[index+1] + except ValueError: + pass - if not tordir: - if inputName and (inputName in pathlist or safeName(inputName) in pathlist): + if inputName and not tordir: + if inputName in pathlist or safeName(inputName) in pathlist: Logger.info("SEARCH: Found torrent directory %s in the directory structure", inputName) tordir = True - elif catdir: - try: - index = pathlist.index(inputCategory) - except ValueError: - index = 0 - if index != 0 and index < (len(pathlist)-1): # if there is another path after category. - tordir = True - if not inputName: inputName = pathlist[index+1] - Logger.info("SEARCH: Found a unique directory %s in the category directory", pathlist[index+1]) else: - Logger.info("SEARCH: Could not find a unique directory for this download. Assume a common directory.") - Logger.info("SEARCH: We will try and determine which files to process, individually") - - if tordir: # in unique directory. - root = 0 - - elif inputName: # not unique directory, but we know the name to search for. - root = 1 - - else: # we need to search the input directory for recently changed files. + root = 1 + if not tordir: root = 2 - - return inputDirectory, inputName, inputCategory, root + + if root > 0: + Logger.info("SEARCH: Could not find a unique directory for this download. Assume a common directory.") + Logger.info("SEARCH: We will try and determine which files to process, individually") + + return inputDirectory, inputName, inputCategory, root, single def is_sample(filePath, inputName, minSampleSize, SampleIDs): # 200 MB in bytes From 41ac893c816be4703a68ce7ee7a0fa72e7b28e1b Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 10 Apr 2014 11:27:33 +0930 Subject: [PATCH 10/11] no need to preserve folder structure when single file. --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 8b5404e4..3fa4ea94 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -92,7 +92,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): for file in filenames: filePath = os.path.join(dirpath, file) fileName, fileExtension = os.path.splitext(file) - if inputCategory in noFlatten: + if inputCategory in noFlatten and not single: newDir = dirpath # find the full path newDir = newDir.replace(inputDirectory, "") #find the extra-depth directory if len(newDir) > 0 and newDir[0] == "/": From a273f9687fa66e0d13c0da4874767e2fb4e87d4d Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Thu, 10 Apr 2014 14:50:43 +0930 Subject: [PATCH 11/11] that was a stupid place to an if. --- TorrentToMedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 3fa4ea94..f9c32cea 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -88,7 +88,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID): if single: dirnames[:] = [] filenames[:] = [filename] # we just want to work with this one file if single = True - Logger.debug("MAIN: Found %s files in %s", str(len(filenames)), dirpath)if + Logger.debug("MAIN: Found %s files in %s", str(len(filenames)), dirpath) for file in filenames: filePath = os.path.join(dirpath, file) fileName, fileExtension = os.path.splitext(file)