diff --git a/autoProcess/autoProcessComics.py b/autoProcess/autoProcessComics.py index 4c6d744e..f0cc2f82 100644 --- a/autoProcess/autoProcessComics.py +++ b/autoProcess/autoProcessComics.py @@ -62,7 +62,7 @@ def processEpisode(dirName, nzbName=None, status=0, inputCategory=None): watch_dir = "" params = {} - nzbName, dirName = converto_to_ascii(nzbName, dirName) + nzbName, dirName = convert_to_ascii(nzbName, dirName) if dirName == "Manual Run" and watch_dir != "": dirName = watch_dir diff --git a/autoProcess/autoProcessGames.py b/autoProcess/autoProcessGames.py index ccc989ad..f0779fa7 100644 --- a/autoProcess/autoProcessGames.py +++ b/autoProcess/autoProcessGames.py @@ -45,7 +45,7 @@ def process(dirName, nzbName=None, status=0, inputCategory=None): else: protocol = "http://" - nzbName, dirName = converto_to_ascii(nzbName, dirName) + nzbName, dirName = convert_to_ascii(nzbName, dirName) baseURL = protocol + host + ":" + port + web_root + "/api?api_key=" + apikey + "&mode=" diff --git a/autoProcess/autoProcessMovie.py b/autoProcess/autoProcessMovie.py index 3e9665b9..f74d9385 100644 --- a/autoProcess/autoProcessMovie.py +++ b/autoProcess/autoProcessMovie.py @@ -181,22 +181,22 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id method = config().get(section, "method") delete_failed = int(config().get(section, "delete_failed")) wait_for = int(config().get(section, "wait_for")) - + try: + TimePerGiB = int(config().get(section, "TimePerGiB")) + except (config.NoOptionError, ValueError): + TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute. try: ssl = int(config().get(section, "ssl")) except (config.NoOptionError, ValueError): ssl = 0 - try: web_root = config().get(section, "web_root") except config.NoOptionError: web_root = "" - try: transcode = int(config().get("Transcoder", "transcode")) except (config.NoOptionError, ValueError): transcode = 0 - try: remoteCPS = int(config().get(section, "remoteCPS")) except (config.NoOptionError, ValueError): @@ -219,10 +219,7 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id movie_id, imdbid, download_id, initial_status, initial_release_status = get_movie_info(baseURL, imdbid, download_id) # get the CPS database movie id for this movie. process_all_exceptions(nzbName.lower(), dirName) - nzbName, dirName = converto_to_ascii(nzbName, dirName) - - TimeOut2 = int(wait_for) * 60 # If transfering files across directories, it now appears CouchPotato can take a while to confirm this url request... Try using wait_for timing. - socket.setdefaulttimeout(int(TimeOut2)) #initialize socket timeout. We may now be able to remove the delays from the wait_for section below? + nzbName, dirName = convert_to_ascii(nzbName, dirName) if status == 0: if transcode == 1: @@ -242,6 +239,11 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id else: command = command + "/?media_folder=" + urllib.quote(dirName) + "&downloader=" + clientAgent + "&download_id=" + download_id + dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time. + TimeOut2 = int(TimePerGiB) * dirSize # Couchpotato needs to complete all moving and renaming before returning the status. + TimeOut2 += 60 # Add an extra minute for over-head/processing/metadata. + socket.setdefaulttimeout(int(TimeOut2)) #initialize socket timeout. We may now be able to remove the delays from the wait_for section below? If true, this should exit on first loop. + url = baseURL + command Logger.info("Waiting for %s seconds to allow CPS to process newly extracted files", str(delay)) diff --git a/autoProcess/autoProcessMusic.py b/autoProcess/autoProcessMusic.py index 95de8792..9c314ace 100644 --- a/autoProcess/autoProcessMusic.py +++ b/autoProcess/autoProcessMusic.py @@ -7,7 +7,6 @@ from nzbToMediaUtil import * Logger = logging.getLogger() -socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout. def process(dirName, nzbName=None, status=0, inputCategory=None): @@ -35,12 +34,14 @@ def process(dirName, nzbName=None, status=0, inputCategory=None): ssl = int(config().get(section, "ssl")) except (config.NoOptionError, ValueError): ssl = 0 - try: web_root = config().get(section, "web_root") except config.NoOptionError: web_root = "" - + try: + TimePerGiB = int(config().get(section, "TimePerGiB")) + except (config.NoOptionError, ValueError): + TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute. if ssl: protocol = "https://" else: @@ -49,7 +50,12 @@ def process(dirName, nzbName=None, status=0, inputCategory=None): if nzbName == "Manual Run": delay = 0 - nzbName, dirName = converto_to_ascii(nzbName, dirName) + nzbName, dirName = convert_to_ascii(nzbName, dirName) + + dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time. + TimeOut = int(TimePerGiB) * dirSize # HeadPhones needs to complete all moving/transcoding and renaming before returning the status. + TimeOut += 60 # Add an extra minute for over-head/processing/metadata. + socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout. baseURL = protocol + host + ":" + port + web_root + "/api?apikey=" + apikey + "&cmd=" diff --git a/autoProcess/autoProcessTV.py b/autoProcess/autoProcessTV.py index 3b7aca1c..aa1d77d3 100644 --- a/autoProcess/autoProcessTV.py +++ b/autoProcess/autoProcessTV.py @@ -85,9 +85,9 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC except (config.NoOptionError, ValueError): delay = 0 try: - wait_for = int(config().get(section, "wait_for")) + TimePerGiB = int(config().get(section, "TimePerGiB")) except (config.NoOptionError, ValueError): - wait_for = 5 + TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute. try: SampleIDs = (config().get("Extensions", "SampleIDs")).split(',') except (config.NoOptionError, ValueError): @@ -100,10 +100,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC process_method = config().get(section, "process_method") except config.NoOptionError: process_method = None - - TimeOut = 60 * int(wait_for) # SickBeard needs to complete all moving and renaming before returning the log sequence via url. - socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout. - + mediaContainer = (config().get("Extensions", "mediaExtensions")).split(',') minSampleSize = int(config().get("Extensions", "minSampleSize")) @@ -122,7 +119,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC if nzbName != "Manual Run" and (not fork in SICKBEARD_TORRENT or (clientAgent in ['nzbget','sabnzbd'] and not nzbExtractionBy == "Destination")): process_all_exceptions(nzbName.lower(), dirName) - nzbName, dirName = converto_to_ascii(nzbName, dirName) + nzbName, dirName = convert_to_ascii(nzbName, dirName) # Now check if movie files exist in destination. Eventually extraction may be done here if nzbExtractionBy == TorrentToMedia video = int(0) @@ -146,6 +143,11 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC if watch_dir != "" and (not host in ['localhost', '127.0.0.1'] or nzbName == "Manual Run"): dirName = watch_dir + dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time. + TimeOut = int(TimePerGiB) * dirSize # SickBeard needs to complete all moving and renaming before returning the log sequence via url. + TimeOut += 60 # Add an extra minute for over-head/processing/metadata. + socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout. + # configure SB params to pass params['quiet'] = 1 if nzbName is not None: @@ -165,7 +167,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC del params[param] # delete any unused params so we don't pass them to SB by mistake - [params.pop(k) for k,v in params.iteritems() if v is None] + [params.pop(k) for k,v in params.items() if v is None] if status == 0: Logger.info("The download succeeded. Sending process request to SickBeard's %s branch", fork) diff --git a/autoProcess/migratecfg.py b/autoProcess/migratecfg.py index b828e0b5..9aae5807 100644 --- a/autoProcess/migratecfg.py +++ b/autoProcess/migratecfg.py @@ -23,7 +23,9 @@ def migrate(): for option, value in configold.items(section) or config(TV_CONFIG_FILE).items(section): if option is "category": # change this old format option = "sbCategory" - if option is "failed_fork": # change this old format + if option == "wait_for": # remove old format + continue + if option == "failed_fork": # change this old format option = "fork" if value not in ["default", "failed", "failed-torrent", "auto"]: value = "auto" @@ -118,15 +120,15 @@ def migrate(): os.unlink(cfgbak_name) os.rename(CONFIG_FILE, cfgbak_name) - # writing our configuration file to 'autoProcessMedia.cfg.sample' + # writing our configuration file to 'autoProcessMedia.cfg' with open(CONFIG_FILE, 'wb') as configFile: confignew.write(configFile) def addnzbget(): confignew = config() section = "CouchPotato" - envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'METHOD', 'DELETE_FAILED', 'REMOTECPS', 'WAIT_FOR'] - cfgKeys = ['cpsCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'method', 'delete_failed', 'remoteCPS', 'wait_for'] + envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'METHOD', 'DELETE_FAILED', 'REMOTECPS', 'WAIT_FOR', 'TIMEPERGIB'] + cfgKeys = ['cpsCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'method', 'delete_failed', 'remoteCPS', 'wait_for', 'TimePerGiB'] for index in range(len(envKeys)): key = 'NZBPO_CPS' + envKeys[index] if os.environ.has_key(key): @@ -136,8 +138,8 @@ def addnzbget(): section = "SickBeard" - envKeys = ['CATEGORY', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'DELAY', 'WAIT_FOR', 'PROCESS_METHOD'] - cfgKeys = ['sbCategory', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'delay', 'wait_for', 'process_method'] + envKeys = ['CATEGORY', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'DELAY', 'TIMEPERGIB', 'PROCESS_METHOD'] + cfgKeys = ['sbCategory', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'delay', 'TimePerGiB', 'process_method'] for index in range(len(envKeys)): key = 'NZBPO_SB' + envKeys[index] if os.environ.has_key(key): @@ -146,8 +148,8 @@ def addnzbget(): confignew.set(section, option, value) section = "HeadPhones" - envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY'] - cfgKeys = ['hpCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay'] + envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'TIMEPERGIB'] + cfgKeys = ['hpCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'TimePerGiB'] for index in range(len(envKeys)): key = 'NZBPO_HP' + envKeys[index] if os.environ.has_key(key): @@ -205,6 +207,13 @@ def addnzbget(): value = os.environ[key] confignew.set(section, option, value) + # create a backup of our old config + if os.path.isfile(CONFIG_FILE): + cfgbak_name = CONFIG_FILE + ".old" + if os.path.isfile(cfgbak_name): # remove older backups + os.unlink(cfgbak_name) + os.rename(CONFIG_FILE, cfgbak_name) + # writing our configuration file to 'autoProcessMedia.cfg' with open(CONFIG_FILE, 'wb') as configFile: - confignew.write(configFile) \ No newline at end of file + confignew.write(configFile) diff --git a/autoProcess/nzbToMediaUtil.py b/autoProcess/nzbToMediaUtil.py index 43fbd73e..df51b484 100644 --- a/autoProcess/nzbToMediaUtil.py +++ b/autoProcess/nzbToMediaUtil.py @@ -13,6 +13,16 @@ from nzbToMediaConfig import * Logger = logging.getLogger() +def getDirectorySize(directory): + dir_size = 0 + for (path, dirs, files) in os.walk(directory): + for file in files: + filename = os.path.join(path, file) + dir_size += os.path.getsize(filename) + dir_size = dir_size / (1024.0 * 1024.0 * 1024.0) # convert to GiB + return dir_size + + def safeName(name): safename = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "", name) #make this name safe for use in directories for windows etc. return safename @@ -339,7 +349,7 @@ def WakeUp(): else: Logger.info("System with mac: %s has been woken. Continuing with the rest of the script.", mac) -def converto_to_ascii(nzbName, dirName): +def convert_to_ascii(nzbName, dirName): if not config(): Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?") return nzbName, dirName diff --git a/autoProcessMedia.cfg.sample b/autoProcessMedia.cfg.sample index f6570b17..97731862 100644 --- a/autoProcessMedia.cfg.sample +++ b/autoProcessMedia.cfg.sample @@ -12,9 +12,10 @@ port = 5050 ssl = 0 web_root = delay = 65 +TimePerGiB = 60 method = renamer delete_failed = 0 -wait_for = 5 +wait_for = 2 #### Set to 1 if CouchPotatoServer is running on a different server to your NZB client remoteCPS = 0 @@ -31,7 +32,7 @@ password = web_root = ssl = 0 delay = 0 -wait_for = 5 +TimePerGiB = 60 watch_dir = fork = auto delete_failed = 0 @@ -51,6 +52,7 @@ port = 8181 ssl = 0 web_root = delay = 65 +TimePerGiB = 60 [Mylar] diff --git a/changelog.txt b/changelog.txt index c221969a..9e9c14c2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -7,6 +7,7 @@ Allow Headphones to remove torrents and data after processing. Delete torrent if uselink = move Added forceClean for outputDir. Works in file permissions prevent CP/SB from moving files. Ignore .x264 from archive "part" checks. +Added dynamic timeout based on directory size. Impacts NZBs Fix setting of Mylar config from NZBGet. diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 6370bf0a..81a50beb 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -53,10 +53,15 @@ # set to 1 to delete failed, or 0 to leave files in place. #cpsdelete_failed=0 +# CouchPotato process Time Per GiB +# +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#cpsTimePerGiB=60 + # CouchPotato wait_for # -# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy a movie. -#cpswait_for=5 +# Set the number of minutes to wait after calling the renamer, to check the movie has changed status. +#cpswait_for=2 # CouchPotatoServer and NZBGet are a different system (0, 1). # diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index d4916f0c..e5edab40 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -43,6 +43,11 @@ # set as required to ensure correct processing. #hpdelay=65 +# HeadPhones process Time Per GiB +# +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#hpTimePerGiB=60 + ## WakeOnLan # use WOL (0, 1). diff --git a/nzbToMedia.py b/nzbToMedia.py index 881cd8e2..538145f6 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -53,10 +53,15 @@ # set to 1 to delete failed, or 0 to leave files in place. #cpsdelete_failed=0 +# CouchPotato process Time Per GiB +# +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#cpsTimePerGiB=60 + # CouchPotato wait_for # -# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy a movie. -#cpswait_for=5 +# Set the number of minutes to wait after calling the renamer, to check the movie has changed status. +#cpswait_for=2 # CouchPotatoServer and NZBGet are a different system (0, 1). # @@ -97,10 +102,10 @@ # Set the number of seconds to wait before calling post-process in SickBeard. #sbdelay=0 -# SickBeard wait_for +# SickBeard process Time Per GiB # -# Set the number of minutes to wait before timing out. If transferring files across drives or network, increase this to longer than the time it takes to copy an episode. -#sbwait_for=5 +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#sbTimePerGiB=60 # SickBeard watch directory. # @@ -153,6 +158,11 @@ # set as required to ensure correct processing. #hpdelay=65 +# HeadPhones process Time Per GiB +# +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#hpTimePerGiB=60 + ## Mylar # Mylar script category. diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 17f40e9b..b330abe1 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -46,10 +46,10 @@ # Set the number of seconds to wait before calling post-process in SickBeard. #sbdelay=0 -# SickBeard wait_for +# SickBeard process Time Per GiB # -# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy an episode. -#sbwait_for=5 +# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed. +#sbTimePerGiB=60 # SickBeard watch directory. #