Merge branch 'clinton-hall/dev'

Conflicts:
	autoProcess/autoProcessMovie.py
	autoProcess/autoProcessMusic.py
	autoProcess/autoProcessTV.py
	autoProcess/migratecfg.py
	autoProcess/nzbToMediaUtil.py

autoProcess/autoProcessComics.py

@@ -62,7 +62,7 @@ def processEpisode(dirName, nzbName=None, status=0, inputCategory=None):
         watch_dir = ""
     params = {}
 
-    nzbName, dirName = converto_to_ascii(nzbName, dirName)
+    nzbName, dirName = convert_to_ascii(nzbName, dirName)
 
     if dirName == "Manual Run" and watch_dir != "":
         dirName = watch_dir

autoProcess/autoProcessGames.py

@@ -45,7 +45,7 @@ def process(dirName, nzbName=None, status=0, inputCategory=None):
     else:
         protocol = "http://"
 
-    nzbName, dirName = converto_to_ascii(nzbName, dirName)
+    nzbName, dirName = convert_to_ascii(nzbName, dirName)
 
     baseURL = protocol + host + ":" + port + web_root + "/api?api_key=" + apikey + "&mode="
 

autoProcess/autoProcessMovie.py

@@ -181,22 +181,22 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id
     method = config().get(section, "method")
     delete_failed = int(config().get(section, "delete_failed"))
     wait_for = int(config().get(section, "wait_for"))
-
+    try:
+        TimePerGiB = int(config().get(section, "TimePerGiB"))
+    except (config.NoOptionError, ValueError):
+        TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute.
     try:
         ssl = int(config().get(section, "ssl"))
     except (config.NoOptionError, ValueError):
         ssl = 0
-
     try:
         web_root = config().get(section, "web_root")
     except config.NoOptionError:
         web_root = ""
-        
     try:    
         transcode = int(config().get("Transcoder", "transcode"))
     except (config.NoOptionError, ValueError):
         transcode = 0
-
     try:
         remoteCPS = int(config().get(section, "remoteCPS"))
     except (config.NoOptionError, ValueError):
@@ -219,10 +219,7 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id
     movie_id, imdbid, download_id, initial_status, initial_release_status = get_movie_info(baseURL, imdbid, download_id) # get the CPS database movie id for this movie.
     
     process_all_exceptions(nzbName.lower(), dirName)
-    nzbName, dirName = converto_to_ascii(nzbName, dirName)
+    nzbName, dirName = convert_to_ascii(nzbName, dirName)
-
-    TimeOut2 = int(wait_for) * 60 # If transfering files across directories, it now appears CouchPotato can take a while to confirm this url request... Try using wait_for timing.
-    socket.setdefaulttimeout(int(TimeOut2)) #initialize socket timeout. We may now be able to remove the delays from the wait_for section below?
 
     if status == 0:
         if transcode == 1:
@@ -242,6 +239,11 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id
                 else:
                     command = command + "/?media_folder=" + urllib.quote(dirName) + "&downloader=" + clientAgent + "&download_id=" + download_id
 
+        dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time.
+        TimeOut2 = int(TimePerGiB) * dirSize # Couchpotato needs to complete all moving and renaming before returning the status.
+        TimeOut2 += 60 # Add an extra minute for over-head/processing/metadata.
+        socket.setdefaulttimeout(int(TimeOut2)) #initialize socket timeout. We may now be able to remove the delays from the wait_for section below? If true, this should exit on first loop.
+
         url = baseURL + command
 
         Logger.info("Waiting for %s seconds to allow CPS to process newly extracted files", str(delay))

autoProcess/autoProcessMusic.py

@@ -7,7 +7,6 @@ from nzbToMediaUtil import *
 
 
 Logger = logging.getLogger()
-socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout.
 
 def process(dirName, nzbName=None, status=0, inputCategory=None):
 
@@ -35,12 +34,14 @@ def process(dirName, nzbName=None, status=0, inputCategory=None):
         ssl = int(config().get(section, "ssl"))
     except (config.NoOptionError, ValueError):
         ssl = 0
-
     try:
         web_root = config().get(section, "web_root")
     except config.NoOptionError:
         web_root = ""
-
+    try:
+        TimePerGiB = int(config().get(section, "TimePerGiB"))
+    except (config.NoOptionError, ValueError):
+        TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute.
     if ssl:
         protocol = "https://"
     else:
@@ -49,7 +50,12 @@ def process(dirName, nzbName=None, status=0, inputCategory=None):
     if nzbName == "Manual Run":
         delay = 0
 
-    nzbName, dirName = converto_to_ascii(nzbName, dirName)
+    nzbName, dirName = convert_to_ascii(nzbName, dirName)
+
+    dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time.
+    TimeOut = int(TimePerGiB) * dirSize # HeadPhones needs to complete all moving/transcoding and renaming before returning the status.
+    TimeOut += 60 # Add an extra minute for over-head/processing/metadata.
+    socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout.
 
     baseURL = protocol + host + ":" + port + web_root + "/api?apikey=" + apikey + "&cmd="
 

autoProcess/autoProcessTV.py

@@ -85,9 +85,9 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC
     except (config.NoOptionError, ValueError):
         delay = 0
     try:
-        wait_for = int(config().get(section, "wait_for"))
+        TimePerGiB = int(config().get(section, "TimePerGiB"))
     except (config.NoOptionError, ValueError):
-        wait_for = 5
+        TimePerGiB = 60 # note, if using Network to transfer on 100Mbit LAN, expect ~ 600 MB/minute.
     try:
         SampleIDs = (config().get("Extensions", "SampleIDs")).split(',')
     except (config.NoOptionError, ValueError):
@@ -100,10 +100,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC
         process_method = config().get(section, "process_method")
     except config.NoOptionError:
         process_method = None
-
+    
-    TimeOut = 60 * int(wait_for) # SickBeard needs to complete all moving and renaming before returning the log sequence via url.
-    socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout.
-
     mediaContainer = (config().get("Extensions", "mediaExtensions")).split(',')
     minSampleSize = int(config().get("Extensions", "minSampleSize"))
 
@@ -122,7 +119,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC
 
     if nzbName != "Manual Run" and (not fork in SICKBEARD_TORRENT or (clientAgent in ['nzbget','sabnzbd'] and not nzbExtractionBy == "Destination")):
         process_all_exceptions(nzbName.lower(), dirName)
-        nzbName, dirName = converto_to_ascii(nzbName, dirName)
+        nzbName, dirName = convert_to_ascii(nzbName, dirName)
 
         # Now check if movie files exist in destination. Eventually extraction may be done here if nzbExtractionBy == TorrentToMedia
         video = int(0)
@@ -146,6 +143,11 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC
     if watch_dir != "" and (not host in ['localhost', '127.0.0.1'] or nzbName == "Manual Run"):
         dirName = watch_dir
 
+    dirSize = getDirectorySize(dirName) # get total directory size to calculate needed processing time.
+    TimeOut = int(TimePerGiB) * dirSize # SickBeard needs to complete all moving and renaming before returning the log sequence via url.
+    TimeOut += 60 # Add an extra minute for over-head/processing/metadata.
+    socket.setdefaulttimeout(int(TimeOut)) #initialize socket timeout.
+
     # configure SB params to pass
     params['quiet'] = 1
     if nzbName is not None:
@@ -165,7 +167,7 @@ def processEpisode(dirName, nzbName=None, failed=False, clientAgent=None, inputC
                 del params[param]
 
     # delete any unused params so we don't pass them to SB by mistake
-    [params.pop(k) for k,v in params.iteritems() if v is None]
+    [params.pop(k) for k,v in params.items() if v is None]
 
     if status == 0:
         Logger.info("The download succeeded. Sending process request to SickBeard's %s branch", fork)

autoProcess/migratecfg.py

@@ -23,7 +23,9 @@ def migrate():
     for option, value in configold.items(section) or config(TV_CONFIG_FILE).items(section):
         if option is "category": # change this old format
             option = "sbCategory"
-        if option is "failed_fork": # change this old format
+        if option == "wait_for": # remove old format
+            continue
+        if option == "failed_fork": # change this old format
             option = "fork"
             if value not in ["default", "failed", "failed-torrent", "auto"]:
                 value = "auto"
@@ -118,15 +120,15 @@ def migrate():
             os.unlink(cfgbak_name)
         os.rename(CONFIG_FILE, cfgbak_name)
 
-    # writing our configuration file to 'autoProcessMedia.cfg.sample'
+    # writing our configuration file to 'autoProcessMedia.cfg'
     with open(CONFIG_FILE, 'wb') as configFile:
         confignew.write(configFile)
 
 def addnzbget():
     confignew = config()
     section = "CouchPotato"
-    envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'METHOD', 'DELETE_FAILED', 'REMOTECPS', 'WAIT_FOR']
+    envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'METHOD', 'DELETE_FAILED', 'REMOTECPS', 'WAIT_FOR', 'TIMEPERGIB']
-    cfgKeys = ['cpsCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'method', 'delete_failed', 'remoteCPS', 'wait_for']
+    cfgKeys = ['cpsCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'method', 'delete_failed', 'remoteCPS', 'wait_for', 'TimePerGiB']
     for index in range(len(envKeys)):
         key = 'NZBPO_CPS' + envKeys[index]
         if os.environ.has_key(key):
@@ -136,8 +138,8 @@ def addnzbget():
 
 
     section = "SickBeard"
-    envKeys = ['CATEGORY', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'DELAY', 'WAIT_FOR', 'PROCESS_METHOD']
+    envKeys = ['CATEGORY', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'DELAY', 'TIMEPERGIB', 'PROCESS_METHOD']
-    cfgKeys = ['sbCategory', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'delay', 'wait_for', 'process_method']
+    cfgKeys = ['sbCategory', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'delay', 'TimePerGiB', 'process_method']
     for index in range(len(envKeys)):
         key = 'NZBPO_SB' + envKeys[index]
         if os.environ.has_key(key):
@@ -146,8 +148,8 @@ def addnzbget():
             confignew.set(section, option, value)
 
     section = "HeadPhones"
-    envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY']
+    envKeys = ['CATEGORY', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'DELAY', 'TIMEPERGIB']
-    cfgKeys = ['hpCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay']
+    cfgKeys = ['hpCategory', 'apikey', 'host', 'port', 'ssl', 'web_root', 'delay', 'TimePerGiB']
     for index in range(len(envKeys)):
         key = 'NZBPO_HP' + envKeys[index]
         if os.environ.has_key(key):
@@ -205,6 +207,13 @@ def addnzbget():
             value = os.environ[key]
             confignew.set(section, option, value)
 
+    # create a backup of our old config
+    if os.path.isfile(CONFIG_FILE):
+        cfgbak_name = CONFIG_FILE + ".old"
+        if os.path.isfile(cfgbak_name):  # remove older backups
+            os.unlink(cfgbak_name)
+        os.rename(CONFIG_FILE, cfgbak_name)
+
     # writing our configuration file to 'autoProcessMedia.cfg'
     with open(CONFIG_FILE, 'wb') as configFile:
-        confignew.write(configFile)
+        confignew.write(configFile)

autoProcess/nzbToMediaUtil.py

@@ -13,6 +13,16 @@ from nzbToMediaConfig import *
 
 Logger = logging.getLogger()
 
+def getDirectorySize(directory):
+    dir_size = 0
+    for (path, dirs, files) in os.walk(directory):
+        for file in files:
+            filename = os.path.join(path, file)
+            dir_size += os.path.getsize(filename)
+    dir_size = dir_size / (1024.0 * 1024.0 * 1024.0) # convert to GiB
+    return dir_size
+
+
 def safeName(name):
     safename = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "", name) #make this name safe for use in directories for windows etc.
     return safename
@@ -339,7 +349,7 @@ def WakeUp():
     else:
         Logger.info("System with mac: %s has been woken. Continuing with the rest of the script.", mac)
 
-def converto_to_ascii(nzbName, dirName):
+def convert_to_ascii(nzbName, dirName):
     if not config():
         Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?")
         return nzbName, dirName

autoProcessMedia.cfg.sample

@@ -12,9 +12,10 @@ port = 5050
 ssl = 0
 web_root =
 delay = 65
+TimePerGiB = 60
 method = renamer
 delete_failed = 0
-wait_for = 5
+wait_for = 2
 #### Set to 1 if CouchPotatoServer is running on a different server to your NZB client
 remoteCPS = 0
 
@@ -31,7 +32,7 @@ password =
 web_root =
 ssl = 0
 delay = 0
-wait_for = 5
+TimePerGiB = 60
 watch_dir =
 fork = auto
 delete_failed = 0
@@ -51,6 +52,7 @@ port = 8181
 ssl = 0
 web_root =
 delay = 65
+TimePerGiB = 60
 
 
 [Mylar]

changelog.txt

@@ -7,6 +7,7 @@ Allow Headphones to remove torrents and data after processing.
 Delete torrent if uselink = move
 Added forceClean for outputDir. Works in file permissions prevent CP/SB from moving files.
 Ignore .x264 from archive "part" checks.
+Added dynamic timeout based on directory size.
 
 Impacts NZBs
 Fix setting of Mylar config from NZBGet.

nzbToCouchPotato.py

@@ -53,10 +53,15 @@
 # set to 1 to delete failed, or 0 to leave files in place.
 #cpsdelete_failed=0
 
+# CouchPotato process Time Per GiB
+#
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
+#cpsTimePerGiB=60
+
 # CouchPotato wait_for
 #
-# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy a movie.
+# Set the number of minutes to wait after calling the renamer, to check the movie has changed status.
-#cpswait_for=5
+#cpswait_for=2
 
 # CouchPotatoServer and NZBGet are a different system (0, 1).
 #

nzbToHeadPhones.py

@@ -43,6 +43,11 @@
 # set as required to ensure correct processing.
 #hpdelay=65
 
+# HeadPhones process Time Per GiB
+#
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
+#hpTimePerGiB=60
+
 ## WakeOnLan
 
 # use WOL (0, 1).

nzbToMedia.py

@@ -53,10 +53,15 @@
 # set to 1 to delete failed, or 0 to leave files in place.
 #cpsdelete_failed=0
 
+# CouchPotato process Time Per GiB
+#
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
+#cpsTimePerGiB=60
+
 # CouchPotato wait_for
 #
-# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy a movie.
+# Set the number of minutes to wait after calling the renamer, to check the movie has changed status.
-#cpswait_for=5
+#cpswait_for=2
 
 # CouchPotatoServer and NZBGet are a different system (0, 1).
 #
@@ -97,10 +102,10 @@
 # Set the number of seconds to wait before calling post-process in SickBeard.
 #sbdelay=0
 
-# SickBeard wait_for
+# SickBeard process Time Per GiB
 #
-# Set the number of minutes to wait before timing out. If transferring files across drives or network, increase this to longer than the time it takes to copy an episode.
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
-#sbwait_for=5
+#sbTimePerGiB=60
 
 # SickBeard watch directory.
 #
@@ -153,6 +158,11 @@
 # set as required to ensure correct processing.
 #hpdelay=65
 
+# HeadPhones process Time Per GiB
+#
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
+#hpTimePerGiB=60
+
 ## Mylar
 
 # Mylar script category.

nzbToSickBeard.py

@@ -46,10 +46,10 @@
 # Set the number of seconds to wait before calling post-process in SickBeard.
 #sbdelay=0
 
-# SickBeard wait_for
+# SickBeard process Time Per GiB
 #
-# Set the number of minutes to wait before timing out. If transfering files across drives or network, increase this to longer than the time it takes to copy an episode.
+# Set the number of seconds to wait, for each GiB of data, before timing out. If transfering files across drives or network, increase this value as needed.
-#sbwait_for=5
+#sbTimePerGiB=60
 
 # SickBeard watch directory.
 #