add category centric processing. fixes #230

This commit is contained in:
clinton-hall 2014-01-07 21:22:47 +10:30
parent c98604c618
commit 6fe287df0d
8 changed files with 135 additions and 95 deletions

View file

@ -9,7 +9,6 @@ import logging
import datetime
import time
import re
from sets import Set
from subprocess import call, Popen
# Custom imports
@ -37,7 +36,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
copy_list = []
Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory)
if inputCategory == sbCategory and sbFork in SICKBEARD_TORRENT:
if inputCategory in sbCategory and sbFork in SICKBEARD_TORRENT:
Logger.info("MAIN: Calling SickBeard's %s branch to post-process: %s",sbFork ,inputName)
result = autoProcessTV.processEpisode(inputDirectory, inputName, int(0))
if result == 1:
@ -71,7 +70,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, os.path.splitext(safeName(inputName))[0]))
Logger.info("MAIN: Output directory set to: %s", outputDestination)
processOnly = [cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory]
processOnly = cpsCategory + sbCategory + hpCategory + mlCategory + gzCategory
if not "NONE" in user_script_categories: # if None, we only process the 5 listed.
if "ALL" in user_script_categories: # All defined categories
processOnly = categories
@ -139,7 +138,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
continue # This file has not been recently moved or created, skip it
if fileExtension in mediaContainer: # If the file is a video file
if is_sample(filePath, inputName, minSampleSize) and not inputCategory == hpCategory: # Ignore samples
if is_sample(filePath, inputName, minSampleSize) and not inputCategory in hpCategory: # Ignore samples
Logger.info("MAIN: Ignoring sample file: %s ", filePath)
continue
else:
@ -159,7 +158,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
Logger.exception("MAIN: Failed to link file: %s", file)
continue
elif fileExtension in compressedContainer:
if inputCategory in [hpCategory]: # We need to link all files for HP in order to move these back to support seeding.
if inputCategory in hpCategory: # We need to link all files for HP in order to move these back to support seeding.
Logger.info("MAIN: Linking compressed archive file %s for file %s", fileExtension, filePath)
try:
copy_link(filePath, targetDirectory, useLink, outputDestination)
@ -176,7 +175,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
continue
Logger.info("MAIN: Found compressed archive %s for file %s", fileExtension, filePath)
try:
if inputCategory == hpCategory: # HP needs to scan the same dir as passed to downloader.
if inputCategory in hpCategory: # HP needs to scan the same dir as passed to downloader.
extractor.extract(filePath, inputDirectory)
else:
extractor.extract(filePath, outputDestination)
@ -185,7 +184,7 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
except:
Logger.exception("MAIN: Extraction failed for: %s", file)
continue
elif not inputCategory in [cpsCategory, sbCategory]: #process all for non-video categories.
elif (not inputCategory in cpsCategory) and (not inputCategory in sbCategory): #process all for non-video categories.
Logger.info("MAIN: Found file %s for category %s", filePath, inputCategory)
copy_link(filePath, targetDirectory, useLink, outputDestination)
copy_list.append([filePath, os.path.join(outputDestination, file)])
@ -193,11 +192,11 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
else:
Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtension, filePath)
continue
if not inputCategory in [hpCategory]: #don't flatten hp in case multi cd albums, and we need to copy this back later.
if not inputCategory in hpCategory: #don't flatten hp in case multi cd albums, and we need to copy this back later.
flatten(outputDestination)
# Now check if movie files exist in destination:
if inputCategory in [cpsCategory, sbCategory]:
if inputCategory in cpsCategory + sbCategory:
for dirpath, dirnames, filenames in os.walk(outputDestination):
for file in filenames:
filePath = os.path.join(dirpath, file)
@ -217,39 +216,39 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
else:
Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video))
processCategories = Set([cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory])
processCategories = cpsCategory + sbCategory + hpCategory + mlCategory + gzCategory
if (inputCategory in user_script_categories and not "NONE" in user_script_categories) or ("ALL" in user_script_categories and not inputCategory in processCategories):
Logger.info("MAIN: Processing user script %s.", user_script)
result = external_script(outputDestination)
elif status == int(0) or (inputCategory in [hpCategory, mlCategory, gzCategory]): # if movies linked/extracted or for other categories.
elif status == int(0) or (inputCategory in hpCategory + mlCategory + gzCategory): # if movies linked/extracted or for other categories.
Logger.debug("MAIN: Calling autoProcess script for successful download.")
status = int(0) # hp, my, gz don't support failed.
else:
Logger.error("MAIN: Something failed! Please check logs. Exiting")
sys.exit(-1)
if inputCategory == cpsCategory:
if inputCategory in cpsCategory:
Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName)
download_id = inputHash
result = autoProcessMovie.process(outputDestination, inputName, status, clientAgent, download_id)
elif inputCategory == sbCategory:
result = autoProcessMovie.process(outputDestination, inputName, status, clientAgent, download_id, inputCategory)
elif inputCategory in sbCategory:
Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName)
result = autoProcessTV.processEpisode(outputDestination, inputName, status)
elif inputCategory == hpCategory:
result = autoProcessTV.processEpisode(outputDestination, inputName, status, inputCategory)
elif inputCategory in hpCategory:
Logger.info("MAIN: Calling HeadPhones to post-process: %s", inputName)
result = autoProcessMusic.process(inputDirectory, inputName, status)
elif inputCategory == mlCategory:
result = autoProcessMusic.process(inputDirectory, inputName, status, inputCategory)
elif inputCategory in mlCategory:
Logger.info("MAIN: Calling Mylar to post-process: %s", inputName)
result = autoProcessComics.processEpisode(outputDestination, inputName, status)
elif inputCategory == gzCategory:
result = autoProcessComics.processEpisode(outputDestination, inputName, status, inputCategory)
elif inputCategory in gzCategory:
Logger.info("MAIN: Calling Gamez to post-process: %s", inputName)
result = autoProcessGames.process(outputDestination, inputName, status)
result = autoProcessGames.process(outputDestination, inputName, status, inputCategory)
if result == 1:
Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding")
if inputCategory == hpCategory:
if inputCategory in hpCategory:
# we need to move the output dir files back...
Logger.debug("MAIN: Moving temporary HeadPhones files back to allow seeding.")
for item in copy_list:
@ -270,10 +269,10 @@ def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
Logger.debug("MAIN: Deleting torrent %s from %s", inputName, clientAgent)
if clientAgent == 'utorrent' and utorrentClass != "":
utorrentClass.removedata(inputHash)
if not inputCategory == hpCategory:
if not inputCategory in hpCategory:
utorrentClass.remove(inputHash)
if clientAgent == 'transmission' and TransmissionClass !="":
if inputCategory == hpCategory: #don't delete actual files for hp category, just remove torrent.
if inputCategory in hpCategory: #don't delete actual files for hp category, just remove torrent.
TransmissionClass.remove_torrent(inputID, False)
else:
TransmissionClass.remove_torrent(inputID, True)
@ -412,17 +411,17 @@ if __name__ == "__main__":
metaContainer = (config.get("Extensions", "metaExtensions")).split(',') # .nfo,.sub,.srt
minSampleSize = int(config.get("Extensions", "minSampleSize")) # 200 (in MB)
cpsCategory = config.get("CouchPotato", "cpsCategory") # movie
sbCategory = config.get("SickBeard", "sbCategory") # tv
cpsCategory = (config.get("CouchPotato", "cpsCategory")).split(',') # movie
sbCategory = (config.get("SickBeard", "sbCategory")).split(',') # tv
sbFork = config.get("SickBeard", "fork") # tv
hpCategory = config.get("HeadPhones", "hpCategory") # music
mlCategory = config.get("Mylar", "mlCategory") # comics
gzCategory = config.get("Gamez", "gzCategory") # games
categories.append(cpsCategory)
categories.append(sbCategory)
categories.append(hpCategory)
categories.append(mlCategory)
categories.append(gzCategory)
hpCategory = (config.get("HeadPhones", "hpCategory")).split(',') # music
mlCategory = (config.get("Mylar", "mlCategory")).split(',') # comics
gzCategory = (config.get("Gamez", "gzCategory")).split(',') # games
categories.extend(cpsCategory)
categories.extend(sbCategory)
categories.extend(hpCategory)
categories.extend(mlCategory)
categories.extend(gzCategory)
user_script_categories = config.get("UserScript", "user_script_categories").split(',') # NONE
if not "NONE" in user_script_categories:

View file

@ -30,7 +30,7 @@ class AuthURLOpener(urllib.FancyURLopener):
return urllib.FancyURLopener.open(self, url)
def processEpisode(dirName, nzbName=None, status=0):
def processEpisode(dirName, nzbName=None, status=0, inputCategory=None):
config = ConfigParser.ConfigParser()
configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg")
@ -41,23 +41,26 @@ def processEpisode(dirName, nzbName=None, status=0):
return 1 # failure
config.read(configFilename)
host = config.get("Mylar", "host")
port = config.get("Mylar", "port")
username = config.get("Mylar", "username")
password = config.get("Mylar", "password")
section = "Mylar"
if inputCategory != None and config.has_section(inputCategory):
section = inputCategory
host = config.get(section, "host")
port = config.get(section, "port")
username = config.get(section, "username")
password = config.get(section, "password")
try:
ssl = int(config.get("Mylar", "ssl"))
ssl = int(config.get(section, "ssl"))
except (ConfigParser.NoOptionError, ValueError):
ssl = 0
try:
web_root = config.get("Mylar", "web_root")
web_root = config.get(section, "web_root")
except ConfigParser.NoOptionError:
web_root = ""
try:
watch_dir = config.get("Mylar", "watch_dir")
watch_dir = config.get(section, "watch_dir")
except ConfigParser.NoOptionError:
watch_dir = ""
params = {}

View file

@ -13,7 +13,7 @@ from nzbToMediaUtil import *
Logger = logging.getLogger()
def process(dirName, nzbName=None, status=0):
def process(dirName, nzbName=None, status=0, inputCategory=None):
status = int(status)
config = ConfigParser.ConfigParser()
@ -26,17 +26,21 @@ def process(dirName, nzbName=None, status=0):
config.read(configFilename)
host = config.get("Gamez", "host")
port = config.get("Gamez", "port")
apikey = config.get("Gamez", "apikey")
section = "Gamez"
if inputCategory != None and config.has_section(inputCategory):
section = inputCategory
host = config.get(section, "host")
port = config.get(section, "port")
apikey = config.get(section, "apikey")
try:
ssl = int(config.get("Gamez", "ssl"))
ssl = int(config.get(section, "ssl"))
except (ConfigParser.NoOptionError, ValueError):
ssl = 0
try:
web_root = config.get("Gamez", "web_root")
web_root = config.get(section, "web_root")
except ConfigParser.NoOptionError:
web_root = ""

View file

@ -190,7 +190,7 @@ def get_status(baseURL, movie_id, clientAgent, download_id):
download_id = "none"
return movie_status, clientAgent, download_id, release_status
def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id = ""):
def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id = "", inputCategory=None):
status = int(status)
config = ConfigParser.ConfigParser()
@ -203,21 +203,25 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id
config.read(configFilename)
host = config.get("CouchPotato", "host")
port = config.get("CouchPotato", "port")
apikey = config.get("CouchPotato", "apikey")
delay = float(config.get("CouchPotato", "delay"))
method = config.get("CouchPotato", "method")
delete_failed = int(config.get("CouchPotato", "delete_failed"))
wait_for = int(config.get("CouchPotato", "wait_for"))
section = "CouchPotato"
if inputCategory != None and config.has_section(inputCategory):
section = inputCategory
host = config.get(section., "host")
port = config.get(section., "port")
apikey = config.get(section., "apikey")
delay = float(config.get(section., "delay"))
method = config.get(section., "method")
delete_failed = int(config.get(section., "delete_failed"))
wait_for = int(config.get(section., "wait_for"))
try:
ssl = int(config.get("CouchPotato", "ssl"))
ssl = int(config.get(section., "ssl"))
except (ConfigParser.NoOptionError, ValueError):
ssl = 0
try:
web_root = config.get("CouchPotato", "web_root")
web_root = config.get(section., "web_root")
except ConfigParser.NoOptionError:
web_root = ""
@ -227,7 +231,7 @@ def process(dirName, nzbName=None, status=0, clientAgent = "manual", download_id
transcode = 0
try:
remoteCPS = int(config.get("CouchPotato", "remoteCPS"))
remoteCPS = int(config.get(section., "remoteCPS"))
except (ConfigParser.NoOptionError, ValueError):
remoteCPS = 0

View file

@ -13,7 +13,7 @@ from nzbToMediaUtil import *
Logger = logging.getLogger()
def process(dirName, nzbName=None, status=0):
def process(dirName, nzbName=None, status=0, inputCategory=None):
status = int(status)
config = ConfigParser.ConfigParser()
@ -26,18 +26,22 @@ def process(dirName, nzbName=None, status=0):
config.read(configFilename)
host = config.get("HeadPhones", "host")
port = config.get("HeadPhones", "port")
apikey = config.get("HeadPhones", "apikey")
delay = float(config.get("HeadPhones", "delay"))
section = "HeadPhones"
if inputCategory != None and config.has_section(inputCategory):
section = inputCategory
host = config.get(section., "host")
port = config.get(section., "port")
apikey = config.get(section., "apikey")
delay = float(config.get(section., "delay"))
try:
ssl = int(config.get("HeadPhones", "ssl"))
ssl = int(config.get(section., "ssl"))
except (ConfigParser.NoOptionError, ValueError):
ssl = 0
try:
web_root = config.get("HeadPhones", "web_root")
web_root = config.get(section., "web_root")
except ConfigParser.NoOptionError:
web_root = ""

View file

@ -41,7 +41,7 @@ def delete(dirName):
Logger.exception("Unable to delete folder %s", dirName)
def processEpisode(dirName, nzbName=None, failed=False):
def processEpisode(dirName, nzbName=None, failed=False, inputCategory=None):
status = int(failed)
config = ConfigParser.ConfigParser()
@ -54,28 +54,32 @@ def processEpisode(dirName, nzbName=None, failed=False):
config.read(configFilename)
section = "SickBeard"
if inputCategory != None and config.has_section(inputCategory):
section = inputCategory
watch_dir = ""
host = config.get("SickBeard", "host")
port = config.get("SickBeard", "port")
username = config.get("SickBeard", "username")
password = config.get("SickBeard", "password")
host = config.get(section., "host")
port = config.get(section., "port")
username = config.get(section., "username")
password = config.get(section., "password")
try:
ssl = int(config.get("SickBeard", "ssl"))
ssl = int(config.get(section., "ssl"))
except (ConfigParser.NoOptionError, ValueError):
ssl = 0
try:
web_root = config.get("SickBeard", "web_root")
web_root = config.get(section., "web_root")
except ConfigParser.NoOptionError:
web_root = ""
try:
watch_dir = config.get("SickBeard", "watch_dir")
watch_dir = config.get(section., "watch_dir")
except ConfigParser.NoOptionError:
watch_dir = ""
try:
fork = config.get("SickBeard", "fork")
fork = config.get(section., "fork")
except ConfigParser.NoOptionError:
fork = "default"
@ -85,11 +89,11 @@ def processEpisode(dirName, nzbName=None, failed=False):
transcode = 0
try:
delete_failed = int(config.get("SickBeard", "delete_failed"))
delete_failed = int(config.get(section., "delete_failed"))
except (ConfigParser.NoOptionError, ValueError):
delete_failed = 0
try:
delay = float(config.get("SickBeard", "delay"))
delay = float(config.get(section., "delay"))
except (ConfigParser.NoOptionError, ValueError):
delay = 0

View file

@ -12,6 +12,8 @@ def migrate():
configold = ConfigParser.ConfigParser()
configold.optionxform = str
categories = []
section = "CouchPotato"
original = []
configFilenameold = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg")
@ -35,6 +37,8 @@ def migrate():
continue
if option in ["username", "password" ]: # these are no-longer needed.
continue
if option == "cpsCategory":
categories.extend(value.split(','))
confignew.set(section, option, value)
section = "SickBeard"
@ -64,6 +68,8 @@ def migrate():
value = os.path.split(os.path.normpath(value))[0]
confignew.set("Torrent", option, value)
continue
if option == "sbCategory":
categories.extend(value.split(','))
confignew.set(section, option, value)
section = "HeadPhones"
@ -79,6 +85,8 @@ def migrate():
if option in ["username", "password" ]: # these are no-longer needed.
continue
option, value = item
if option == "hpCategory":
categories.extend(value.split(','))
confignew.set(section, option, value)
section = "Mylar"
@ -89,6 +97,8 @@ def migrate():
pass
for item in original:
option, value = item
if option == "mlCategory":
categories.extend(value.split(','))
confignew.set(section, option, value)
section = "Gamez"
@ -98,11 +108,23 @@ def migrate():
except:
pass
for item in original:
option, value = item
if option in ["username", "password" ]: # these are no-longer needed.
continue
option, value = item
if option == "gzCategory":
categories.extend(value.split(','))
confignew.set(section, option, value)
for section in categories:
original = []
try:
original = configold.items(section)
except:
continue
for item in original:
option, value = item
confignew.set(section, option, value)
section = "Torrent"
original = []
try:

View file

@ -279,11 +279,11 @@ if not os.path.isfile(configFilename):
Logger.info("MAIN: Loading config from %s", configFilename)
config.read(configFilename)
cpsCategory = config.get("CouchPotato", "cpsCategory") # movie
sbCategory = config.get("SickBeard", "sbCategory") # tv
hpCategory = config.get("HeadPhones", "hpCategory") # music
mlCategory = config.get("Mylar", "mlCategory") # comics
gzCategory = config.get("Gamez", "gzCategory") # games
cpsCategory = (config.get("CouchPotato", "cpsCategory")).split(',') # movie
sbCategory = (config.get("SickBeard", "sbCategory")).split(',') # tv
hpCategory = (config.get("HeadPhones", "hpCategory")).split(',') # music
mlCategory = (config.get("Mylar", "mlCategory")).split(',') # comics
gzCategory = (config.get("Gamez", "gzCategory")).split(',') # games
# NZBGet V11+
# Check if the script is called from nzbget 11.0 or later
@ -386,23 +386,23 @@ else: # only CPS supports this manual run for now.
clientAgent = "manual"
nzbDir, inputName, status, inputCategory, download_id = ('Manual Run', 'Manual Run', 0, cpsCategory, '')
if inputCategory == cpsCategory:
if inputCategory in cpsCategory:
Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName)
result = autoProcessMovie.process(nzbDir, inputName, status, clientAgent, download_id)
elif inputCategory == sbCategory:
result = autoProcessMovie.process(nzbDir, inputName, status, clientAgent, download_id, inputCategory)
elif inputCategory in sbCategory:
Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName)
result = autoProcessTV.processEpisode(nzbDir, inputName, status)
elif inputCategory == hpCategory:
result = autoProcessTV.processEpisode(nzbDir, inputName, status, inputCategory)
elif inputCategory in hpCategory:
Logger.info("MAIN: Calling HeadPhones to post-process: %s", inputName)
result = autoProcessMusic.process(nzbDir, inputName, status)
elif inputCategory == mlCategory:
result = autoProcessMusic.process(nzbDir, inputName, status, inputCategory)
elif inputCategory in mlCategory:
Logger.info("MAIN: Calling Mylar to post-process: %s", inputName)
result = autoProcessComics.processEpisode(nzbDir, inputName, status)
elif inputCategory == gzCategory:
result = autoProcessComics.processEpisode(nzbDir, inputName, status, inputCategory)
elif inputCategory in gzCategory:
Logger.info("MAIN: Calling Gamez to post-process: %s", inputName)
result = autoProcessGames.process(nzbDir, inputName, status)
result = autoProcessGames.process(nzbDir, inputName, status, inputCategory)
else:
Logger.warning("MAIN: The download category %s does not match any category defines in autoProcessMedia.cfg. Exiting.", inputCategory)
Logger.warning("MAIN: The download category %s does not match any category defined in autoProcessMedia.cfg. Exiting.", inputCategory)
sys.exit(POSTPROCESS_ERROR)
if result == 0: