mirror of
https://github.com/clinton-hall/nzbToMedia.git
synced 2025-07-12 16:22:53 -07:00
449 lines
24 KiB
Python
Executable file
449 lines
24 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
#System imports
|
|
import os
|
|
import sys
|
|
import ConfigParser
|
|
import shutil
|
|
import logging
|
|
import datetime
|
|
import time
|
|
import re
|
|
from sets import Set
|
|
from subprocess import call, Popen
|
|
|
|
# Custom imports
|
|
import autoProcess.migratecfg as migratecfg
|
|
import extractor.extractor as extractor
|
|
import autoProcess.autoProcessComics as autoProcessComics
|
|
import autoProcess.autoProcessGames as autoProcessGames
|
|
import autoProcess.autoProcessMusic as autoProcessMusic
|
|
import autoProcess.autoProcessTV as autoProcessTV
|
|
import autoProcess.autoProcessMovie as autoProcessMovie
|
|
from autoProcess.nzbToMediaEnv import *
|
|
from autoProcess.nzbToMediaUtil import *
|
|
from utorrent.client import UTorrentClient
|
|
from transmissionrpc.client import Client as TransmissionClient
|
|
|
|
def main(inputDirectory, inputName, inputCategory, inputHash, inputID):
|
|
|
|
status = int(1) # 1 = failed | 0 = success
|
|
root = int(0)
|
|
video = int(0)
|
|
video2 = int(0)
|
|
foundFile = int(0)
|
|
extracted_folder = []
|
|
extractionSuccess = False
|
|
copy_list = []
|
|
|
|
Logger.debug("MAIN: Received Directory: %s | Name: %s | Category: %s", inputDirectory, inputName, inputCategory)
|
|
if inputCategory == sbCategory and sbFork in SICKBEARD_TORRENT:
|
|
Logger.info("MAIN: Calling SickBeard's %s branch to post-process: %s",sbFork ,inputName)
|
|
result = autoProcessTV.processEpisode(inputDirectory, inputName, int(0))
|
|
if result == 1:
|
|
Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was pasued we will resume seeding")
|
|
Logger.info("MAIN: All done.")
|
|
sys.exit()
|
|
|
|
inputDirectory, inputName, inputCategory, root = category_search(inputDirectory, inputName, inputCategory, root, categories) # Confirm the category by parsing directory structure
|
|
|
|
outputDestination = ""
|
|
for category in categories:
|
|
if category == inputCategory:
|
|
if os.path.basename(inputDirectory) == inputName:
|
|
Logger.info("MAIN: Download is a directory")
|
|
outputDestination = os.path.normpath(os.path.join(outputDirectory, category, safeName(inputName)))
|
|
else:
|
|
Logger.info("MAIN: Download is not a directory")
|
|
outputDestination = os.path.normpath(os.path.join(outputDirectory, category, os.path.splitext(safeName(inputName))[0]))
|
|
Logger.info("MAIN: Output directory set to: %s", outputDestination)
|
|
break
|
|
else:
|
|
continue
|
|
if outputDestination == "":
|
|
if inputCategory == "":
|
|
inputCategory = "UNCAT"
|
|
if os.path.basename(inputDirectory) == inputName:
|
|
Logger.info("MAIN: Download is a directory")
|
|
outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, safeName(inputName)))
|
|
else:
|
|
Logger.info("MAIN: Download is not a directory")
|
|
outputDestination = os.path.normpath(os.path.join(outputDirectory, inputCategory, os.path.splitext(safeName(inputName))[0]))
|
|
Logger.info("MAIN: Output directory set to: %s", outputDestination)
|
|
|
|
processOnly = [cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory]
|
|
if not "NONE" in user_script_categories: # if None, we only process the 5 listed.
|
|
if "ALL" in user_script_categories: # All defined categories
|
|
processOnly = categories
|
|
processOnly.extend(user_script_categories) # Adds all categories to be processed by userscript.
|
|
|
|
if not inputCategory in processOnly:
|
|
Logger.info("MAIN: No processing to be done for category: %s. Exiting", inputCategory)
|
|
Logger.info("MAIN: All done.")
|
|
sys.exit()
|
|
|
|
# Hardlink solution for uTorrent, need to implent support for deluge, transmission
|
|
if clientAgent in ['utorrent', 'transmission'] and inputHash and useLink != "no":
|
|
if clientAgent == 'utorrent':
|
|
try:
|
|
Logger.debug("MAIN: Connecting to %s: %s", clientAgent, uTorrentWEBui)
|
|
utorrentClass = UTorrentClient(uTorrentWEBui, uTorrentUSR, uTorrentPWD)
|
|
except:
|
|
Logger.exception("MAIN: Failed to connect to uTorrent")
|
|
utorrentClass = ""
|
|
if clientAgent == 'transmission':
|
|
try:
|
|
Logger.debug("MAIN: Connecting to %s: http://%s:%s", clientAgent, TransmissionHost, TransmissionPort)
|
|
TransmissionClass = TransmissionClient(TransmissionHost, TransmissionPort, TransmissionUSR, TransmissionPWD)
|
|
except:
|
|
Logger.exception("MAIN: Failed to connect to Transmission")
|
|
TransmissionClass = ""
|
|
|
|
# if we are using links with uTorrent it means we need to pause it in order to access the files
|
|
Logger.debug("MAIN: Stoping torrent %s in %s while processing", inputName, clientAgent)
|
|
if clientAgent == 'utorrent' and utorrentClass != "":
|
|
utorrentClass.stop(inputHash)
|
|
if clientAgent == 'transmission' and TransmissionClass !="":
|
|
TransmissionClass.stop_torrent(inputID)
|
|
time.sleep(5) # Give Torrent client some time to catch up with the change
|
|
|
|
Logger.debug("MAIN: Scanning files in directory: %s", inputDirectory)
|
|
|
|
now = datetime.datetime.now()
|
|
for dirpath, dirnames, filenames in os.walk(inputDirectory):
|
|
for file in filenames:
|
|
|
|
filePath = os.path.join(dirpath, file)
|
|
fileName, fileExtension = os.path.splitext(file)
|
|
targetDirectory = os.path.join(outputDestination, file)
|
|
|
|
if root == 1:
|
|
if foundFile == int(0):
|
|
Logger.debug("MAIN: Looking for %s in: %s", inputName, file)
|
|
if (safeName(inputName) in safeName(file)) or (safeName(fileName) in safeName(inputName)):
|
|
#pass # This file does match the Torrent name
|
|
foundFile = 1
|
|
Logger.debug("MAIN: Found file %s that matches Torrent Name %s", file, inputName)
|
|
else:
|
|
continue # This file does not match the Torrent name, skip it
|
|
|
|
if root == 2:
|
|
Logger.debug("MAIN: Looking for files with modified/created dates less than 5 minutes old.")
|
|
mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(dirpath, file)))
|
|
ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, file)))
|
|
if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)):
|
|
#pass # This file does match the date time criteria
|
|
foundFile = 1
|
|
Logger.debug("MAIN: Found file %s with date modifed/created less than 5 minutes ago.", file)
|
|
else:
|
|
continue # This file has not been recently moved or created, skip it
|
|
|
|
if fileExtension in mediaContainer: # If the file is a video file
|
|
if is_sample(filePath, inputName, minSampleSize) and not inputCategory == hpCategory: # Ignore samples
|
|
Logger.info("MAIN: Ignoring sample file: %s ", filePath)
|
|
continue
|
|
else:
|
|
video = video + 1
|
|
Logger.info("MAIN: Found video file %s in %s", fileExtension, filePath)
|
|
try:
|
|
copy_link(filePath, targetDirectory, useLink, outputDestination)
|
|
copy_list.append([filePath, os.path.join(outputDestination, file)])
|
|
except:
|
|
Logger.exception("MAIN: Failed to link file: %s", file)
|
|
elif fileExtension in metaContainer:
|
|
Logger.info("MAIN: Found metadata file %s for file %s", fileExtension, filePath)
|
|
try:
|
|
copy_link(filePath, targetDirectory, useLink, outputDestination)
|
|
copy_list.append([filePath, os.path.join(outputDestination, file)])
|
|
except:
|
|
Logger.exception("MAIN: Failed to link file: %s", file)
|
|
continue
|
|
elif fileExtension in compressedContainer:
|
|
if inputCategory in [hpCategory]: # We need to link all files for HP in order to move these back to support seeding.
|
|
Logger.info("MAIN: Linking compressed archive file %s for file %s", fileExtension, filePath)
|
|
try:
|
|
copy_link(filePath, targetDirectory, useLink, outputDestination)
|
|
copy_list.append([filePath, os.path.join(outputDestination, file)])
|
|
except:
|
|
Logger.exception("MAIN: Failed to link file: %s", file)
|
|
# find part numbers in second "extension" from right, if we have more than 1 compressed file in the same directory.
|
|
if re.search(r'\d+', os.path.splitext(fileName)[1]) and os.path.dirname(filePath) in extracted_folder and not (os.path.splitext(fileName)[1] in ['.720p','.1080p']):
|
|
part = int(re.search(r'\d+', os.path.splitext(fileName)[1]).group())
|
|
if part == 1: # we only want to extract the primary part.
|
|
Logger.debug("MAIN: Found primary part of a multi-part archive %s. Extracting", file)
|
|
else:
|
|
Logger.debug("MAIN: Found part %s of a multi-part archive %s. Ignoring", part, file)
|
|
continue
|
|
Logger.info("MAIN: Found compressed archive %s for file %s", fileExtension, filePath)
|
|
try:
|
|
if inputCategory == hpCategory: # HP needs to scan the same dir as passed to downloader.
|
|
extractor.extract(filePath, inputDirectory)
|
|
else:
|
|
extractor.extract(filePath, outputDestination)
|
|
extractionSuccess = True # we use this variable to determine if we need to pause a torrent or not in uTorrent (don't need to pause archived content)
|
|
extracted_folder.append(os.path.dirname(filePath))
|
|
except:
|
|
Logger.exception("MAIN: Extraction failed for: %s", file)
|
|
continue
|
|
elif not inputCategory in [cpsCategory, sbCategory]: #process all for non-video categories.
|
|
Logger.info("MAIN: Found file %s for category %s", filePath, inputCategory)
|
|
copy_link(filePath, targetDirectory, useLink, outputDestination)
|
|
copy_list.append([filePath, os.path.join(outputDestination, file)])
|
|
continue
|
|
else:
|
|
Logger.debug("MAIN: Ignoring unknown filetype %s for file %s", fileExtension, filePath)
|
|
continue
|
|
if not inputCategory in [hpCategory]: #don't flatten hp in case multi cd albums, and we need to copy this back later.
|
|
flatten(outputDestination)
|
|
|
|
# Now check if movie files exist in destination:
|
|
if inputCategory in [cpsCategory, sbCategory]:
|
|
for dirpath, dirnames, filenames in os.walk(outputDestination):
|
|
for file in filenames:
|
|
filePath = os.path.join(dirpath, file)
|
|
fileName, fileExtension = os.path.splitext(file)
|
|
if fileExtension in mediaContainer: # If the file is a video file
|
|
if is_sample(filePath, inputName, minSampleSize):
|
|
Logger.debug("MAIN: Removing sample file: %s", filePath)
|
|
os.unlink(filePath) # remove samples
|
|
else:
|
|
Logger.debug("MAIN: Found media file: %s", filePath)
|
|
video2 = video2 + 1
|
|
else:
|
|
Logger.debug("MAIN: File %s is not a media file", filePath)
|
|
if video2 >= video and video2 > int(0): # Check that all video files were moved
|
|
Logger.debug("MAIN: Found %s media files", str(video2))
|
|
status = int(0)
|
|
else:
|
|
Logger.debug("MAIN: Found %s media files in output. %s were found in input", str(video2), str(video))
|
|
|
|
processCategories = Set([cpsCategory, sbCategory, hpCategory, mlCategory, gzCategory])
|
|
|
|
if (inputCategory in user_script_categories and not "NONE" in user_script_categories) or ("ALL" in user_script_categories and not inputCategory in processCategories):
|
|
Logger.info("MAIN: Processing user script %s.", user_script)
|
|
result = external_script(outputDestination)
|
|
elif status == int(0) or (inputCategory in [hpCategory, mlCategory, gzCategory]): # if movies linked/extracted or for other categories.
|
|
Logger.debug("MAIN: Calling autoProcess script for successful download.")
|
|
status = int(0) # hp, my, gz don't support failed.
|
|
else:
|
|
Logger.error("MAIN: Something failed! Please check logs. Exiting")
|
|
sys.exit(-1)
|
|
|
|
if inputCategory == cpsCategory:
|
|
Logger.info("MAIN: Calling CouchPotatoServer to post-process: %s", inputName)
|
|
download_id = inputHash
|
|
result = autoProcessMovie.process(outputDestination, inputName, status, clientAgent, download_id)
|
|
elif inputCategory == sbCategory:
|
|
Logger.info("MAIN: Calling Sick-Beard to post-process: %s", inputName)
|
|
result = autoProcessTV.processEpisode(outputDestination, inputName, status)
|
|
elif inputCategory == hpCategory:
|
|
Logger.info("MAIN: Calling HeadPhones to post-process: %s", inputName)
|
|
result = autoProcessMusic.process(inputDirectory, inputName, status)
|
|
elif inputCategory == mlCategory:
|
|
Logger.info("MAIN: Calling Mylar to post-process: %s", inputName)
|
|
result = autoProcessComics.processEpisode(outputDestination, inputName, status)
|
|
elif inputCategory == gzCategory:
|
|
Logger.info("MAIN: Calling Gamez to post-process: %s", inputName)
|
|
result = autoProcessGames.process(outputDestination, inputName, status)
|
|
|
|
if result == 1:
|
|
Logger.info("MAIN: A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding")
|
|
|
|
if inputCategory == hpCategory:
|
|
# we need to move the output dir files back...
|
|
Logger.debug("MAIN: Moving temporary HeadPhones files back to allow seeding.")
|
|
for item in copy_list:
|
|
if os.path.isfile(os.path.normpath(item[1])): # check to ensure temp files still exist.
|
|
if os.path.isfile(os.path.normpath(item[0])): # both exist, remove temp version
|
|
Logger.debug("MAIN: File %s still present. Removing tempoary file %s", str(item[0]), str(item[1]))
|
|
os.unlink(os.path.normpath(item[1]))
|
|
continue
|
|
else: # move temp version back to allow seeding or Torrent removal.
|
|
Logger.debug("MAIN: Moving %s to %s", str(item[1]), str(item[0]))
|
|
shutil.move(os.path.normpath(item[1]), os.path.normpath(item[0]))
|
|
continue
|
|
|
|
# Hardlink solution for uTorrent, need to implent support for deluge, transmission
|
|
if clientAgent in ['utorrent', 'transmission'] and inputHash and useLink != "no":
|
|
# Delete torrent and torrentdata from Torrent client if processing was successful.
|
|
if deleteOriginal == 1 and result != 1:
|
|
Logger.debug("MAIN: Deleting torrent %s from %s", inputName, clientAgent)
|
|
if clientAgent == 'utorrent' and utorrentClass != "":
|
|
utorrentClass.removedata(inputHash)
|
|
if not inputCategory == hpCategory:
|
|
utorrentClass.remove(inputHash)
|
|
if clientAgent == 'transmission' and TransmissionClass !="":
|
|
if inputCategory == hpCategory: #don't delete actual files for hp category, just remove torrent.
|
|
TransmissionClass.remove_torrent(inputID, False)
|
|
else:
|
|
TransmissionClass.remove_torrent(inputID, True)
|
|
# we always want to resume seeding, for now manually find out what is wrong when extraction fails
|
|
else:
|
|
Logger.debug("MAIN: Starting torrent %s in %s", inputName, clientAgent)
|
|
if clientAgent == 'utorrent' and utorrentClass != "":
|
|
utorrentClass.start(inputHash)
|
|
if clientAgent == 'transmission' and TransmissionClass !="":
|
|
TransmissionClass.start_torrent(inputID)
|
|
time.sleep(5)
|
|
#cleanup
|
|
if inputCategory in processCategories and result == 0 and os.path.isdir(outputDestination):
|
|
num_files_new = int(0)
|
|
file_list = []
|
|
for dirpath, dirnames, filenames in os.walk(outputDestination):
|
|
for file in filenames:
|
|
filePath = os.path.join(dirpath, file)
|
|
fileName, fileExtension = os.path.splitext(file)
|
|
if fileExtension in mediaContainer or fileExtension in metaContainer:
|
|
num_files_new = num_files_new + 1
|
|
file_list.append(file)
|
|
if num_files_new == int(0):
|
|
Logger.info("All files have been processed. Cleaning outputDirectory %s", outputDestination)
|
|
shutil.rmtree(outputDestination)
|
|
else:
|
|
Logger.info("outputDirectory %s still contains %s media and/or meta files. This directory will not be removed.", outputDestination, num_files_new)
|
|
for item in file_list:
|
|
Logger.debug("media/meta file found: %s", item)
|
|
Logger.info("MAIN: All done.")
|
|
|
|
def external_script(outputDestination):
|
|
|
|
result_final = int(0) # start at 0.
|
|
num_files = int(0)
|
|
for dirpath, dirnames, filenames in os.walk(outputDestination):
|
|
for file in filenames:
|
|
|
|
filePath = os.path.join(dirpath, file)
|
|
fileName, fileExtension = os.path.splitext(file)
|
|
|
|
if fileExtension in user_script_mediaExtensions or user_script_mediaExtensions == "ALL":
|
|
num_files = num_files + 1
|
|
command = [user_script]
|
|
for param in user_script_param:
|
|
if param == "FN":
|
|
command.append(file)
|
|
continue
|
|
elif param == "FP":
|
|
command.append(filePath)
|
|
continue
|
|
elif param == "DN":
|
|
command.append(dirpath)
|
|
continue
|
|
else:
|
|
command.append(param)
|
|
continue
|
|
Logger.info("Running script %s on file %s.", command, filePath)
|
|
try:
|
|
p = Popen(command)
|
|
res = p.wait()
|
|
if str(res) in user_script_successCodes: # Linux returns 0 for successful.
|
|
Logger.info("UserScript %s was successfull", command[0])
|
|
result = int(0)
|
|
else:
|
|
Logger.error("UserScript %s has failed with return code: %s", command[0], res)
|
|
Logger.info("If the UserScript completed successfully you should add %s to the user_script_successCodes", res)
|
|
result = int(1)
|
|
except:
|
|
Logger.exception("UserScript %s has failed", command[0])
|
|
result = int(1)
|
|
final_result = final_result + result
|
|
|
|
time.sleep(user_delay)
|
|
num_files_new = int(0)
|
|
for dirpath, dirnames, filenames in os.walk(outputDestination):
|
|
for file in filenames:
|
|
|
|
filePath = os.path.join(dirpath, file)
|
|
fileName, fileExtension = os.path.splitext(file)
|
|
|
|
if fileExtension in user_script_mediaExtensions or user_script_mediaExtensions == "ALL":
|
|
num_files_new = num_files_new + 1
|
|
|
|
if user_script_clean == int(1) and num_files_new == int(0) and final_result == int(0):
|
|
Logger.info("All files have been processed. Cleaning outputDirectory %s", outputDestination)
|
|
shutil.rmtree(outputDestination)
|
|
elif user_script_clean == int(1) and num_files_new != int(0):
|
|
Logger.info("%s files were processed, but %s still remain. outputDirectory will not be cleaned.", num_files, num_files_new)
|
|
return final_result
|
|
|
|
if __name__ == "__main__":
|
|
|
|
#check to migrate old cfg before trying to load.
|
|
if os.path.isfile(os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg.sample")):
|
|
migratecfg.migrate()
|
|
|
|
# Logging
|
|
nzbtomedia_configure_logging(os.path.dirname(sys.argv[0]))
|
|
Logger = logging.getLogger(__name__)
|
|
|
|
Logger.info("====================") # Seperate old from new log
|
|
Logger.info("TorrentToMedia %s", VERSION)
|
|
|
|
WakeUp()
|
|
|
|
config = ConfigParser.ConfigParser()
|
|
configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessMedia.cfg")
|
|
|
|
if not os.path.isfile(configFilename):
|
|
Logger.error("You need an autoProcessMedia.cfg file - did you rename and edit the .sample?")
|
|
sys.exit(-1)
|
|
|
|
# CONFIG FILE
|
|
Logger.info("MAIN: Loading config from %s", configFilename)
|
|
config.read(configFilename)
|
|
# EXAMPLE VALUES:
|
|
clientAgent = config.get("Torrent", "clientAgent") # utorrent | deluge | transmission | other
|
|
useLink = config.get("Torrent", "useLink") # no | hard | sym
|
|
outputDirectory = config.get("Torrent", "outputDirectory") # /abs/path/to/complete/
|
|
categories = (config.get("Torrent", "categories")).split(',') # music,music_videos,pictures,software
|
|
|
|
uTorrentWEBui = config.get("Torrent", "uTorrentWEBui") # http://localhost:8090/gui/
|
|
uTorrentUSR = config.get("Torrent", "uTorrentUSR") # mysecretusr
|
|
uTorrentPWD = config.get("Torrent", "uTorrentPWD") # mysecretpwr
|
|
|
|
TransmissionHost = config.get("Torrent", "TransmissionHost") # localhost
|
|
TransmissionPort = config.get("Torrent", "TransmissionPort") # 8084
|
|
TransmissionUSR = config.get("Torrent", "TransmissionUSR") # mysecretusr
|
|
TransmissionPWD = config.get("Torrent", "TransmissionPWD") # mysecretpwr
|
|
|
|
deleteOriginal = int(config.get("Torrent", "deleteOriginal")) # 0
|
|
|
|
compressedContainer = (config.get("Extensions", "compressedExtensions")).split(',') # .zip,.rar,.7z
|
|
mediaContainer = (config.get("Extensions", "mediaExtensions")).split(',') # .mkv,.avi,.divx
|
|
metaContainer = (config.get("Extensions", "metaExtensions")).split(',') # .nfo,.sub,.srt
|
|
minSampleSize = int(config.get("Extensions", "minSampleSize")) # 200 (in MB)
|
|
|
|
cpsCategory = config.get("CouchPotato", "cpsCategory") # movie
|
|
sbCategory = config.get("SickBeard", "sbCategory") # tv
|
|
sbFork = config.get("SickBeard", "fork") # tv
|
|
hpCategory = config.get("HeadPhones", "hpCategory") # music
|
|
mlCategory = config.get("Mylar", "mlCategory") # comics
|
|
gzCategory = config.get("Gamez", "gzCategory") # games
|
|
categories.append(cpsCategory)
|
|
categories.append(sbCategory)
|
|
categories.append(hpCategory)
|
|
categories.append(mlCategory)
|
|
categories.append(gzCategory)
|
|
|
|
user_script_categories = config.get("UserScript", "user_script_categories").split(',') # NONE
|
|
if not "NONE" in user_script_categories:
|
|
user_script_mediaExtensions = (config.get("UserScript", "user_script_mediaExtensions")).split(',')
|
|
user_script = config.get("UserScript", "user_script_path")
|
|
user_script_param = (config.get("UserScript", "user_script_param")).split(',')
|
|
user_script_successCodes = (config.get("UserScript", "user_script_successCodes")).split(',')
|
|
user_script_clean = int(config.get("UserScript", "user_script_clean"))
|
|
user_delay = int(config.get("UserScript", "delay"))
|
|
|
|
transcode = int(config.get("Transcoder", "transcode"))
|
|
|
|
n = 0
|
|
for arg in sys.argv:
|
|
Logger.debug("arg %s is: %s", n, arg)
|
|
n = n+1
|
|
|
|
try:
|
|
inputDirectory, inputName, inputCategory, inputHash, inputID = parse_args(clientAgent)
|
|
except:
|
|
Logger.exception("MAIN: There was a problem loading variables")
|
|
sys.exit(-1)
|
|
|
|
main(inputDirectory, inputName, inputCategory, inputHash, inputID)
|