From 51e790743dde75007e4e3a3b77b419740328d411 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Tue, 10 Jun 2014 14:23:25 +0930 Subject: [PATCH] add Chartranslator. Fixes #424 --- TorrentToMedia.py | 3 + autoProcessMedia.cfg.spec | 2 +- nzbtomedia/autoProcess/autoProcessMovie.py | 39 ++++++++-- nzbtomedia/autoProcess/autoProcessMusic.py | 20 +++++ nzbtomedia/autoProcess/autoProcessTV.py | 31 ++++++-- nzbtomedia/nzbToMediaUtil.py | 90 ++++++++++++++++++---- 6 files changed, 159 insertions(+), 26 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 51ede50d..ea0e04e1 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -8,6 +8,7 @@ import nzbtomedia from subprocess import Popen from nzbtomedia import logger, nzbToMediaDB +from nzbtomedia.nzbToMediaUtil import convert_to_ascii def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent): status = 1 # 1 = failed | 0 = success @@ -173,6 +174,8 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, if not inputCategory in nzbtomedia.NOFLATTEN: #don't flatten hp in case multi cd albums, and we need to copy this back later. nzbtomedia.flatten(outputDestination) + inputName, outputDestination = convert_to_ascii(inputName, outputDestination) + if extract == 1: logger.debug('Checking for archives to extract in directory: %s' % (outputDestination)) nzbtomedia.extractFiles(outputDestination) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 250cae87..647666fd 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -283,7 +283,7 @@ delay = 120 [ASCII] - #Set convert =1 if you want to convert any "foreign" characters to ASCII before passing to SB/CP etc. Default is disabled (0). + #Set convert =1 if you want to convert any "foreign" characters to ASCII (UTF8) before passing to SB/CP etc. Default is disabled (0). convert = 0 [passwords] diff --git a/nzbtomedia/autoProcess/autoProcessMovie.py b/nzbtomedia/autoProcess/autoProcessMovie.py index 06bc5469..710246cc 100644 --- a/nzbtomedia/autoProcess/autoProcessMovie.py +++ b/nzbtomedia/autoProcess/autoProcessMovie.py @@ -91,11 +91,6 @@ class autoProcessMovie: return results def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", download_id="", inputCategory=None): - # Check video files for corruption - status = int(status) - for video in listMediaFiles(dirName): - if not transcoder.isVideoGood(video): - status = 1 host = nzbtomedia.CFG[section][inputCategory]["host"] port = nzbtomedia.CFG[section][inputCategory]["port"] @@ -142,9 +137,43 @@ class autoProcessMovie: except: pass + if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name. + dirName = os.path.split(os.path.normpath(dirName))[0] + + SpecificPath = os.path.join(dirName, str(inputName)) + cleanName = os.path.splitext(SpecificPath) + if cleanName[1] == ".nzb": + SpecificPath = cleanName[0] + if os.path.isdir(SpecificPath): + dirName = SpecificPath + process_all_exceptions(inputName.lower(), dirName) inputName, dirName = convert_to_ascii(inputName, dirName) + if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True): + logger.debug('Checking for archives to extract in directory: %s' % (dirName)) + nzbtomedia.extractFiles(dirName) + inputName, dirName = convert_to_ascii(inputName, dirName) + + good_files = 0 + num_files = 0 + # Check video files for corruption + status = int(status) + for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): + num_files += 1 + if transcoder.isVideoGood(video): + good_files += 1 + if num_files > 0 and good_files == num_files: + if status: + logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as successful." % (str(good_files)), section) + status = 0 + elif clientAgent == "manual": + logger.warning("No media files found in directory %s to manually process." % (dirName), section) + return 0 # Success (as far as this script is concerned) + else: + logger.warning("No media files found in directory %s. Processing this as a failed download" % (dirName), section) + status = 1 + if status == 0: if nzbtomedia.TRANSCODE == 1: result = transcoder.Transcode_directory(dirName) diff --git a/nzbtomedia/autoProcess/autoProcessMusic.py b/nzbtomedia/autoProcess/autoProcessMusic.py index cc3d3b33..b5e9efaf 100644 --- a/nzbtomedia/autoProcess/autoProcessMusic.py +++ b/nzbtomedia/autoProcess/autoProcessMusic.py @@ -56,8 +56,28 @@ class autoProcessMusic: else: protocol = "http://" + if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name. + dirName = os.path.split(os.path.normpath(dirName))[0] + + SpecificPath = os.path.join(dirName, str(inputName)) + cleanName = os.path.splitext(SpecificPath) + if cleanName[1] == ".nzb": + SpecificPath = cleanName[0] + if os.path.isdir(SpecificPath): + dirName = SpecificPath + + process_all_exceptions(inputName.lower(), dirName) inputName, dirName = convert_to_ascii(inputName, dirName) + if not listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True): + logger.debug('Checking for archives to extract in directory: %s' % (dirName)) + nzbtomedia.extractFiles(dirName) + inputName, dirName = convert_to_ascii(inputName, dirName) + + if listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and status: + logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as successful." % (str(good_files)), section) + status = 0 + url = "%s%s:%s%s/api" % (protocol,host,port,web_root) if status == 0: diff --git a/nzbtomedia/autoProcess/autoProcessTV.py b/nzbtomedia/autoProcess/autoProcessTV.py index de17f99d..3a4dc7f8 100644 --- a/nzbtomedia/autoProcess/autoProcessTV.py +++ b/nzbtomedia/autoProcess/autoProcessTV.py @@ -34,12 +34,6 @@ class autoProcessTV: # auto-detect correct fork fork, fork_params = autoFork(section, inputCategory) - # Check video files for corruption - status = int(failed) - for video in listMediaFiles(dirName): - if not transcoder.isVideoGood(video): - status = 1 - host = nzbtomedia.CFG[section][inputCategory]["host"] port = nzbtomedia.CFG[section][inputCategory]["port"] try: @@ -91,14 +85,37 @@ class autoProcessTV: if os.path.isdir(SpecificPath): dirName = SpecificPath + # Check video files for corruption + status = int(failed) + for video in listMediaFiles(dirName): + if not transcoder.isVideoGood(video): + status = 1 + failed = 1 + if fork not in nzbtomedia.SICKBEARD_TORRENT or (clientAgent in ['nzbget','sabnzbd'] and nzbExtractionBy != "Destination"): if inputName: process_all_exceptions(inputName.lower(), dirName) inputName, dirName = convert_to_ascii(inputName, dirName) # Now check if tv files exist in destination. Eventually extraction may be done here if nzbExtractionBy == TorrentToMedia - if listMediaFiles(dirName): # Check that a video exists. if not, assume failed. + if listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): # Check that a video exists. if not, assume failed. flatten(dirName) # to make sure SickBeard can find the video (not in sub-folder) + elif listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True): + logger.debug('Checking for archives to extract in directory: %s' % (dirName)) + nzbtomedia.extractFiles(dirName) + inputName, dirName = convert_to_ascii(inputName, dirName) + good_files = 0 + num_files = 0 + for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): + num_files += 1 + if transcoder.isVideoGood(video): + good_files += 1 + if num_files > 0 and good_files == num_files: + status = 0 + failed = 0 + + if listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): # Check that a video exists. if not, assume failed. + flatten(dirName) elif clientAgent == "manual": logger.warning("No media files found in directory %s to manually process." % (dirName), section) return 0 # Success (as far as this script is concerned) diff --git a/nzbtomedia/nzbToMediaUtil.py b/nzbtomedia/nzbToMediaUtil.py index eea0a3c0..db8493ff 100644 --- a/nzbtomedia/nzbToMediaUtil.py +++ b/nzbtomedia/nzbToMediaUtil.py @@ -308,24 +308,88 @@ def WakeUp(): logger.info("System with mac: %s has been woken. Continuing with the rest of the script." % (mac)) +def CharReplace(fileDirName): + # Special character hex range: + # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) + # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF + # ISO-8859-15: 0xA6-0xFF + # The function will detect if fileDirName contains a special character + # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding + encodingDetected = False + # parsing all files/directories in odrer to detect if CP850 is used + for Idx in range(len(fileDirName)): + encodingDetected = False + # /!\ detection is done 2char by 2char for UTF-8 special character + if (len(fileDirName) != 1) & (Idx < (len(fileDirName) - 1)): + # Detect UTF-8 + if ((fileDirName[Idx] == '\xC2') | (fileDirName[Idx] == '\xC3')) & ((fileDirName[Idx+1] >= '\xA0') & (fileDirName[Idx+1] <= '\xFF')): + utf8Name = fileDirName + break; + # Detect CP850 + elif ((fileDirName[Idx] >= '\x80') & (fileDirName[Idx] <= '\xA5')): + utf8Name = fileDirName.decode('cp850') + utf8Name = utf8Name.encode('utf-8') + logger.debug("Renaming CP850 encoding %s to utf8 %s" %(fileDirName, utf8Name)) + encodingDetected = True + break; + # Detect ISO-8859-15 + elif (fileDirName[Idx] >= '\xA6') & (fileDirName[Idx] <= '\xFF'): + utf8Name = fileDirName.decode('iso-8859-15') + utf8Name = utf8Name.encode('utf-8') + logger.debug("Renamed iso-8859-15 encoding %s to utf8 %s" %(fileDirName, utf8Name)) + encodingDetected = True + break; + else: + # Detect CP850 + if ((fileDirName[Idx] >= '\x80') & (fileDirName[Idx] <= '\xA5')): + utf8Name = fileDirName.decode('cp850') + utf8Name = utf8Name.encode('utf-8') + logger.debug("Renamed CP850 encoding %s to utf8 %s" %(fileDirName, utf8Name)) + encodingDetected = True + break; + # Detect ISO-8859-15 + elif (fileDirName[Idx] >= '\xA6') & (fileDirName[Idx] <= '\xFF'): + utf8Name = fileDirName.decode('iso-8859-15') + utf8Name = utf8Name.encode('utf-8') + logger.debug("Renamed iso-8859-15 encoding %s to utf8 %s" %(fileDirName, utf8Name)) + encodingDetected = True + break; + if encodingDetected == False: + utf8Name = fileDirName + return encodingDetected, utf8Name + + def convert_to_ascii(inputName, dirName): ascii_convert = int(nzbtomedia.CFG["ASCII"]["convert"]) if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and "\" is replaced!. return inputName, dirName - inputName2 = str(inputName.decode('ascii', 'replace').replace(u'\ufffd', '_')) - dirName2 = str(dirName.decode('ascii', 'replace').replace(u'\ufffd', '_')) - if dirName != dirName2: - logger.info("Renaming directory:%s to: %s." % (dirName, dirName2)) - shutil.move(dirName, dirName2) - for dirpath, dirnames, filesnames in os.walk(dirName2): - for filename in filesnames: - filename2 = str(filename.decode('ascii', 'replace').replace(u'\ufffd', '_')) - if filename != filename2: - logger.info("Renaming file:%s to: %s." % (filename, filename2)) - shutil.move(filename, filename2) - inputName = inputName2 - dirName = dirName2 + encoded, inputName = CharReplace(inputName) + + dir, base = os.path.split(dirName) + if not base: # ended with "/" + dir, base = os.path.split(dir) + + encoded, base2 = CharReplace(base) + if encoded: + dirName = os.path.join(dir, base2) + logger.info("Renaming directory: %s to: %s." % (base, base2)) + os.rename(os.path.join(dir,base), dirName) + + for dirname, dirnames, filenames in os.walk(dirName, topdown=False): + for subdirname in dirnames: + encoded, subdirname2 = CharReplace(subdirname) + if encoded: + logger.info("Renaming directory: %s to: %s." % (subdirname, subdirname2)) + os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) + + for dirname, dirnames, filenames in os.walk(dirName): + for filename in filenames: + encoded, filename2 = CharReplace(filename) + if encoded: + logger.info("Renaming file: %s to: %s." % (filename, filename2)) + os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) + return inputName, dirName