add Chartranslator. Fixes #424

TorrentToMedia.py

@@ -8,6 +8,7 @@ import nzbtomedia
 
 from subprocess import Popen
 from nzbtomedia import logger, nzbToMediaDB
+from nzbtomedia.nzbToMediaUtil import convert_to_ascii
 
 def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent):
     status = 1  # 1 = failed | 0 = success
@@ -173,6 +174,8 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
     if not inputCategory in nzbtomedia.NOFLATTEN:  #don't flatten hp in case multi cd albums, and we need to copy this back later.
         nzbtomedia.flatten(outputDestination)
 
+    inputName, outputDestination = convert_to_ascii(inputName, outputDestination)
+
     if extract == 1:
         logger.debug('Checking for archives to extract in directory: %s' % (outputDestination))
         nzbtomedia.extractFiles(outputDestination)

autoProcessMedia.cfg.spec

@@ -283,7 +283,7 @@
         delay = 120
 
 [ASCII]
-    #Set convert =1 if you want to convert any "foreign" characters to ASCII before passing to SB/CP etc. Default is disabled (0).
+    #Set convert =1 if you want to convert any "foreign" characters to ASCII (UTF8) before passing to SB/CP etc. Default is disabled (0).
     convert = 0
 
 [passwords]

nzbtomedia/autoProcess/autoProcessMovie.py

@@ -91,11 +91,6 @@ class autoProcessMovie:
         return results
 
     def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", download_id="", inputCategory=None):
-        # Check video files for corruption
-        status = int(status)
-        for video in listMediaFiles(dirName):
-            if not transcoder.isVideoGood(video):
-                status = 1
 
         host = nzbtomedia.CFG[section][inputCategory]["host"]
         port = nzbtomedia.CFG[section][inputCategory]["port"]
@@ -142,9 +137,43 @@ class autoProcessMovie:
             except:
                 pass
 
+        if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name.
+            dirName = os.path.split(os.path.normpath(dirName))[0]
+
+        SpecificPath = os.path.join(dirName, str(inputName))
+        cleanName = os.path.splitext(SpecificPath)
+        if cleanName[1] == ".nzb":
+            SpecificPath = cleanName[0]
+        if os.path.isdir(SpecificPath):
+            dirName = SpecificPath
+
         process_all_exceptions(inputName.lower(), dirName)
         inputName, dirName = convert_to_ascii(inputName, dirName)
 
+        if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True):
+            logger.debug('Checking for archives to extract in directory: %s' % (dirName))
+            nzbtomedia.extractFiles(dirName)
+            inputName, dirName = convert_to_ascii(inputName, dirName)
+
+        good_files = 0
+        num_files = 0
+        # Check video files for corruption
+        status = int(status)
+        for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
+            num_files += 1
+            if transcoder.isVideoGood(video):
+                good_files += 1
+        if num_files > 0 and good_files == num_files:
+            if status:
+                logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as successful." % (str(good_files)), section)
+                status = 0
+        elif clientAgent == "manual":
+            logger.warning("No media files found in directory %s to manually process." % (dirName), section)
+            return 0  # Success (as far as this script is concerned)
+        else:
+            logger.warning("No media files found in directory %s. Processing this as a failed download" % (dirName), section)
+            status = 1
+
         if status == 0:
             if nzbtomedia.TRANSCODE == 1:
                 result = transcoder.Transcode_directory(dirName)

nzbtomedia/autoProcess/autoProcessMusic.py

@@ -56,8 +56,28 @@ class autoProcessMusic:
         else:
             protocol = "http://"
 
+        if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name.
+            dirName = os.path.split(os.path.normpath(dirName))[0]
+
+        SpecificPath = os.path.join(dirName, str(inputName))
+        cleanName = os.path.splitext(SpecificPath)
+        if cleanName[1] == ".nzb":
+            SpecificPath = cleanName[0]
+        if os.path.isdir(SpecificPath):
+            dirName = SpecificPath
+
+        process_all_exceptions(inputName.lower(), dirName)
         inputName, dirName = convert_to_ascii(inputName, dirName)
 
+        if not listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True):
+            logger.debug('Checking for archives to extract in directory: %s' % (dirName))
+            nzbtomedia.extractFiles(dirName)
+            inputName, dirName = convert_to_ascii(inputName, dirName)
+
+        if listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and status:
+            logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as successful." % (str(good_files)), section)
+            status = 0
+
         url = "%s%s:%s%s/api" % (protocol,host,port,web_root)
 
         if status == 0:

nzbtomedia/autoProcess/autoProcessTV.py

@@ -34,12 +34,6 @@ class autoProcessTV:
         # auto-detect correct fork
         fork, fork_params = autoFork(section, inputCategory)
 
-        # Check video files for corruption
-        status = int(failed)
-        for video in listMediaFiles(dirName):
-            if not transcoder.isVideoGood(video):
-                status = 1
-
         host = nzbtomedia.CFG[section][inputCategory]["host"]
         port = nzbtomedia.CFG[section][inputCategory]["port"]
         try:
@@ -91,14 +85,37 @@ class autoProcessTV:
         if os.path.isdir(SpecificPath):
             dirName = SpecificPath
 
+        # Check video files for corruption
+        status = int(failed)
+        for video in listMediaFiles(dirName):
+            if not transcoder.isVideoGood(video):
+                status = 1
+                failed = 1
+
         if fork not in nzbtomedia.SICKBEARD_TORRENT or (clientAgent in ['nzbget','sabnzbd'] and nzbExtractionBy != "Destination"):
             if inputName:
                 process_all_exceptions(inputName.lower(), dirName)
                 inputName, dirName = convert_to_ascii(inputName, dirName)
 
             # Now check if tv files exist in destination. Eventually extraction may be done here if nzbExtractionBy == TorrentToMedia
-            if listMediaFiles(dirName):  # Check that a video exists. if not, assume failed.
+            if listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):  # Check that a video exists. if not, assume failed.
                 flatten(dirName) # to make sure SickBeard can find the video (not in sub-folder)
+            elif listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True):
+                logger.debug('Checking for archives to extract in directory: %s' % (dirName))
+                nzbtomedia.extractFiles(dirName)
+                inputName, dirName = convert_to_ascii(inputName, dirName)
+                good_files = 0
+                num_files = 0
+                for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
+                    num_files += 1
+                    if transcoder.isVideoGood(video):
+                        good_files += 1
+                if num_files > 0 and good_files == num_files:
+                    status = 0
+                    failed = 0
+
+            if listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):  # Check that a video exists. if not, assume failed.
+                flatten(dirName) 
             elif clientAgent == "manual":
                 logger.warning("No media files found in directory %s to manually process." % (dirName), section)
                 return 0  # Success (as far as this script is concerned)

nzbtomedia/nzbToMediaUtil.py

@@ -308,24 +308,88 @@ def WakeUp():
         logger.info("System with mac: %s has been woken. Continuing with the rest of the script." % (mac))
 
 
+def CharReplace(fileDirName):
+    # Special character hex range:
+    # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15)
+    # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF
+    # ISO-8859-15: 0xA6-0xFF
+    # The function will detect if fileDirName contains a special character
+    # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding
+    encodingDetected = False
+    # parsing all files/directories in odrer to detect if CP850 is used
+    for Idx in range(len(fileDirName)):
+        encodingDetected = False
+        # /!\ detection is done 2char by 2char for UTF-8 special character
+        if (len(fileDirName) != 1) & (Idx < (len(fileDirName) - 1)):
+            # Detect UTF-8
+            if ((fileDirName[Idx] == '\xC2') | (fileDirName[Idx] == '\xC3')) & ((fileDirName[Idx+1] >= '\xA0') & (fileDirName[Idx+1] <= '\xFF')):
+                utf8Name = fileDirName
+                break;
+            # Detect CP850
+            elif ((fileDirName[Idx] >= '\x80') & (fileDirName[Idx] <= '\xA5')):
+                utf8Name = fileDirName.decode('cp850')
+                utf8Name = utf8Name.encode('utf-8')
+                logger.debug("Renaming CP850 encoding %s to utf8 %s" %(fileDirName, utf8Name))
+                encodingDetected = True
+                break;
+            # Detect ISO-8859-15
+            elif (fileDirName[Idx] >= '\xA6') & (fileDirName[Idx] <= '\xFF'):
+                utf8Name = fileDirName.decode('iso-8859-15')
+                utf8Name = utf8Name.encode('utf-8')
+                logger.debug("Renamed iso-8859-15 encoding %s to utf8 %s" %(fileDirName, utf8Name))
+                encodingDetected = True
+                break;
+        else:
+            # Detect CP850
+            if ((fileDirName[Idx] >= '\x80') & (fileDirName[Idx] <= '\xA5')):
+                utf8Name = fileDirName.decode('cp850')
+                utf8Name = utf8Name.encode('utf-8')
+                logger.debug("Renamed CP850 encoding %s to utf8 %s" %(fileDirName, utf8Name))
+                encodingDetected = True
+                break;
+            # Detect ISO-8859-15
+            elif (fileDirName[Idx] >= '\xA6') & (fileDirName[Idx] <= '\xFF'):
+                utf8Name = fileDirName.decode('iso-8859-15')
+                utf8Name = utf8Name.encode('utf-8')
+                logger.debug("Renamed iso-8859-15 encoding %s to utf8 %s" %(fileDirName, utf8Name))
+                encodingDetected = True
+                break;
+    if encodingDetected == False:
+        utf8Name = fileDirName
+    return encodingDetected, utf8Name
+
+
 def convert_to_ascii(inputName, dirName):
     ascii_convert = int(nzbtomedia.CFG["ASCII"]["convert"])
     if ascii_convert == 0 or os.name == 'nt':  # just return if we don't want to convert or on windows os and "\" is replaced!.
         return inputName, dirName
 
-    inputName2 = str(inputName.decode('ascii', 'replace').replace(u'\ufffd', '_'))
+    encoded, inputName = CharReplace(inputName)
-    dirName2 = str(dirName.decode('ascii', 'replace').replace(u'\ufffd', '_'))
+
-    if dirName != dirName2:
+    dir, base = os.path.split(dirName)
-        logger.info("Renaming directory:%s  to: %s." % (dirName, dirName2))
+    if not base:  # ended with "/"
-        shutil.move(dirName, dirName2)
+        dir, base = os.path.split(dir)
-    for dirpath, dirnames, filesnames in os.walk(dirName2):
+
-        for filename in filesnames:
+    encoded, base2 = CharReplace(base)
-            filename2 = str(filename.decode('ascii', 'replace').replace(u'\ufffd', '_'))
+    if encoded:
-            if filename != filename2:
+        dirName = os.path.join(dir, base2)
-                logger.info("Renaming file:%s  to: %s." % (filename, filename2))
+        logger.info("Renaming directory: %s to: %s." % (base, base2))
-                shutil.move(filename, filename2)
+        os.rename(os.path.join(dir,base), dirName)
-    inputName = inputName2
+
-    dirName = dirName2
+    for dirname, dirnames, filenames in os.walk(dirName, topdown=False):
+        for subdirname in dirnames:
+            encoded, subdirname2 = CharReplace(subdirname)
+            if encoded:
+                logger.info("Renaming directory: %s to: %s." % (subdirname, subdirname2))
+                os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2))
+
+    for dirname, dirnames, filenames in os.walk(dirName):
+        for filename in filenames:
+            encoded, filename2 = CharReplace(filename)
+            if encoded:
+                logger.info("Renaming file: %s to: %s." % (filename, filename2))
+                os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2))
+
     return inputName, dirName