Transcode patch 1 (#1627)

* Add Piping of stderr to capture transcoding failures. #1619
* Allow passing absolute nice command. #1619
* Change .cfg description for niceness
* Fix errors due to VM packages out of date (ffmpeg)
* Fix Sqlite import error on tests
* Fix Azure issues

https://developercommunity.visualstudio.com/content/problem/598264/known-issue-azure-pipelines-images-missing-sqlite3.html

autoProcessMedia.cfg.spec

@@ -36,7 +36,9 @@
 [Posix]
     ### Process priority setting for External commands (Extractor and Transcoder) on Posix (Unix/Linux/OSX) systems.
     # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process).
-    niceness = 0
+    # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). 
+    # If entering a comma separated list e.g. 'niceness = nice,4' this will be passed as 'nice 4' (Safer).
+    niceness = nice,-n0
     # Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle.
     ionice_class = 0
     # Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data.

azure-pipelines.yml

@@ -24,6 +24,42 @@ jobs:
     maxParallel: 4
 
   steps:
+  - script: |
+      # Make sure all packages are pulled from latest
+      sudo apt-get update
+
+      # Fail out if any setups fail
+      set -e
+
+      # Delete old Pythons
+      rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16
+      rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7
+      rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3
+
+      # Download new Pythons
+      azcopy --recursive \
+        --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \
+        --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15
+
+      azcopy --recursive \
+        --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \
+        --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5
+
+      azcopy --recursive \
+        --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \
+        --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2
+
+      # Install new Pythons
+      original_directory=$PWD
+      setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh)
+      for setup in $setups; do
+          chmod +x $setup;
+          cd $(dirname $setup);
+          ./$(basename $setup);
+          cd $original_directory;
+      done;
+    displayName: 'Workaround: update apt and roll back Python versions'
+  
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(python.version)'
@@ -41,10 +77,10 @@ jobs:
     displayName: 'pytest'
 
   - script: |
-        rm -rf .git
-        python cleanup.py
-        python TorrentToMedia.py
-        python nzbToMedia.py
+      rm -rf .git
+      python cleanup.py
+      python TorrentToMedia.py
+      python nzbToMedia.py
     displayName: 'Test source install cleanup'
 
   - task: PublishTestResults@2

core/__init__.py

@@ -457,7 +457,10 @@ def configure_niceness():
     with open(os.devnull, 'w') as devnull:
         try:
             subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate()
-            NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))])
+            if len(CFG['Posix']['niceness'].split(',')) > 1: #Allow passing of absolute command, not just value.
+                NICENESS.extend(CFG['Posix']['niceness'].split(','))
+            else:
+                NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))])
         except Exception:
             pass
         try:

core/transcoder.py

@@ -607,7 +607,7 @@ def extract_subs(file, newfile_path, bitbucket):
         result = 1  # set result to failed in case call fails.
         try:
             proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket)
-            proc.communicate()
+            out, err = proc.communicate()
             result = proc.returncode
         except Exception:
             logger.error('Extracting subtitle has failed')
@@ -930,17 +930,19 @@ def transcode_directory(dir_name):
         result = 1  # set result to failed in case call fails.
         try:
             if isinstance(file, string_types):
-                proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket)
+                proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE)
             else:
                 img, data = next(iteritems(file))
-                proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE)
+                proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
                 for vob in data['files']:
                     procin = zip_out(vob, img, bitbucket)
                     if procin:
                         logger.debug('Feeding in file: {0} to Transcoder'.format(vob))
                         shutil.copyfileobj(procin.stdout, proc.stdin)
                         procin.stdout.close()
-            proc.communicate()
+            out, err = proc.communicate()
+            if err:
+                logger.error('Transcoder returned:{0} has failed'.format(err))
             result = proc.returncode
         except Exception:
             logger.error('Transcoding of video {0} has failed'.format(newfile_path))

nzbToCouchPotato.py

@@ -113,7 +113,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToGamez.py

@@ -69,7 +69,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToHeadPhones.py

@@ -83,7 +83,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToLazyLibrarian.py

@@ -77,7 +77,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToLidarr.py

@@ -98,7 +98,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToMedia.py

@@ -469,7 +469,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToMylar.py

@@ -74,7 +74,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToNzbDrone.py

@@ -103,7 +103,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToRadarr.py

@@ -108,7 +108,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #

nzbToSickBeard.py

@@ -114,7 +114,9 @@
 # Niceness for external tasks Extractor and Transcoder.
 #
 # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). 
-#niceness=10
+# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). 
+# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer).
+#niceness=nice,-n0
 
 # ionice scheduling class (0, 1, 2, 3).
 #