From 0f08619ac5357f5f274970ad794029933f3ca174 Mon Sep 17 00:00:00 2001 From: echel0n Date: Sun, 20 Apr 2014 22:55:06 -0700 Subject: [PATCH] Added in a ffmpeg auto-installer function to our Transcoder class --- getffmpeg.sh | 31 ++- nzbtomedia/Transcoder.py | 132 ------------- nzbtomedia/__init__.py | 8 +- nzbtomedia/autoProcess/autoProcessMovie.py | 7 +- nzbtomedia/autoProcess/autoProcessTV.py | 7 +- nzbtomedia/transcoder/Transcoder.py | 219 +++++++++++++++++++++ nzbtomedia/transcoder/__init__.py | 1 + tests/general.py | 5 +- 8 files changed, 257 insertions(+), 153 deletions(-) delete mode 100644 nzbtomedia/Transcoder.py create mode 100644 nzbtomedia/transcoder/Transcoder.py create mode 100644 nzbtomedia/transcoder/__init__.py diff --git a/getffmpeg.sh b/getffmpeg.sh index a37120c0..2fb3a137 100755 --- a/getffmpeg.sh +++ b/getffmpeg.sh @@ -1,11 +1,28 @@ -#!/bin/sh +#!/bin/sh -git clone https://github.com/FFmpeg/FFmpeg.git - -cd FFmpeg - -./configure --disable-yasm +# get ffmpeg/yasm/x264 +git clone git://source.ffmpeg.org/ffmpeg.git FFmpeg +git clone git://github.com/yasm/yasm.git FFmpeg/yasm +git clone git://git.videolan.org/x264.git FFmpeg/x264 +# compile/install yasm +cd FFmpeg/yasm +./autogen.sh +./configure make - make install +cd - + +# compile/install x264 +cd FFmpeg/x264 +./configure --enable-static --enable-shared +make +make install +ldconfig +cd - + +# compile/install ffmpeg +cd FFmpeg +./configure --disable-asm --enable-libx264 --enable-gpl +make install +cd - \ No newline at end of file diff --git a/nzbtomedia/Transcoder.py b/nzbtomedia/Transcoder.py deleted file mode 100644 index 545e0743..00000000 --- a/nzbtomedia/Transcoder.py +++ /dev/null @@ -1,132 +0,0 @@ -import errno -import os -import platform -import nzbtomedia -from subprocess import call -from nzbtomedia import logger -from nzbToMediaUtil import listMediaFiles - -class Transcoder: - def isVideoGood(self, videofile): - fileNameExt = os.path.basename(videofile) - fileName, fileExt = os.path.splitext(fileNameExt) - if fileExt not in nzbtomedia.MEDIACONTAINER: - return True - - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') - - if not nzbtomedia.FFPROBE: - logger.error("Cannot detect corrupt video files!, set your ffmpeg_path in your autoProcessMedia.cfg ...", 'TRANSCODER') - return False - - command = [nzbtomedia.FFPROBE, videofile] - try: - logger.info('Checking [%s] for corruption, please stand by ...' % (fileNameExt), 'TRANSCODER') - result = call(command, stdout=bitbucket, stderr=bitbucket) - except: - logger.error("Checking [%s] for corruption has failed" % (fileNameExt), 'TRANSCODER') - return False - - if result == 0: - logger.info("SUCCESS: [%s] has no corruption." % (fileNameExt), 'TRANSCODER') - return True - else: - logger.error("FAILED: [%s] is corrupted!" % (fileNameExt), 'TRANSCODER') - return False - - def Transcode_directory(self, dirName): - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') - - if not nzbtomedia.FFMPEG: - logger.error("Cannot transcode files!, set your ffmpeg_path in your autoProcessMedia.cfg ...") - return 1 - - logger.info("Checking for files to be transcoded") - final_result = 0 # initialize as successful - for file in listMediaFiles(dirName): - name, ext = os.path.splitext(file) - if ext in [i.replace(".","") for i in nzbtomedia.MEDIACONTAINER]: - continue - if ext == nzbtomedia.OUTPUTVIDEOEXTENSION: # we need to change the name to prevent overwriting itself. - nzbtomedia.OUTPUTVIDEOEXTENSION = '-transcoded' + nzbtomedia.OUTPUTVIDEOEXTENSION # adds '-transcoded.ext' - - newfilePath = os.path.normpath(name + nzbtomedia.OUTPUTVIDEOEXTENSION) - - command = [nzbtomedia.FFMPEG, '-loglevel', 'warning', '-i', file, '-map', '0'] # -map 0 takes all input streams - if platform.system() != 'Windows': - command = ['nice', '-%d' % nzbtomedia.NICENESS] + command - - if len(nzbtomedia.OUTPUTVIDEOCODEC) > 0: - command.append('-c:v') - command.append(nzbtomedia.OUTPUTVIDEOCODEC) - if nzbtomedia.OUTPUTVIDEOCODEC == 'libx264' and nzbtomedia.OUTPUTVIDEOPRESET: - command.append('-pre') - command.append(nzbtomedia.OUTPUTVIDEOPRESET) - else: - command.append('-c:v') - command.append('copy') - if len(nzbtomedia.OUTPUTVIDEOFRAMERATE) > 0: - command.append('-r') - command.append(str(nzbtomedia.OUTPUTVIDEOFRAMERATE)) - if len(nzbtomedia.OUTPUTVIDEOBITRATE) > 0: - command.append('-b:v') - command.append(str(nzbtomedia.OUTPUTVIDEOBITRATE)) - if len(nzbtomedia.OUTPUTAUDIOCODEC) > 0: - command.append('-c:a') - command.append(nzbtomedia.OUTPUTAUDIOCODEC) - if nzbtomedia.OUTPUTAUDIOCODEC == 'aac': # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg - command.append('-strict') - command.append('-2') - else: - command.append('-c:a') - command.append('copy') - if len(nzbtomedia.OUTPUTAUDIOBITRATE) > 0: - command.append('-b:a') - command.append(str(nzbtomedia.OUTPUTAUDIOBITRATE)) - if nzbtomedia.OUTPUTFASTSTART > 0: - command.append('-movflags') - command.append('+faststart') - if nzbtomedia.OUTPUTQUALITYPERCENT > 0: - command.append('-q:a') - command.append(str(nzbtomedia.OUTPUTQUALITYPERCENT)) - if len(nzbtomedia.OUTPUTSUBTITLECODEC) > 0: # Not every subtitle codec can be used for every video container format! - command.append('-c:s') - command.append(nzbtomedia.OUTPUTSUBTITLECODEC) # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options - else: - command.append('-sn') # Don't copy the subtitles over - command.append(newfilePath) - - try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) - os.remove(newfilePath) - except OSError, e: - if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist - logger.debug("Error when removing transcoding target: %s" % (e)) - except Exception, e: - logger.debug("Error when removing transcoding target: %s" % (e)) - - logger.info("Transcoding video: %s" % (file)) - cmd = "" - for item in command: - cmd = cmd + " " + item - logger.debug("calling command:%s" % (cmd)) - result = 1 # set result to failed in case call fails. - try: - result = call(command, stdout=bitbucket, stderr=bitbucket) - except: - logger.error("Transcoding of video %s has failed" % (file)) - - if result == 0: - logger.info("Transcoding of video %s to %s succeeded" % (file, newfilePath)) - if nzbtomedia.DUPLICATE == 0: # we get rid of the original file - os.unlink(file) - else: - logger.error("Transcoding of video %s to %s failed" % (file, newfilePath)) - # this will be 0 (successful) it all are successful, else will return a positive integer for failure. - final_result = final_result + result - return final_result diff --git a/nzbtomedia/__init__.py b/nzbtomedia/__init__.py index bf674444..a573a41c 100644 --- a/nzbtomedia/__init__.py +++ b/nzbtomedia/__init__.py @@ -21,7 +21,8 @@ sys.path.insert(0, os.path.abspath(os.path.join(PROGRAM_DIR, 'lib'))) from nzbtomedia import logger, versionCheck from nzbtomedia.nzbToMediaConfig import config -from nzbtomedia.nzbToMediaUtil import WakeUp, makeDir, joinPath, cleanProcDirs, create_torrent_class +from nzbtomedia.nzbToMediaUtil import WakeUp, makeDir, joinPath, cleanProcDirs, create_torrent_class, listMediaFiles +from nzbtomedia.transcoder import transcoder # sabnzbd constants SABNZB_NO_OF_ARGUMENTS = 8 @@ -278,15 +279,12 @@ def initialize(section=None): FFMPEG = None FFPROBE = None else: - bitbucket = open('/dev/null') - FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip() FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip() if not (FFMPEG or FFPROBE): # Auto-install FFMPEG and FFPROBE - res = subprocess.call([joinPath(PROGRAM_DIR, 'getffmpeg.sh')], stdout=bitbucket, stderr=bitbucket) - if res == 0: + if transcoder.install_ffmpeg(): FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip() FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip() else: diff --git a/nzbtomedia/autoProcess/autoProcessMovie.py b/nzbtomedia/autoProcess/autoProcessMovie.py index c7b1a699..368a1837 100644 --- a/nzbtomedia/autoProcess/autoProcessMovie.py +++ b/nzbtomedia/autoProcess/autoProcessMovie.py @@ -1,11 +1,12 @@ import os import time + import nzbtomedia from lib import requests -from nzbtomedia.Transcoder import Transcoder from nzbtomedia.nzbToMediaSceneExceptions import process_all_exceptions from nzbtomedia.nzbToMediaUtil import convert_to_ascii, rmDir, find_imdbid, find_download, joinPath, listMediaFiles from nzbtomedia import logger +from nzbtomedia.transcoder import transcoder class autoProcessMovie: @@ -93,7 +94,7 @@ class autoProcessMovie: # Check video files for corruption status = int(status) for video in listMediaFiles(dirName): - if not Transcoder().isVideoGood(video): + if not transcoder.isVideoGood(video): status = 1 host = nzbtomedia.CFG[section][inputCategory]["host"] @@ -146,7 +147,7 @@ class autoProcessMovie: if status == 0: if nzbtomedia.TRANSCODE == 1: - result = Transcoder().Transcode_directory(dirName) + result = transcoder.Transcode_directory(dirName) if result == 0: logger.debug("Transcoding succeeded for files in %s" % (dirName), section) else: diff --git a/nzbtomedia/autoProcess/autoProcessTV.py b/nzbtomedia/autoProcess/autoProcessTV.py index a8f7aa1d..ea99bec3 100644 --- a/nzbtomedia/autoProcess/autoProcessTV.py +++ b/nzbtomedia/autoProcess/autoProcessTV.py @@ -1,12 +1,13 @@ import copy import os + import nzbtomedia from lib import requests -from nzbtomedia.Transcoder import Transcoder from nzbtomedia.nzbToMediaAutoFork import autoFork from nzbtomedia.nzbToMediaSceneExceptions import process_all_exceptions from nzbtomedia.nzbToMediaUtil import convert_to_ascii, flatten, rmDir, joinPath, listMediaFiles from nzbtomedia import logger +from nzbtomedia.transcoder import transcoder class autoProcessTV: def processEpisode(self, dirName, nzbName=None, failed=False, clientAgent = "manual", inputCategory=None): @@ -23,7 +24,7 @@ class autoProcessTV: # Check video files for corruption status = int(failed) for video in listMediaFiles(dirName): - if not Transcoder().isVideoGood(video): + if not transcoder.isVideoGood(video): status = 1 host = nzbtomedia.CFG[section][inputCategory]["host"] @@ -138,7 +139,7 @@ class autoProcessTV: return 0 # Success (as far as this script is concerned) if status == 0 and nzbtomedia.TRANSCODE == 1: # only transcode successful downlaods - result = Transcoder().Transcode_directory(dirName) + result = transcoder.Transcode_directory(dirName) if result == 0: logger.debug("SUCCESS: Transcoding succeeded for files in %s" % (dirName), section) else: diff --git a/nzbtomedia/transcoder/Transcoder.py b/nzbtomedia/transcoder/Transcoder.py new file mode 100644 index 00000000..154cdc7d --- /dev/null +++ b/nzbtomedia/transcoder/Transcoder.py @@ -0,0 +1,219 @@ +import errno +import os +import platform +import urllib2 +import traceback +import nzbtomedia +from subprocess import call +from nzbtomedia import logger + +def isVideoGood(videofile): + fileNameExt = os.path.basename(videofile) + fileName, fileExt = os.path.splitext(fileNameExt) + if fileExt not in nzbtomedia.MEDIACONTAINER: + return True + + if platform.system() == 'Windows': + bitbucket = open('NUL') + else: + bitbucket = open('/dev/null') + + if not nzbtomedia.FFPROBE: + logger.error("Cannot detect corrupt video files!, set your ffmpeg_path in your autoProcessMedia.cfg ...", 'TRANSCODER') + return False + + command = [nzbtomedia.FFPROBE, videofile] + try: + logger.info('Checking [%s] for corruption, please stand by ...' % (fileNameExt), 'TRANSCODER') + result = call(command, stdout=bitbucket, stderr=bitbucket) + except: + logger.error("Checking [%s] for corruption has failed" % (fileNameExt), 'TRANSCODER') + return False + + if result == 0: + logger.info("SUCCESS: [%s] has no corruption." % (fileNameExt), 'TRANSCODER') + return True + else: + logger.error("FAILED: [%s] is corrupted!" % (fileNameExt), 'TRANSCODER') + return False + +def Transcode_directory(dirName): + if platform.system() == 'Windows': + bitbucket = open('NUL') + else: + bitbucket = open('/dev/null') + + if not nzbtomedia.FFMPEG: + logger.error("Cannot transcode files!, set your ffmpeg_path in your autoProcessMedia.cfg ...") + return 1 + + logger.info("Checking for files to be transcoded") + final_result = 0 # initialize as successful + for file in nzbtomedia.listMediaFiles(dirName): + name, ext = os.path.splitext(file) + if ext in [i.replace(".","") for i in nzbtomedia.MEDIACONTAINER]: + continue + if ext == nzbtomedia.OUTPUTVIDEOEXTENSION: # we need to change the name to prevent overwriting itself. + nzbtomedia.OUTPUTVIDEOEXTENSION = '-transcoded' + nzbtomedia.OUTPUTVIDEOEXTENSION # adds '-transcoded.ext' + + newfilePath = os.path.normpath(name + nzbtomedia.OUTPUTVIDEOEXTENSION) + + command = [nzbtomedia.FFMPEG, '-loglevel', 'warning', '-i', file, '-map', '0'] # -map 0 takes all input streams + if platform.system() != 'Windows': + command = ['nice', '-%d' % nzbtomedia.NICENESS] + command + + if len(nzbtomedia.OUTPUTVIDEOCODEC) > 0: + command.append('-c:v') + command.append(nzbtomedia.OUTPUTVIDEOCODEC) + if nzbtomedia.OUTPUTVIDEOCODEC == 'libx264' and nzbtomedia.OUTPUTVIDEOPRESET: + command.append('-pre') + command.append(nzbtomedia.OUTPUTVIDEOPRESET) + else: + command.append('-c:v') + command.append('copy') + if len(nzbtomedia.OUTPUTVIDEOFRAMERATE) > 0: + command.append('-r') + command.append(str(nzbtomedia.OUTPUTVIDEOFRAMERATE)) + if len(nzbtomedia.OUTPUTVIDEOBITRATE) > 0: + command.append('-b:v') + command.append(str(nzbtomedia.OUTPUTVIDEOBITRATE)) + if len(nzbtomedia.OUTPUTAUDIOCODEC) > 0: + command.append('-c:a') + command.append(nzbtomedia.OUTPUTAUDIOCODEC) + if nzbtomedia.OUTPUTAUDIOCODEC == 'aac': # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg + command.append('-strict') + command.append('-2') + else: + command.append('-c:a') + command.append('copy') + if len(nzbtomedia.OUTPUTAUDIOBITRATE) > 0: + command.append('-b:a') + command.append(str(nzbtomedia.OUTPUTAUDIOBITRATE)) + if nzbtomedia.OUTPUTFASTSTART > 0: + command.append('-movflags') + command.append('+faststart') + if nzbtomedia.OUTPUTQUALITYPERCENT > 0: + command.append('-q:a') + command.append(str(nzbtomedia.OUTPUTQUALITYPERCENT)) + if len(nzbtomedia.OUTPUTSUBTITLECODEC) > 0: # Not every subtitle codec can be used for every video container format! + command.append('-c:s') + command.append(nzbtomedia.OUTPUTSUBTITLECODEC) # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options + else: + command.append('-sn') # Don't copy the subtitles over + command.append(newfilePath) + + try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) + os.remove(newfilePath) + except OSError, e: + if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist + logger.debug("Error when removing transcoding target: %s" % (e)) + except Exception, e: + logger.debug("Error when removing transcoding target: %s" % (e)) + + logger.info("Transcoding video: %s" % (file)) + cmd = "" + for item in command: + cmd = cmd + " " + item + logger.debug("calling command:%s" % (cmd)) + result = 1 # set result to failed in case call fails. + try: + result = call(command, stdout=bitbucket, stderr=bitbucket) + except: + logger.error("Transcoding of video %s has failed" % (file)) + + if result == 0: + logger.info("Transcoding of video %s to %s succeeded" % (file, newfilePath)) + if nzbtomedia.DUPLICATE == 0: # we get rid of the original file + os.unlink(file) + else: + logger.error("Transcoding of video %s to %s failed" % (file, newfilePath)) + # this will be 0 (successful) it all are successful, else will return a positive integer for failure. + final_result = final_result + result + return final_result + +def install_ffmpeg(): + # goto script folder + os.chdir(os.path.join(os.getcwd(), os.path.dirname(__file__))) + + # path relative to this script file + # where will the dependency packages be downloaded + DOWNLOAD_DIR = "download" + + def my_mkdir(path): + if os.path.exists(path) == False: + os.makedirs(path) + + def my_remove(path): + if os.path.exists(path): + os.remove(path) + + def my_exec(cmd): + print "[cmd] %s" % cmd + os.system(cmd) + + def prepare_package(pkg_url): + ret = True + my_mkdir(DOWNLOAD_DIR) + pkg_name = os.path.basename(urllib2.urlparse.urlparse(pkg_url).path) + pkg_fpath = os.path.join(DOWNLOAD_DIR, pkg_name) + tmp_fpath = os.path.join(DOWNLOAD_DIR, pkg_name + ".downloading") + my_remove(tmp_fpath) + if os.path.exists(pkg_fpath): + print "%s already downloaded" % pkg_name + else: + f = open(tmp_fpath, "wb") + try: + print "downloading %s from %s" % (pkg_name, pkg_url) + f.write(urllib2.urlopen(pkg_url).read()) + os.rename(tmp_fpath, pkg_fpath) + except: + traceback.print_exc() + my_remove(tmp_fpath) + exit(1) + finally: + f.close() + if ret == True: + if pkg_name.endswith(".tar.bz2"): + if os.path.exists(os.path.join(DOWNLOAD_DIR, pkg_name[:-8])) == False: + print "extracting %s" % pkg_name + my_exec("cd '%s' ; tar xjf %s" % (DOWNLOAD_DIR, pkg_name)) + if pkg_name.endswith(".tar.gz"): + if os.path.exists(os.path.join(DOWNLOAD_DIR, pkg_name[:-7])) == False: + print "extracting %s" % pkg_name + my_exec("cd '%s' ; tar xzf %s" % (DOWNLOAD_DIR, pkg_name)) + return ret + + # build yasm + prepare_package("http://www.tortall.net/projects/yasm/releases/yasm-1.2.0.tar.gz") + my_exec(""" + cd %s/yasm-1.2.0 + ./configure + make + make install + """ % (DOWNLOAD_DIR)) + + + # build x264 + prepare_package("ftp://ftp.videolan.org/pub/x264/snapshots/last_stable_x264.tar.bz2") + my_exec(""" + cd %s/x264-snapshot-20120302-2245-stable + ./configure --enable-static' + make + make install + """ % (DOWNLOAD_DIR)) + + # build ffmpeg + prepare_package("http://ffmpeg.org/releases/ffmpeg-2.2.1.tar.bz2") + my_exec(""" + cd %s/ffmpeg-0.10 + ./configure \ + --enable-gpl --enable-nonfree \ + --disable-ffserver \ + --disable-shared --enable-static \ + --extra-cflags='-static' \ + --enable-libx264 --enable-pthreads + make + make install + """ % (DOWNLOAD_DIR)) + + return True \ No newline at end of file diff --git a/nzbtomedia/transcoder/__init__.py b/nzbtomedia/transcoder/__init__.py new file mode 100644 index 00000000..1f47cffe --- /dev/null +++ b/nzbtomedia/transcoder/__init__.py @@ -0,0 +1 @@ +__author__ = 'Justin' diff --git a/tests/general.py b/tests/general.py index 3988916b..cb493d2b 100644 --- a/tests/general.py +++ b/tests/general.py @@ -4,7 +4,6 @@ from nzbtomedia.nzbToMediaUtil import extractFiles # Initialize the config nzbtomedia.initialize() -inputDirectory = "Z:\complete\movie\lego movie" -inputName = "lego movie" +inputDirectory = "Z:\complete\tv\Game.of.Thrones.S04E03.HDTV.XviD-RARBG" +inputName = "Game of Thrones - S04E03 - Breaker of Chains" -extractFiles(inputDirectory) \ No newline at end of file